PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file('/qemu-server/',
 106                   \&parse_vm_config,
 107                   \&write_vm_config);
 108
 109 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 110     description => "Some command save/restore state from this location.",
 111     type => 'string',
 112     maxLength => 128,
 113     optional => 1,
 114 });
 115
 116 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 117         description => "Specifies the Qemu machine type.",
 118         type => 'string',
 119         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 120         maxLength => 40,
 121         optional => 1,
 122 });
 123
 124 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 125 my $nodename_cache;
 126 sub nodename {
 127     $nodename_cache //= PVE::INotify::nodename();
 128     return $nodename_cache;
 129 }
 130
 131 my $watchdog_fmt = {
 132     model => {
 133         default_key => 1,
 134         type => 'string',
 135         enum => [qw(i6300esb ib700)],
 136         description => "Watchdog type to emulate.",
 137         default => 'i6300esb',
 138         optional => 1,
 139     },
 140     action => {
 141         type => 'string',
 142         enum => [qw(reset shutdown poweroff pause debug none)],
 143         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 144         optional => 1,
 145     },
 146 };
 147 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 148
 149 my $agent_fmt = {
 150     enabled => {
 151         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 152         type => 'boolean',
 153         default => 0,
 154         default_key => 1,
 155     },
 156     fstrim_cloned_disks => {
 157         description => "Run fstrim after moving a disk or migrating the VM.",
 158         type => 'boolean',
 159         optional => 1,
 160         default => 0
 161     },
 162     type => {
 163         description => "Select the agent type",
 164         type => 'string',
 165         default => 'virtio',
 166         optional => 1,
 167         enum => [qw(virtio isa)],
 168     },
 169 };
 170
 171 my $vga_fmt = {
 172     type => {
 173         description => "Select the VGA type.",
 174         type => 'string',
 175         default => 'std',
 176         optional => 1,
 177         default_key => 1,
 178         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 179     },
 180     memory => {
 181         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 182         type => 'integer',
 183         optional => 1,
 184         minimum => 4,
 185         maximum => 512,
 186     },
 187 };
 188
 189 my $ivshmem_fmt = {
 190     size => {
 191         type => 'integer',
 192         minimum => 1,
 193         description => "The size of the file in MB.",
 194     },
 195     name => {
 196         type => 'string',
 197         pattern => '[a-zA-Z0-9\-]+',
 198         optional => 1,
 199         format_description => 'string',
 200         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 201     },
 202 };
 203
 204 my $audio_fmt = {
 205     device => {
 206         type => 'string',
 207         enum => [qw(ich9-intel-hda intel-hda AC97)],
 208         description =>  "Configure an audio device."
 209     },
 210     driver =>  {
 211         type => 'string',
 212         enum => ['spice', 'none'],
 213         default => 'spice',
 214         optional => 1,
 215         description => "Driver backend for the audio device."
 216     },
 217 };
 218
 219 my $spice_enhancements_fmt = {
 220     foldersharing => {
 221         type => 'boolean',
 222         optional => 1,
 223         default => '0',
 224         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 225     },
 226     videostreaming =>  {
 227         type => 'string',
 228         enum => ['off', 'all', 'filter'],
 229         default => 'off',
 230         optional => 1,
 231         description => "Enable video streaming. Uses compression for detected video streams."
 232     },
 233 };
 234
 235 my $rng_fmt = {
 236     source => {
 237         type => 'string',
 238         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 239         default_key => 1,
 240         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 241             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 242             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 243             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 244             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 245             ." a hardware RNG from the host.",
 246     },
 247     max_bytes => {
 248         type => 'integer',
 249         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 250             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 251             ." `0` to disable limiting (potentially dangerous!).",
 252         optional => 1,
 253
 254         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 255         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 256         # reading from /dev/urandom
 257         default => 1024,
 258     },
 259     period => {
 260         type => 'integer',
 261         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 262             ." the guest to retrieve another 'max_bytes' of entropy.",
 263         optional => 1,
 264         default => 1000,
 265     },
 266 };
 267
 268 my $meta_info_fmt = {
 269     'ctime' => {
 270         type => 'integer',
 271         description => "The guest creation timestamp as UNIX epoch time",
 272         minimum => 0,
 273         optional => 1,
 274     },
 275     'creation-qemu' => {
 276         type => 'string',
 277         description => "The QEMU (machine) version from the time this VM was created.",
 278         pattern => '\d+(\.\d+)+',
 279         optional => 1,
 280     },
 281 };
 282
 283 my $confdesc = {
 284     onboot => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Specifies whether a VM will be started during system bootup.",
 288         default => 0,
 289     },
 290     autostart => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Automatic restart after crash (currently ignored).",
 294         default => 0,
 295     },
 296     hotplug => {
 297         optional => 1,
 298         type => 'string', format => 'pve-hotplug-features',
 299         description => "Selectively enable hotplug features. This is a comma separated list of"
 300             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 301             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 302             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 303             ." windows > 7.",
 304         default => 'network,disk,usb',
 305     },
 306     reboot => {
 307         optional => 1,
 308         type => 'boolean',
 309         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 310         default => 1,
 311     },
 312     lock => {
 313         optional => 1,
 314         type => 'string',
 315         description => "Lock/unlock the VM.",
 316         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 317     },
 318     cpulimit => {
 319         optional => 1,
 320         type => 'number',
 321         description => "Limit of CPU usage.",
 322         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 323             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 324         minimum => 0,
 325         maximum => 128,
 326         default => 0,
 327     },
 328     cpuunits => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 332         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 333             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 334             ." weights of all the other running VMs.",
 335         minimum => 1,
 336         maximum => 262144,
 337         default => 'cgroup v1: 1024, cgroup v2: 100',
 338     },
 339     memory => {
 340         optional => 1,
 341         type => 'integer',
 342         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 343             ." you use the balloon device.",
 344         minimum => 16,
 345         default => 512,
 346     },
 347     balloon => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 351         minimum => 0,
 352     },
 353     shares => {
 354         optional => 1,
 355         type => 'integer',
 356         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 357             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 358             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 359         minimum => 0,
 360         maximum => 50000,
 361         default => 1000,
 362     },
 363     keyboard => {
 364         optional => 1,
 365         type => 'string',
 366         description => "Keyboard layout for VNC server. This option is generally not required and"
 367         ." is often better handled from within the guest OS.",
 368         enum => PVE::Tools::kvmkeymaplist(),
 369         default => undef,
 370     },
 371     name => {
 372         optional => 1,
 373         type => 'string', format => 'dns-name',
 374         description => "Set a name for the VM. Only used on the configuration web interface.",
 375     },
 376     scsihw => {
 377         optional => 1,
 378         type => 'string',
 379         description => "SCSI controller model",
 380         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 381         default => 'lsi',
 382     },
 383     description => {
 384         optional => 1,
 385         type => 'string',
 386         description => "Description for the VM. Shown in the web-interface VM's summary."
 387             ." This is saved as comment inside the configuration file.",
 388         maxLength => 1024 * 8,
 389     },
 390     ostype => {
 391         optional => 1,
 392         type => 'string',
 393         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 394         description => "Specify guest operating system.",
 395         verbose_description => <<EODESC,
 396 Specify guest operating system. This is used to enable special
 397 optimization/features for specific operating systems:
 398
 399 [horizontal]
 400 other;; unspecified OS
 401 wxp;; Microsoft Windows XP
 402 w2k;; Microsoft Windows 2000
 403 w2k3;; Microsoft Windows 2003
 404 w2k8;; Microsoft Windows 2008
 405 wvista;; Microsoft Windows Vista
 406 win7;; Microsoft Windows 7
 407 win8;; Microsoft Windows 8/2012/2012r2
 408 win10;; Microsoft Windows 10/2016/2019
 409 win11;; Microsoft Windows 11/2022
 410 l24;; Linux 2.4 Kernel
 411 l26;; Linux 2.6 - 5.X Kernel
 412 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 413 EODESC
 414     },
 415     boot => {
 416         optional => 1,
 417         type => 'string', format => 'pve-qm-boot',
 418         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 419             ." key or 'legacy=' is deprecated.",
 420     },
 421     bootdisk => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-bootdisk',
 424         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 425         pattern => '(ide|sata|scsi|virtio)\d+',
 426     },
 427     smp => {
 428         optional => 1,
 429         type => 'integer',
 430         description => "The number of CPUs. Please use option -sockets instead.",
 431         minimum => 1,
 432         default => 1,
 433     },
 434     sockets => {
 435         optional => 1,
 436         type => 'integer',
 437         description => "The number of CPU sockets.",
 438         minimum => 1,
 439         default => 1,
 440     },
 441     cores => {
 442         optional => 1,
 443         type => 'integer',
 444         description => "The number of cores per socket.",
 445         minimum => 1,
 446         default => 1,
 447     },
 448     numa => {
 449         optional => 1,
 450         type => 'boolean',
 451         description => "Enable/disable NUMA.",
 452         default => 0,
 453     },
 454     hugepages => {
 455         optional => 1,
 456         type => 'string',
 457         description => "Enable/disable hugepages memory.",
 458         enum => [qw(any 2 1024)],
 459     },
 460     keephugepages => {
 461         optional => 1,
 462         type => 'boolean',
 463         default => 0,
 464         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 465             ." after VM shutdown and can be used for subsequent starts.",
 466     },
 467     vcpus => {
 468         optional => 1,
 469         type => 'integer',
 470         description => "Number of hotplugged vcpus.",
 471         minimum => 1,
 472         default => 0,
 473     },
 474     acpi => {
 475         optional => 1,
 476         type => 'boolean',
 477         description => "Enable/disable ACPI.",
 478         default => 1,
 479     },
 480     agent => {
 481         optional => 1,
 482         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 483         type => 'string',
 484         format => $agent_fmt,
 485     },
 486     kvm => {
 487         optional => 1,
 488         type => 'boolean',
 489         description => "Enable/disable KVM hardware virtualization.",
 490         default => 1,
 491     },
 492     tdf => {
 493         optional => 1,
 494         type => 'boolean',
 495         description => "Enable/disable time drift fix.",
 496         default => 0,
 497     },
 498     localtime => {
 499         optional => 1,
 500         type => 'boolean',
 501         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 502             ." the `ostype` indicates a Microsoft Windows OS.",
 503     },
 504     freeze => {
 505         optional => 1,
 506         type => 'boolean',
 507         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 508     },
 509     vga => {
 510         optional => 1,
 511         type => 'string', format => $vga_fmt,
 512         description => "Configure the VGA hardware.",
 513         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 514             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 515             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 516             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 517             ." display server. For win* OS you can select how many independent displays you want,"
 518             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 519             ." using a serial device as terminal.",
 520     },
 521     watchdog => {
 522         optional => 1,
 523         type => 'string', format => 'pve-qm-watchdog',
 524         description => "Create a virtual hardware watchdog device.",
 525         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 526             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 527             ." else the watchdog will reset the guest (or execute the respective action specified)",
 528     },
 529     startdate => {
 530         optional => 1,
 531         type => 'string',
 532         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 533         description => "Set the initial date of the real time clock. Valid format for date are:"
 534             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 535         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 536         default => 'now',
 537     },
 538     startup =>  get_standard_option('pve-startup-order'),
 539     template => {
 540         optional => 1,
 541         type => 'boolean',
 542         description => "Enable/disable Template.",
 543         default => 0,
 544     },
 545     args => {
 546         optional => 1,
 547         type => 'string',
 548         description => "Arbitrary arguments passed to kvm.",
 549         verbose_description => <<EODESCR,
 550 Arbitrary arguments passed to kvm, for example:
 551
 552 args: -no-reboot -no-hpet
 553
 554 NOTE: this option is for experts only.
 555 EODESCR
 556     },
 557     tablet => {
 558         optional => 1,
 559         type => 'boolean',
 560         default => 1,
 561         description => "Enable/disable the USB tablet device.",
 562         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 563             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 564             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 565             ." may consider disabling this to save some context switches. This is turned off by"
 566             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 567     },
 568     migrate_speed => {
 569         optional => 1,
 570         type => 'integer',
 571         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 572         minimum => 0,
 573         default => 0,
 574     },
 575     migrate_downtime => {
 576         optional => 1,
 577         type => 'number',
 578         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 579         minimum => 0,
 580         default => 0.1,
 581     },
 582     cdrom => {
 583         optional => 1,
 584         type => 'string', format => 'pve-qm-ide',
 585         typetext => '<volume>',
 586         description => "This is an alias for option -ide2",
 587     },
 588     cpu => {
 589         optional => 1,
 590         description => "Emulated CPU type.",
 591         type => 'string',
 592         format => 'pve-vm-cpu-conf',
 593     },
 594     parent => get_standard_option('pve-snapshot-name', {
 595         optional => 1,
 596         description => "Parent snapshot name. This is used internally, and should not be modified.",
 597     }),
 598     snaptime => {
 599         optional => 1,
 600         description => "Timestamp for snapshots.",
 601         type => 'integer',
 602         minimum => 0,
 603     },
 604     vmstate => {
 605         optional => 1,
 606         type => 'string', format => 'pve-volume-id',
 607         description => "Reference to a volume which stores the VM state. This is used internally"
 608             ." for snapshots.",
 609     },
 610     vmstatestorage => get_standard_option('pve-storage-id', {
 611         description => "Default storage for VM state volumes/files.",
 612         optional => 1,
 613     }),
 614     runningmachine => get_standard_option('pve-qemu-machine', {
 615         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 616             ." for snapshots.",
 617     }),
 618     runningcpu => {
 619         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 620             ." internally for snapshots.",
 621         optional => 1,
 622         type => 'string',
 623         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 624         format_description => 'QEMU -cpu parameter'
 625     },
 626     machine => get_standard_option('pve-qemu-machine'),
 627     arch => {
 628         description => "Virtual processor architecture. Defaults to the host.",
 629         optional => 1,
 630         type => 'string',
 631         enum => [qw(x86_64 aarch64)],
 632     },
 633     smbios1 => {
 634         description => "Specify SMBIOS type 1 fields.",
 635         type => 'string', format => 'pve-qm-smbios1',
 636         maxLength => 512,
 637         optional => 1,
 638     },
 639     protection => {
 640         optional => 1,
 641         type => 'boolean',
 642         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 643             ." remove disk operations.",
 644         default => 0,
 645     },
 646     bios => {
 647         optional => 1,
 648         type => 'string',
 649         enum => [ qw(seabios ovmf) ],
 650         description => "Select BIOS implementation.",
 651         default => 'seabios',
 652     },
 653     vmgenid => {
 654         type => 'string',
 655         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 656         format_description => 'UUID',
 657         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 658             ." to disable explicitly.",
 659         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 660             ." value identifier to the guest OS. This allows to notify the guest operating system"
 661             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 662             ." execution or creation from a template). The guest operating system notices the"
 663             ." change, and is then able to react as appropriate by marking its copies of"
 664             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 665             ."Note that auto-creation only works when done through API/CLI create or update methods"
 666             .", but not when manually editing the config file.",
 667         default => "1 (autogenerated)",
 668         optional => 1,
 669     },
 670     hookscript => {
 671         type => 'string',
 672         format => 'pve-volume-id',
 673         optional => 1,
 674         description => "Script that will be executed during various steps in the vms lifetime.",
 675     },
 676     ivshmem => {
 677         type => 'string',
 678         format => $ivshmem_fmt,
 679         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 680             ." the host.",
 681         optional => 1,
 682     },
 683     audio0 => {
 684         type => 'string',
 685         format => $audio_fmt,
 686         description => "Configure a audio device, useful in combination with QXL/Spice.",
 687         optional => 1
 688     },
 689     spice_enhancements => {
 690         type => 'string',
 691         format => $spice_enhancements_fmt,
 692         description => "Configure additional enhancements for SPICE.",
 693         optional => 1
 694     },
 695     tags => {
 696         type => 'string', format => 'pve-tag-list',
 697         description => 'Tags of the VM. This is only meta information.',
 698         optional => 1,
 699     },
 700     rng0 => {
 701         type => 'string',
 702         format => $rng_fmt,
 703         description => "Configure a VirtIO-based Random Number Generator.",
 704         optional => 1,
 705     },
 706     meta => {
 707         type => 'string',
 708         format => $meta_info_fmt,
 709         description => "Some (read-only) meta-information about this guest.",
 710         optional => 1,
 711     },
 712     affinity => {
 713         type => 'string', format => 'pve-cpuset',
 714         description => "List of host cores used to execute guest processes.",
 715         optional => 1,
 716     },
 717 };
 718
 719 my $cicustom_fmt = {
 720     meta => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all meta data passed to the VM via"
 724             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728     network => {
 729         type => 'string',
 730         optional => 1,
 731         description => 'Specify a custom file containing all network data passed to the VM via'
 732             .' cloud-init.',
 733         format => 'pve-volume-id',
 734         format_description => 'volume',
 735     },
 736     user => {
 737         type => 'string',
 738         optional => 1,
 739         description => 'Specify a custom file containing all user data passed to the VM via'
 740             .' cloud-init.',
 741         format => 'pve-volume-id',
 742         format_description => 'volume',
 743     },
 744     vendor => {
 745     type => 'string',
 746     optional => 1,
 747     description => 'Specify a custom file containing all vendor data passed to the VM via'
 748      .' cloud-init.',
 749     format => 'pve-volume-id',
 750     format_description => 'volume',
 751     },
 752 };
 753 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 754
 755 my $confdesc_cloudinit = {
 756     citype => {
 757         optional => 1,
 758         type => 'string',
 759         description => 'Specifies the cloud-init configuration format. The default depends on the'
 760             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 761             .' and `configdrive2` for windows.',
 762         enum => ['configdrive2', 'nocloud', 'opennebula'],
 763     },
 764     ciuser => {
 765         optional => 1,
 766         type => 'string',
 767         description => "cloud-init: User name to change ssh keys and password for instead of the"
 768             ." image's configured default user.",
 769     },
 770     cipassword => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'cloud-init: Password to assign the user. Using this is generally not'
 774             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 775             .' support hashed passwords.',
 776     },
 777     cicustom => {
 778         optional => 1,
 779         type => 'string',
 780         description => 'cloud-init: Specify custom files to replace the automatically generated'
 781             .' ones at start.',
 782         format => 'pve-qm-cicustom',
 783     },
 784     searchdomain => {
 785         optional => 1,
 786         type => 'string',
 787         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 788             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 789             .' are set.',
 790     },
 791     nameserver => {
 792         optional => 1,
 793         type => 'string', format => 'address-list',
 794         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 795             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 796             .' are set.',
 797     },
 798     sshkeys => {
 799         optional => 1,
 800         type => 'string',
 801         format => 'urlencoded',
 802         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 803     },
 804 };
 805
 806 # what about other qemu settings ?
 807 #cpu => 'string',
 808 #machine => 'string',
 809 #fda => 'file',
 810 #fdb => 'file',
 811 #mtdblock => 'file',
 812 #sd => 'file',
 813 #pflash => 'file',
 814 #snapshot => 'bool',
 815 #bootp => 'file',
 816 ##tftp => 'dir',
 817 ##smb => 'dir',
 818 #kernel => 'file',
 819 #append => 'string',
 820 #initrd => 'file',
 821 ##soundhw => 'string',
 822
 823 while (my ($k, $v) = each %$confdesc) {
 824     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 825 }
 826
 827 my $MAX_USB_DEVICES = 14;
 828 my $MAX_NETS = 32;
 829 my $MAX_SERIAL_PORTS = 4;
 830 my $MAX_PARALLEL_PORTS = 3;
 831 my $MAX_NUMA = 8;
 832
 833 my $numa_fmt = {
 834     cpus => {
 835         type => "string",
 836         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 837         description => "CPUs accessing this NUMA node.",
 838         format_description => "id[-id];...",
 839     },
 840     memory => {
 841         type => "number",
 842         description => "Amount of memory this NUMA node provides.",
 843         optional => 1,
 844     },
 845     hostnodes => {
 846         type => "string",
 847         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 848         description => "Host NUMA nodes to use.",
 849         format_description => "id[-id];...",
 850         optional => 1,
 851     },
 852     policy => {
 853         type => 'string',
 854         enum => [qw(preferred bind interleave)],
 855         description => "NUMA allocation policy.",
 856         optional => 1,
 857     },
 858 };
 859 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 860 my $numadesc = {
 861     optional => 1,
 862     type => 'string', format => $numa_fmt,
 863     description => "NUMA topology.",
 864 };
 865 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 866
 867 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 868     $confdesc->{"numa$i"} = $numadesc;
 869 }
 870
 871 my $nic_model_list = [
 872     'e1000',
 873     'e1000-82540em',
 874     'e1000-82544gc',
 875     'e1000-82545em',
 876     'e1000e',
 877     'i82551',
 878     'i82557b',
 879     'i82559er',
 880     'ne2k_isa',
 881     'ne2k_pci',
 882     'pcnet',
 883     'rtl8139',
 884     'virtio',
 885     'vmxnet3',
 886 ];
 887 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 888
 889 my $net_fmt_bridge_descr = <<__EOD__;
 890 Bridge to attach the network device to. The Proxmox VE standard bridge
 891 is called 'vmbr0'.
 892
 893 If you do not specify a bridge, we create a kvm user (NATed) network
 894 device, which provides DHCP and DNS services. The following addresses
 895 are used:
 896
 897  10.0.2.2   Gateway
 898  10.0.2.3   DNS Server
 899  10.0.2.4   SMB Server
 900
 901 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 902 __EOD__
 903
 904 my $net_fmt = {
 905     macaddr  => get_standard_option('mac-addr', {
 906         description => "MAC address. That address must be unique withing your network. This is"
 907             ." automatically generated if not specified.",
 908     }),
 909     model => {
 910         type => 'string',
 911         description => "Network Card Model. The 'virtio' model provides the best performance with"
 912             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 913             ." best to use 'e1000'.",
 914         enum => $nic_model_list,
 915         default_key => 1,
 916     },
 917     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 918     bridge => get_standard_option('pve-bridge-id', {
 919         description => $net_fmt_bridge_descr,
 920         optional => 1,
 921     }),
 922     queues => {
 923         type => 'integer',
 924         minimum => 0, maximum => 16,
 925         description => 'Number of packet queues to be used on the device.',
 926         optional => 1,
 927     },
 928     rate => {
 929         type => 'number',
 930         minimum => 0,
 931         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 932         optional => 1,
 933     },
 934     tag => {
 935         type => 'integer',
 936         minimum => 1, maximum => 4094,
 937         description => 'VLAN tag to apply to packets on this interface.',
 938         optional => 1,
 939     },
 940     trunks => {
 941         type => 'string',
 942         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 943         description => 'VLAN trunks to pass through this interface.',
 944         format_description => 'vlanid[;vlanid...]',
 945         optional => 1,
 946     },
 947     firewall => {
 948         type => 'boolean',
 949         description => 'Whether this interface should be protected by the firewall.',
 950         optional => 1,
 951     },
 952     link_down => {
 953         type => 'boolean',
 954         description => 'Whether this interface should be disconnected (like pulling the plug).',
 955         optional => 1,
 956     },
 957     mtu => {
 958         type => 'integer',
 959         minimum => 1, maximum => 65520,
 960         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 961         optional => 1,
 962     },
 963 };
 964
 965 my $netdesc = {
 966     optional => 1,
 967     type => 'string', format => $net_fmt,
 968     description => "Specify network devices.",
 969 };
 970
 971 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 972
 973 my $ipconfig_fmt = {
 974     ip => {
 975         type => 'string',
 976         format => 'pve-ipv4-config',
 977         format_description => 'IPv4Format/CIDR',
 978         description => 'IPv4 address in CIDR format.',
 979         optional => 1,
 980         default => 'dhcp',
 981     },
 982     gw => {
 983         type => 'string',
 984         format => 'ipv4',
 985         format_description => 'GatewayIPv4',
 986         description => 'Default gateway for IPv4 traffic.',
 987         optional => 1,
 988         requires => 'ip',
 989     },
 990     ip6 => {
 991         type => 'string',
 992         format => 'pve-ipv6-config',
 993         format_description => 'IPv6Format/CIDR',
 994         description => 'IPv6 address in CIDR format.',
 995         optional => 1,
 996         default => 'dhcp',
 997     },
 998     gw6 => {
 999         type => 'string',
1000         format => 'ipv6',
1001         format_description => 'GatewayIPv6',
1002         description => 'Default gateway for IPv6 traffic.',
1003         optional => 1,
1004         requires => 'ip6',
1005     },
1006 };
1007 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1008 my $ipconfigdesc = {
1009     optional => 1,
1010     type => 'string', format => 'pve-qm-ipconfig',
1011     description => <<'EODESCR',
1012 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1013
1014 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1015
1016 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1017 gateway should be provided.
1018 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1019 cloud-init 19.4 or newer.
1020
1021 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1022 dhcp on IPv4.
1023 EODESCR
1024 };
1025 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1026
1027 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1028     $confdesc->{"net$i"} = $netdesc;
1029     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1030 }
1031
1032 foreach my $key (keys %$confdesc_cloudinit) {
1033     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1034 }
1035
1036 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1037 sub pve_verify_cpuset {
1038     my ($set_text, $noerr) = @_;
1039
1040     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1041
1042     if ($@) {
1043         return if $noerr;
1044         die "unable to parse cpuset option\n";
1045     }
1046
1047     return PVE::CpuSet->new($members)->short_string();
1048 }
1049
1050 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1051 sub verify_volume_id_or_qm_path {
1052     my ($volid, $noerr) = @_;
1053
1054     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1055
1056     return verify_volume_id_or_absolute_path($volid, $noerr);
1057 }
1058
1059 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1060 sub verify_volume_id_or_absolute_path {
1061     my ($volid, $noerr) = @_;
1062
1063     return $volid if $volid =~ m|^/|;
1064
1065     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1066     if ($@) {
1067         return if $noerr;
1068         die $@;
1069     }
1070     return $volid;
1071 }
1072
1073 my $usb_fmt = {
1074     host => {
1075         default_key => 1,
1076         type => 'string', format => 'pve-qm-usb-device',
1077         format_description => 'HOSTUSBDEVICE|spice',
1078         description => <<EODESCR,
1079 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1080
1081  'bus-port(.port)*' (decimal numbers) or
1082  'vendor_id:product_id' (hexadeciaml numbers) or
1083  'spice'
1084
1085 You can use the 'lsusb -t' command to list existing usb devices.
1086
1087 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1088 machines - use with special care.
1089
1090 The value 'spice' can be used to add a usb redirection devices for spice.
1091 EODESCR
1092     },
1093     usb3 => {
1094         optional => 1,
1095         type => 'boolean',
1096         description => "Specifies whether if given host option is a USB3 device or port."
1097             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1098             ." is irrelevant (all devices are plugged into a xhci controller).",
1099         default => 0,
1100     },
1101 };
1102
1103 my $usbdesc = {
1104     optional => 1,
1105     type => 'string', format => $usb_fmt,
1106     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1107         ." l26 or windows > 7, n can be up to 14).",
1108 };
1109 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1110
1111 my $serialdesc = {
1112         optional => 1,
1113         type => 'string',
1114         pattern => '(/dev/.+|socket)',
1115         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1116         verbose_description =>  <<EODESCR,
1117 Create a serial device inside the VM (n is 0 to 3), and pass through a
1118 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1119 host side (use 'qm terminal' to open a terminal connection).
1120
1121 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1122 use with special care.
1123
1124 CAUTION: Experimental! User reported problems with this option.
1125 EODESCR
1126 };
1127
1128 my $paralleldesc= {
1129         optional => 1,
1130         type => 'string',
1131         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1132         description =>  "Map host parallel devices (n is 0 to 2).",
1133         verbose_description =>  <<EODESCR,
1134 Map host parallel devices (n is 0 to 2).
1135
1136 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1137 machines - use with special care.
1138
1139 CAUTION: Experimental! User reported problems with this option.
1140 EODESCR
1141 };
1142
1143 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1144     $confdesc->{"parallel$i"} = $paralleldesc;
1145 }
1146
1147 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1148     $confdesc->{"serial$i"} = $serialdesc;
1149 }
1150
1151 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1152     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1153 }
1154
1155 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1156     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1157 }
1158
1159 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1160     $confdesc->{"usb$i"} = $usbdesc;
1161 }
1162
1163 my $boot_fmt = {
1164     legacy => {
1165         optional => 1,
1166         default_key => 1,
1167         type => 'string',
1168         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1169                      . " Deprecated, use 'order=' instead.",
1170         pattern => '[acdn]{1,4}',
1171         format_description => "[acdn]{1,4}",
1172
1173         # note: this is also the fallback if boot: is not given at all
1174         default => 'cdn',
1175     },
1176     order => {
1177         optional => 1,
1178         type => 'string',
1179         format => 'pve-qm-bootdev-list',
1180         format_description => "device[;device...]",
1181         description => <<EODESC,
1182 The guest will attempt to boot from devices in the order they appear here.
1183
1184 Disks, optical drives and passed-through storage USB devices will be directly
1185 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1186 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1187
1188 Note that only devices in this list will be marked as bootable and thus loaded
1189 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1190 (e.g. software-raid), you need to specify all of them here.
1191
1192 Overrides the deprecated 'legacy=[acdn]*' value when given.
1193 EODESC
1194     },
1195 };
1196 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1197
1198 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1199 sub verify_bootdev {
1200     my ($dev, $noerr) = @_;
1201
1202     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1203     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1204
1205     my $check = sub {
1206         my ($base) = @_;
1207         return 0 if $dev !~ m/^$base\d+$/;
1208         return 0 if !$confdesc->{$dev};
1209         return 1;
1210     };
1211
1212     return $dev if $check->("net");
1213     return $dev if $check->("usb");
1214     return $dev if $check->("hostpci");
1215
1216     return if $noerr;
1217     die "invalid boot device '$dev'\n";
1218 }
1219
1220 sub print_bootorder {
1221     my ($devs) = @_;
1222     return "" if !@$devs;
1223     my $data = { order => join(';', @$devs) };
1224     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1225 }
1226
1227 my $kvm_api_version = 0;
1228
1229 sub kvm_version {
1230     return $kvm_api_version if $kvm_api_version;
1231
1232     open my $fh, '<', '/dev/kvm' or return;
1233
1234     # 0xae00 => KVM_GET_API_VERSION
1235     $kvm_api_version = ioctl($fh, 0xae00, 0);
1236     close($fh);
1237
1238     return $kvm_api_version;
1239 }
1240
1241 my $kvm_user_version = {};
1242 my $kvm_mtime = {};
1243
1244 sub kvm_user_version {
1245     my ($binary) = @_;
1246
1247     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1248     my $st = stat($binary);
1249
1250     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1251     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1252         $cachedmtime == $st->mtime;
1253
1254     $kvm_user_version->{$binary} = 'unknown';
1255     $kvm_mtime->{$binary} = $st->mtime;
1256
1257     my $code = sub {
1258         my $line = shift;
1259         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1260             $kvm_user_version->{$binary} = $2;
1261         }
1262     };
1263
1264     eval { run_command([$binary, '--version'], outfunc => $code); };
1265     warn $@ if $@;
1266
1267     return $kvm_user_version->{$binary};
1268
1269 }
1270 my sub extract_version {
1271     my ($machine_type, $version) = @_;
1272     $version = kvm_user_version() if !defined($version);
1273     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1274 }
1275
1276 sub kernel_has_vhost_net {
1277     return -c '/dev/vhost-net';
1278 }
1279
1280 sub option_exists {
1281     my $key = shift;
1282     return defined($confdesc->{$key});
1283 }
1284
1285 my $cdrom_path;
1286 sub get_cdrom_path {
1287
1288     return  $cdrom_path if $cdrom_path;
1289
1290     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1291     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1292     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1293 }
1294
1295 sub get_iso_path {
1296     my ($storecfg, $vmid, $cdrom) = @_;
1297
1298     if ($cdrom eq 'cdrom') {
1299         return get_cdrom_path();
1300     } elsif ($cdrom eq 'none') {
1301         return '';
1302     } elsif ($cdrom =~ m|^/|) {
1303         return $cdrom;
1304     } else {
1305         return PVE::Storage::path($storecfg, $cdrom);
1306     }
1307 }
1308
1309 # try to convert old style file names to volume IDs
1310 sub filename_to_volume_id {
1311     my ($vmid, $file, $media) = @_;
1312
1313      if (!($file eq 'none' || $file eq 'cdrom' ||
1314           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1315
1316         return if $file =~ m|/|;
1317
1318         if ($media && $media eq 'cdrom') {
1319             $file = "local:iso/$file";
1320         } else {
1321             $file = "local:$vmid/$file";
1322         }
1323     }
1324
1325     return $file;
1326 }
1327
1328 sub verify_media_type {
1329     my ($opt, $vtype, $media) = @_;
1330
1331     return if !$media;
1332
1333     my $etype;
1334     if ($media eq 'disk') {
1335         $etype = 'images';
1336     } elsif ($media eq 'cdrom') {
1337         $etype = 'iso';
1338     } else {
1339         die "internal error";
1340     }
1341
1342     return if ($vtype eq $etype);
1343
1344     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1345 }
1346
1347 sub cleanup_drive_path {
1348     my ($opt, $storecfg, $drive) = @_;
1349
1350     # try to convert filesystem paths to volume IDs
1351
1352     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1353         ($drive->{file} !~ m|^/dev/.+|) &&
1354         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1355         ($drive->{file} !~ m/^\d+$/)) {
1356         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1357         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1358             if !$vtype;
1359         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1360         verify_media_type($opt, $vtype, $drive->{media});
1361         $drive->{file} = $volid;
1362     }
1363
1364     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1365 }
1366
1367 sub parse_hotplug_features {
1368     my ($data) = @_;
1369
1370     my $res = {};
1371
1372     return $res if $data eq '0';
1373
1374     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1375
1376     foreach my $feature (PVE::Tools::split_list($data)) {
1377         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1378             $res->{$1} = 1;
1379         } else {
1380             die "invalid hotplug feature '$feature'\n";
1381         }
1382     }
1383     return $res;
1384 }
1385
1386 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1387 sub pve_verify_hotplug_features {
1388     my ($value, $noerr) = @_;
1389
1390     return $value if parse_hotplug_features($value);
1391
1392     return if $noerr;
1393
1394     die "unable to parse hotplug option\n";
1395 }
1396
1397 sub scsi_inquiry {
1398     my($fh, $noerr) = @_;
1399
1400     my $SG_IO = 0x2285;
1401     my $SG_GET_VERSION_NUM = 0x2282;
1402
1403     my $versionbuf = "\x00" x 8;
1404     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1405     if (!$ret) {
1406         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1407         return;
1408     }
1409     my $version = unpack("I", $versionbuf);
1410     if ($version < 30000) {
1411         die "scsi generic interface too old\n"  if !$noerr;
1412         return;
1413     }
1414
1415     my $buf = "\x00" x 36;
1416     my $sensebuf = "\x00" x 8;
1417     my $cmd = pack("C x3 C x1", 0x12, 36);
1418
1419     # see /usr/include/scsi/sg.h
1420     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1421
1422     my $packet = pack(
1423         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1424     );
1425
1426     $ret = ioctl($fh, $SG_IO, $packet);
1427     if (!$ret) {
1428         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1429         return;
1430     }
1431
1432     my @res = unpack($sg_io_hdr_t, $packet);
1433     if ($res[17] || $res[18]) {
1434         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1435         return;
1436     }
1437
1438     my $res = {};
1439     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1440
1441     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1442     $res->{type} &= 0x1F;
1443
1444     return $res;
1445 }
1446
1447 sub path_is_scsi {
1448     my ($path) = @_;
1449
1450     my $fh = IO::File->new("+<$path") || return;
1451     my $res = scsi_inquiry($fh, 1);
1452     close($fh);
1453
1454     return $res;
1455 }
1456
1457 sub print_tabletdevice_full {
1458     my ($conf, $arch) = @_;
1459
1460     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1461
1462     # we use uhci for old VMs because tablet driver was buggy in older qemu
1463     my $usbbus;
1464     if ($q35 || $arch eq 'aarch64') {
1465         $usbbus = 'ehci';
1466     } else {
1467         $usbbus = 'uhci';
1468     }
1469
1470     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1471 }
1472
1473 sub print_keyboarddevice_full {
1474     my ($conf, $arch) = @_;
1475
1476     return if $arch ne 'aarch64';
1477
1478     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1479 }
1480
1481 my sub get_drive_id {
1482     my ($drive) = @_;
1483     return "$drive->{interface}$drive->{index}";
1484 }
1485
1486 sub print_drivedevice_full {
1487     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1488
1489     my $device = '';
1490     my $maxdev = 0;
1491
1492     my $drive_id = get_drive_id($drive);
1493     if ($drive->{interface} eq 'virtio') {
1494         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1495         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1496         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1497     } elsif ($drive->{interface} eq 'scsi') {
1498
1499         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1500         my $unit = $drive->{index} % $maxdev;
1501         my $devicetype = 'hd';
1502         my $path = '';
1503         if (drive_is_cdrom($drive)) {
1504             $devicetype = 'cd';
1505         } else {
1506             if ($drive->{file} =~ m|^/|) {
1507                 $path = $drive->{file};
1508                 if (my $info = path_is_scsi($path)) {
1509                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1510                         $devicetype = 'block';
1511                     } elsif ($info->{type} == 1) { # tape
1512                         $devicetype = 'generic';
1513                     }
1514                 }
1515             } else {
1516                  $path = PVE::Storage::path($storecfg, $drive->{file});
1517             }
1518
1519             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1520             my $version = extract_version($machine_type, kvm_user_version());
1521             if ($path =~ m/^iscsi\:\/\// &&
1522                !min_version($version, 4, 1)) {
1523                 $devicetype = 'generic';
1524             }
1525         }
1526
1527         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1528             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1529         } else {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1531                 .",lun=$drive->{index}";
1532         }
1533         $device .= ",drive=drive-$drive_id,id=$drive_id";
1534
1535         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1536             $device .= ",rotation_rate=1";
1537         }
1538         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1539
1540     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1541         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1542         my $controller = int($drive->{index} / $maxdev);
1543         my $unit = $drive->{index} % $maxdev;
1544         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1545
1546         $device = "ide-$devicetype";
1547         if ($drive->{interface} eq 'ide') {
1548             $device .= ",bus=ide.$controller,unit=$unit";
1549         } else {
1550             $device .= ",bus=ahci$controller.$unit";
1551         }
1552         $device .= ",drive=drive-$drive_id,id=$drive_id";
1553
1554         if ($devicetype eq 'hd') {
1555             if (my $model = $drive->{model}) {
1556                 $model = URI::Escape::uri_unescape($model);
1557                 $device .= ",model=$model";
1558             }
1559             if ($drive->{ssd}) {
1560                 $device .= ",rotation_rate=1";
1561             }
1562         }
1563         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1564     } elsif ($drive->{interface} eq 'usb') {
1565         die "implement me";
1566         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1567     } else {
1568         die "unsupported interface type";
1569     }
1570
1571     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1572
1573     if (my $serial = $drive->{serial}) {
1574         $serial = URI::Escape::uri_unescape($serial);
1575         $device .= ",serial=$serial";
1576     }
1577
1578
1579     return $device;
1580 }
1581
1582 sub get_initiator_name {
1583     my $initiator;
1584
1585     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1586     while (defined(my $line = <$fh>)) {
1587         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1588         $initiator = $1;
1589         last;
1590     }
1591     $fh->close();
1592
1593     return $initiator;
1594 }
1595
1596 sub print_drive_commandline_full {
1597     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1598
1599     my $path;
1600     my $volid = $drive->{file};
1601     my $format = $drive->{format};
1602     my $drive_id = get_drive_id($drive);
1603
1604     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1605     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1606
1607     if (drive_is_cdrom($drive)) {
1608         $path = get_iso_path($storecfg, $vmid, $volid);
1609         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1610     } else {
1611         if ($storeid) {
1612             $path = PVE::Storage::path($storecfg, $volid);
1613             $format //= qemu_img_format($scfg, $volname);
1614         } else {
1615             $path = $volid;
1616             $format //= "raw";
1617         }
1618    }
1619
1620    my $is_rbd = $path =~ m/^rbd:/;
1621
1622     my $opts = '';
1623     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1624     foreach my $o (@qemu_drive_options) {
1625         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1626     }
1627
1628     # snapshot only accepts on|off
1629     if (defined($drive->{snapshot})) {
1630         my $v = $drive->{snapshot} ? 'on' : 'off';
1631         $opts .= ",snapshot=$v";
1632     }
1633
1634     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1635         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1636     }
1637
1638     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1639         my ($dir, $qmpname) = @$type;
1640         if (my $v = $drive->{"mbps$dir"}) {
1641             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1642         }
1643         if (my $v = $drive->{"mbps${dir}_max"}) {
1644             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1645         }
1646         if (my $v = $drive->{"bps${dir}_max_length"}) {
1647             $opts .= ",throttling.bps$qmpname-max-length=$v";
1648         }
1649         if (my $v = $drive->{"iops${dir}"}) {
1650             $opts .= ",throttling.iops$qmpname=$v";
1651         }
1652         if (my $v = $drive->{"iops${dir}_max"}) {
1653             $opts .= ",throttling.iops$qmpname-max=$v";
1654         }
1655         if (my $v = $drive->{"iops${dir}_max_length"}) {
1656             $opts .= ",throttling.iops$qmpname-max-length=$v";
1657         }
1658     }
1659
1660     if ($pbs_name) {
1661         $format = "rbd" if $is_rbd;
1662         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1663             if !$format;
1664         $opts .= ",format=alloc-track,file.driver=$format";
1665     } elsif ($format) {
1666         $opts .= ",format=$format";
1667     }
1668
1669     my $cache_direct = 0;
1670
1671     if (my $cache = $drive->{cache}) {
1672         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1673     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1674         $opts .= ",cache=none";
1675         $cache_direct = 1;
1676     }
1677
1678     # io_uring with cache mode writeback or writethrough on krbd will hang...
1679     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1680
1681     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1682     # sometimes, just plain disable...
1683     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1684
1685     # io_uring causes problems when used with CIFS since kernel 5.15
1686     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1687     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1688
1689     if (!$drive->{aio}) {
1690         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1691             # io_uring supports all cache modes
1692             $opts .= ",aio=io_uring";
1693         } else {
1694             # aio native works only with O_DIRECT
1695             if($cache_direct) {
1696                 $opts .= ",aio=native";
1697             } else {
1698                 $opts .= ",aio=threads";
1699             }
1700         }
1701     }
1702
1703     if (!drive_is_cdrom($drive)) {
1704         my $detectzeroes;
1705         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1706             $detectzeroes = 'off';
1707         } elsif ($drive->{discard}) {
1708             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1709         } else {
1710             # This used to be our default with discard not being specified:
1711             $detectzeroes = 'on';
1712         }
1713
1714         # note: 'detect-zeroes' works per blockdev and we want it to persist
1715         # after the alloc-track is removed, so put it on 'file' directly
1716         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1717         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1718     }
1719
1720     if ($pbs_name) {
1721         $opts .= ",backing=$pbs_name";
1722         $opts .= ",auto-remove=on";
1723     }
1724
1725     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1726     my $file_param = "file";
1727     if ($pbs_name) {
1728         # non-rbd drivers require the underlying file to be a seperate block
1729         # node, so add a second .file indirection
1730         $file_param .= ".file" if !$is_rbd;
1731         $file_param .= ".filename";
1732     }
1733     my $pathinfo = $path ? "$file_param=$path," : '';
1734
1735     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1736 }
1737
1738 sub print_pbs_blockdev {
1739     my ($pbs_conf, $pbs_name) = @_;
1740     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1741     $blockdev .= ",repository=$pbs_conf->{repository}";
1742     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1743     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1744     $blockdev .= ",archive=$pbs_conf->{archive}";
1745     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1746     return $blockdev;
1747 }
1748
1749 sub print_netdevice_full {
1750     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1751
1752     my $device = $net->{model};
1753     if ($net->{model} eq 'virtio') {
1754          $device = 'virtio-net-pci';
1755      };
1756
1757     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1758     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1759     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1760         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1761         # and out of each queue plus one config interrupt and control vector queue
1762         my $vectors = $net->{queues} * 2 + 2;
1763         $tmpstr .= ",vectors=$vectors,mq=on";
1764         if (min_version($machine_version, 7, 1)) {
1765             $tmpstr .= ",packed=on";
1766         }
1767     }
1768
1769     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1770         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1771     }
1772
1773     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1774
1775     if (my $mtu = $net->{mtu}) {
1776         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1777             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1778             if ($mtu == 1) {
1779                  $mtu = $bridge_mtu;
1780             } elsif ($mtu < 576) {
1781                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1782             } elsif ($mtu > $bridge_mtu) {
1783                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1784             }
1785             $tmpstr .= ",host_mtu=$mtu";
1786         } else {
1787             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1788         }
1789     }
1790
1791     if ($use_old_bios_files) {
1792         my $romfile;
1793         if ($device eq 'virtio-net-pci') {
1794             $romfile = 'pxe-virtio.rom';
1795         } elsif ($device eq 'e1000') {
1796             $romfile = 'pxe-e1000.rom';
1797         } elsif ($device eq 'e1000e') {
1798             $romfile = 'pxe-e1000e.rom';
1799         } elsif ($device eq 'ne2k') {
1800             $romfile = 'pxe-ne2k_pci.rom';
1801         } elsif ($device eq 'pcnet') {
1802             $romfile = 'pxe-pcnet.rom';
1803         } elsif ($device eq 'rtl8139') {
1804             $romfile = 'pxe-rtl8139.rom';
1805         }
1806         $tmpstr .= ",romfile=$romfile" if $romfile;
1807     }
1808
1809     return $tmpstr;
1810 }
1811
1812 sub print_netdev_full {
1813     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1814
1815     my $i = '';
1816     if ($netid =~ m/^net(\d+)$/) {
1817         $i = int($1);
1818     }
1819
1820     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1821
1822     my $ifname = "tap${vmid}i$i";
1823
1824     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1825     die "interface name '$ifname' is too long (max 15 character)\n"
1826         if length($ifname) >= 16;
1827
1828     my $vhostparam = '';
1829     if (is_native($arch)) {
1830         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1831     }
1832
1833     my $vmname = $conf->{name} || "vm$vmid";
1834
1835     my $netdev = "";
1836     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1837
1838     if ($net->{bridge}) {
1839         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1840             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1841     } else {
1842         $netdev = "type=user,id=$netid,hostname=$vmname";
1843     }
1844
1845     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1846
1847     return $netdev;
1848 }
1849
1850 my $vga_map = {
1851     'cirrus' => 'cirrus-vga',
1852     'std' => 'VGA',
1853     'vmware' => 'vmware-svga',
1854     'virtio' => 'virtio-vga',
1855     'virtio-gl' => 'virtio-vga-gl',
1856 };
1857
1858 sub print_vga_device {
1859     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1860
1861     my $type = $vga_map->{$vga->{type}};
1862     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1863         $type = 'virtio-gpu';
1864     }
1865     my $vgamem_mb = $vga->{memory};
1866
1867     my $max_outputs = '';
1868     if ($qxlnum) {
1869         $type = $id ? 'qxl' : 'qxl-vga';
1870
1871         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1872             # set max outputs so linux can have up to 4 qxl displays with one device
1873             if (min_version($machine_version, 4, 1)) {
1874                 $max_outputs = ",max_outputs=4";
1875             }
1876         }
1877     }
1878
1879     die "no devicetype for $vga->{type}\n" if !$type;
1880
1881     my $memory = "";
1882     if ($vgamem_mb) {
1883         if ($vga->{type} =~ /^virtio/) {
1884             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1885             $memory = ",max_hostmem=$bytes";
1886         } elsif ($qxlnum) {
1887             # from https://www.spice-space.org/multiple-monitors.html
1888             $memory = ",vgamem_mb=$vga->{memory}";
1889             my $ram = $vgamem_mb * 4;
1890             my $vram = $vgamem_mb * 2;
1891             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1892         } else {
1893             $memory = ",vgamem_mb=$vga->{memory}";
1894         }
1895     } elsif ($qxlnum && $id) {
1896         $memory = ",ram_size=67108864,vram_size=33554432";
1897     }
1898
1899     my $edidoff = "";
1900     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1901         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1902     }
1903
1904     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1905     my $vgaid = "vga" . ($id // '');
1906     my $pciaddr;
1907     if ($q35 && $vgaid eq 'vga') {
1908         # the first display uses pcie.0 bus on q35 machines
1909         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1910     } else {
1911         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1912     }
1913
1914     if ($vga->{type} eq 'virtio-gl') {
1915         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1916         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1917             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1918
1919         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1920             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1921     }
1922
1923     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1924 }
1925
1926 sub parse_number_sets {
1927     my ($set) = @_;
1928     my $res = [];
1929     foreach my $part (split(/;/, $set)) {
1930         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1931             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1932             push @$res, [ $1, $2 ];
1933         } else {
1934             die "invalid range: $part\n";
1935         }
1936     }
1937     return $res;
1938 }
1939
1940 sub parse_numa {
1941     my ($data) = @_;
1942
1943     my $res = parse_property_string($numa_fmt, $data);
1944     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1945     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1946     return $res;
1947 }
1948
1949 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1950 sub parse_net {
1951     my ($data, $disable_mac_autogen) = @_;
1952
1953     my $res = eval { parse_property_string($net_fmt, $data) };
1954     if ($@) {
1955         warn $@;
1956         return;
1957     }
1958     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1959         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1960         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1961     }
1962     return $res;
1963 }
1964
1965 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1966 sub parse_ipconfig {
1967     my ($data) = @_;
1968
1969     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1970     if ($@) {
1971         warn $@;
1972         return;
1973     }
1974
1975     if ($res->{gw} && !$res->{ip}) {
1976         warn 'gateway specified without specifying an IP address';
1977         return;
1978     }
1979     if ($res->{gw6} && !$res->{ip6}) {
1980         warn 'IPv6 gateway specified without specifying an IPv6 address';
1981         return;
1982     }
1983     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1984         warn 'gateway specified together with DHCP';
1985         return;
1986     }
1987     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1988         # gw6 + auto/dhcp
1989         warn "IPv6 gateway specified together with $res->{ip6} address";
1990         return;
1991     }
1992
1993     if (!$res->{ip} && !$res->{ip6}) {
1994         return { ip => 'dhcp', ip6 => 'dhcp' };
1995     }
1996
1997     return $res;
1998 }
1999
2000 sub print_net {
2001     my $net = shift;
2002
2003     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2004 }
2005
2006 sub add_random_macs {
2007     my ($settings) = @_;
2008
2009     foreach my $opt (keys %$settings) {
2010         next if $opt !~ m/^net(\d+)$/;
2011         my $net = parse_net($settings->{$opt});
2012         next if !$net;
2013         $settings->{$opt} = print_net($net);
2014     }
2015 }
2016
2017 sub vm_is_volid_owner {
2018     my ($storecfg, $vmid, $volid) = @_;
2019
2020     if ($volid !~  m|^/|) {
2021         my ($path, $owner);
2022         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2023         if ($owner && ($owner == $vmid)) {
2024             return 1;
2025         }
2026     }
2027
2028     return;
2029 }
2030
2031 sub vmconfig_register_unused_drive {
2032     my ($storecfg, $vmid, $conf, $drive) = @_;
2033
2034     if (drive_is_cloudinit($drive)) {
2035         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2036         warn $@ if $@;
2037         delete $conf->{cloudinit};
2038     } elsif (!drive_is_cdrom($drive)) {
2039         my $volid = $drive->{file};
2040         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2041             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2042         }
2043     }
2044 }
2045
2046 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2047 my $smbios1_fmt = {
2048     uuid => {
2049         type => 'string',
2050         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2051         format_description => 'UUID',
2052         description => "Set SMBIOS1 UUID.",
2053         optional => 1,
2054     },
2055     version => {
2056         type => 'string',
2057         pattern => '[A-Za-z0-9+\/]+={0,2}',
2058         format_description => 'Base64 encoded string',
2059         description => "Set SMBIOS1 version.",
2060         optional => 1,
2061     },
2062     serial => {
2063         type => 'string',
2064         pattern => '[A-Za-z0-9+\/]+={0,2}',
2065         format_description => 'Base64 encoded string',
2066         description => "Set SMBIOS1 serial number.",
2067         optional => 1,
2068     },
2069     manufacturer => {
2070         type => 'string',
2071         pattern => '[A-Za-z0-9+\/]+={0,2}',
2072         format_description => 'Base64 encoded string',
2073         description => "Set SMBIOS1 manufacturer.",
2074         optional => 1,
2075     },
2076     product => {
2077         type => 'string',
2078         pattern => '[A-Za-z0-9+\/]+={0,2}',
2079         format_description => 'Base64 encoded string',
2080         description => "Set SMBIOS1 product ID.",
2081         optional => 1,
2082     },
2083     sku => {
2084         type => 'string',
2085         pattern => '[A-Za-z0-9+\/]+={0,2}',
2086         format_description => 'Base64 encoded string',
2087         description => "Set SMBIOS1 SKU string.",
2088         optional => 1,
2089     },
2090     family => {
2091         type => 'string',
2092         pattern => '[A-Za-z0-9+\/]+={0,2}',
2093         format_description => 'Base64 encoded string',
2094         description => "Set SMBIOS1 family string.",
2095         optional => 1,
2096     },
2097     base64 => {
2098         type => 'boolean',
2099         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2100         optional => 1,
2101     },
2102 };
2103
2104 sub parse_smbios1 {
2105     my ($data) = @_;
2106
2107     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2108     warn $@ if $@;
2109     return $res;
2110 }
2111
2112 sub print_smbios1 {
2113     my ($smbios1) = @_;
2114     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2115 }
2116
2117 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2118
2119 sub parse_watchdog {
2120     my ($value) = @_;
2121
2122     return if !$value;
2123
2124     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2125     warn $@ if $@;
2126     return $res;
2127 }
2128
2129 sub parse_guest_agent {
2130     my ($conf) = @_;
2131
2132     return {} if !defined($conf->{agent});
2133
2134     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2135     warn $@ if $@;
2136
2137     # if the agent is disabled ignore the other potentially set properties
2138     return {} if !$res->{enabled};
2139     return $res;
2140 }
2141
2142 sub get_qga_key {
2143     my ($conf, $key) = @_;
2144     return undef if !defined($conf->{agent});
2145
2146     my $agent = parse_guest_agent($conf);
2147     return $agent->{$key};
2148 }
2149
2150 sub parse_vga {
2151     my ($value) = @_;
2152
2153     return {} if !$value;
2154     my $res = eval { parse_property_string($vga_fmt, $value) };
2155     warn $@ if $@;
2156     return $res;
2157 }
2158
2159 sub parse_rng {
2160     my ($value) = @_;
2161
2162     return if !$value;
2163
2164     my $res = eval { parse_property_string($rng_fmt, $value) };
2165     warn $@ if $@;
2166     return $res;
2167 }
2168
2169 sub parse_meta_info {
2170     my ($value) = @_;
2171
2172     return if !$value;
2173
2174     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2175     warn $@ if $@;
2176     return $res;
2177 }
2178
2179 sub new_meta_info_string {
2180     my () = @_; # for now do not allow to override any value
2181
2182     return PVE::JSONSchema::print_property_string(
2183         {
2184             'creation-qemu' => kvm_user_version(),
2185             ctime => "". int(time()),
2186         },
2187         $meta_info_fmt
2188     );
2189 }
2190
2191 sub qemu_created_version_fixups {
2192     my ($conf, $forcemachine, $kvmver) = @_;
2193
2194     my $meta = parse_meta_info($conf->{meta}) // {};
2195     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2196
2197     # check if we need to apply some handling for VMs that always use the latest machine version but
2198     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2199     # would be required (we do not want to pin machine version for non-windows OS type)
2200     if (
2201         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2202         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2203         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2204         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2205     ) {
2206         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2207         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2208             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2209             # and thus with the predictable interface naming of systemd
2210             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2211         }
2212     }
2213     return;
2214 }
2215
2216 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2217 sub verify_usb_device {
2218     my ($value, $noerr) = @_;
2219
2220     return $value if parse_usb_device($value);
2221
2222     return if $noerr;
2223
2224     die "unable to parse usb device\n";
2225 }
2226
2227 # add JSON properties for create and set function
2228 sub json_config_properties {
2229     my ($prop, $with_disk_alloc) = @_;
2230
2231     my $skip_json_config_opts = {
2232         parent => 1,
2233         snaptime => 1,
2234         vmstate => 1,
2235         runningmachine => 1,
2236         runningcpu => 1,
2237         meta => 1,
2238     };
2239
2240     foreach my $opt (keys %$confdesc) {
2241         next if $skip_json_config_opts->{$opt};
2242
2243         if ($with_disk_alloc && is_valid_drivename($opt)) {
2244             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2245         } else {
2246             $prop->{$opt} = $confdesc->{$opt};
2247         }
2248     }
2249
2250     return $prop;
2251 }
2252
2253 # Properties that we can read from an OVF file
2254 sub json_ovf_properties {
2255     my $prop = {};
2256
2257     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2258         $prop->{$device} = {
2259             type => 'string',
2260             format => 'pve-volume-id-or-absolute-path',
2261             description => "Disk image that gets imported to $device",
2262             optional => 1,
2263         };
2264     }
2265
2266     $prop->{cores} = {
2267         type => 'integer',
2268         description => "The number of CPU cores.",
2269         optional => 1,
2270     };
2271     $prop->{memory} = {
2272         type => 'integer',
2273         description => "Amount of RAM for the VM in MB.",
2274         optional => 1,
2275     };
2276     $prop->{name} = {
2277         type => 'string',
2278         description => "Name of the VM.",
2279         optional => 1,
2280     };
2281
2282     return $prop;
2283 }
2284
2285 # return copy of $confdesc_cloudinit to generate documentation
2286 sub cloudinit_config_properties {
2287
2288     return dclone($confdesc_cloudinit);
2289 }
2290
2291 sub check_type {
2292     my ($key, $value) = @_;
2293
2294     die "unknown setting '$key'\n" if !$confdesc->{$key};
2295
2296     my $type = $confdesc->{$key}->{type};
2297
2298     if (!defined($value)) {
2299         die "got undefined value\n";
2300     }
2301
2302     if ($value =~ m/[\n\r]/) {
2303         die "property contains a line feed\n";
2304     }
2305
2306     if ($type eq 'boolean') {
2307         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2308         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2309         die "type check ('boolean') failed - got '$value'\n";
2310     } elsif ($type eq 'integer') {
2311         return int($1) if $value =~ m/^(\d+)$/;
2312         die "type check ('integer') failed - got '$value'\n";
2313     } elsif ($type eq 'number') {
2314         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2315         die "type check ('number') failed - got '$value'\n";
2316     } elsif ($type eq 'string') {
2317         if (my $fmt = $confdesc->{$key}->{format}) {
2318             PVE::JSONSchema::check_format($fmt, $value);
2319             return $value;
2320         }
2321         $value =~ s/^\"(.*)\"$/$1/;
2322         return $value;
2323     } else {
2324         die "internal error"
2325     }
2326 }
2327
2328 sub destroy_vm {
2329     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2330
2331     my $conf = PVE::QemuConfig->load_config($vmid);
2332
2333     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2334
2335     if ($conf->{template}) {
2336         # check if any base image is still used by a linked clone
2337         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2338                 my ($ds, $drive) = @_;
2339                 return if drive_is_cdrom($drive);
2340
2341                 my $volid = $drive->{file};
2342                 return if !$volid || $volid =~ m|^/|;
2343
2344                 die "base volume '$volid' is still in use by linked cloned\n"
2345                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2346
2347         });
2348     }
2349
2350     my $volids = {};
2351     my $remove_owned_drive = sub {
2352         my ($ds, $drive) = @_;
2353         return if drive_is_cdrom($drive, 1);
2354
2355         my $volid = $drive->{file};
2356         return if !$volid || $volid =~ m|^/|;
2357         return if $volids->{$volid};
2358
2359         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2360         return if !$path || !$owner || ($owner != $vmid);
2361
2362         $volids->{$volid} = 1;
2363         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2364         warn "Could not remove disk '$volid', check manually: $@" if $@;
2365     };
2366
2367     # only remove disks owned by this VM (referenced in the config)
2368     my $include_opts = {
2369         include_unused => 1,
2370         extra_keys => ['vmstate'],
2371     };
2372     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2373
2374     for my $snap (values %{$conf->{snapshots}}) {
2375         next if !defined($snap->{vmstate});
2376         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2377         next if !defined($drive);
2378         $remove_owned_drive->('vmstate', $drive);
2379     }
2380
2381     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2382
2383     if ($purge_unreferenced) { # also remove unreferenced disk
2384         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2385         PVE::Storage::foreach_volid($vmdisks, sub {
2386             my ($volid, $sid, $volname, $d) = @_;
2387             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2388             warn $@ if $@;
2389         });
2390     }
2391
2392     if (defined $replacement_conf) {
2393         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2394     } else {
2395         PVE::QemuConfig->destroy_config($vmid);
2396     }
2397 }
2398
2399 sub parse_vm_config {
2400     my ($filename, $raw, $strict) = @_;
2401
2402     return if !defined($raw);
2403
2404     my $res = {
2405         digest => Digest::SHA::sha1_hex($raw),
2406         snapshots => {},
2407         pending => {},
2408         cloudinit => {},
2409     };
2410
2411     my $handle_error = sub {
2412         my ($msg) = @_;
2413
2414         if ($strict) {
2415             die $msg;
2416         } else {
2417             warn $msg;
2418         }
2419     };
2420
2421     $filename =~ m|/qemu-server/(\d+)\.conf$|
2422         || die "got strange filename '$filename'";
2423
2424     my $vmid = $1;
2425
2426     my $conf = $res;
2427     my $descr;
2428     my $section = '';
2429
2430     my @lines = split(/\n/, $raw);
2431     foreach my $line (@lines) {
2432         next if $line =~ m/^\s*$/;
2433
2434         if ($line =~ m/^\[PENDING\]\s*$/i) {
2435             $section = 'pending';
2436             if (defined($descr)) {
2437                 $descr =~ s/\s+$//;
2438                 $conf->{description} = $descr;
2439             }
2440             $descr = undef;
2441             $conf = $res->{$section} = {};
2442             next;
2443         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2444             $section = 'cloudinit';
2445             $descr = undef;
2446             $conf = $res->{$section} = {};
2447             next;
2448
2449         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2450             $section = $1;
2451             if (defined($descr)) {
2452                 $descr =~ s/\s+$//;
2453                 $conf->{description} = $descr;
2454             }
2455             $descr = undef;
2456             $conf = $res->{snapshots}->{$section} = {};
2457             next;
2458         }
2459
2460         if ($line =~ m/^\#(.*)$/) {
2461             $descr = '' if !defined($descr);
2462             $descr .= PVE::Tools::decode_text($1) . "\n";
2463             next;
2464         }
2465
2466         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2467             $descr = '' if !defined($descr);
2468             $descr .= PVE::Tools::decode_text($2);
2469         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2470             $conf->{snapstate} = $1;
2471         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2472             my $key = $1;
2473             my $value = $2;
2474             $conf->{$key} = $value;
2475         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2476             my $value = $1;
2477             if ($section eq 'pending') {
2478                 $conf->{delete} = $value; # we parse this later
2479             } else {
2480                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2481             }
2482         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2483             my $key = $1;
2484             my $value = $2;
2485             eval { $value = check_type($key, $value); };
2486             if ($@) {
2487                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2488             } else {
2489                 $key = 'ide2' if $key eq 'cdrom';
2490                 my $fmt = $confdesc->{$key}->{format};
2491                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2492                     my $v = parse_drive($key, $value);
2493                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2494                         $v->{file} = $volid;
2495                         $value = print_drive($v);
2496                     } else {
2497                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2498                         next;
2499                     }
2500                 }
2501
2502                 $conf->{$key} = $value;
2503             }
2504         } else {
2505             $handle_error->("vm $vmid - unable to parse config: $line\n");
2506         }
2507     }
2508
2509     if (defined($descr)) {
2510         $descr =~ s/\s+$//;
2511         $conf->{description} = $descr;
2512     }
2513     delete $res->{snapstate}; # just to be sure
2514
2515     return $res;
2516 }
2517
2518 sub write_vm_config {
2519     my ($filename, $conf) = @_;
2520
2521     delete $conf->{snapstate}; # just to be sure
2522
2523     if ($conf->{cdrom}) {
2524         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2525         $conf->{ide2} = $conf->{cdrom};
2526         delete $conf->{cdrom};
2527     }
2528
2529     # we do not use 'smp' any longer
2530     if ($conf->{sockets}) {
2531         delete $conf->{smp};
2532     } elsif ($conf->{smp}) {
2533         $conf->{sockets} = $conf->{smp};
2534         delete $conf->{cores};
2535         delete $conf->{smp};
2536     }
2537
2538     my $used_volids = {};
2539
2540     my $cleanup_config = sub {
2541         my ($cref, $pending, $snapname) = @_;
2542
2543         foreach my $key (keys %$cref) {
2544             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2545                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2546             my $value = $cref->{$key};
2547             if ($key eq 'delete') {
2548                 die "propertry 'delete' is only allowed in [PENDING]\n"
2549                     if !$pending;
2550                 # fixme: check syntax?
2551                 next;
2552             }
2553             eval { $value = check_type($key, $value); };
2554             die "unable to parse value of '$key' - $@" if $@;
2555
2556             $cref->{$key} = $value;
2557
2558             if (!$snapname && is_valid_drivename($key)) {
2559                 my $drive = parse_drive($key, $value);
2560                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2561             }
2562         }
2563     };
2564
2565     &$cleanup_config($conf);
2566
2567     &$cleanup_config($conf->{pending}, 1);
2568
2569     &$cleanup_config($conf->{cloudinit});
2570
2571     foreach my $snapname (keys %{$conf->{snapshots}}) {
2572         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2573         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2574     }
2575
2576     # remove 'unusedX' settings if we re-add a volume
2577     foreach my $key (keys %$conf) {
2578         my $value = $conf->{$key};
2579         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2580             delete $conf->{$key};
2581         }
2582     }
2583
2584     my $generate_raw_config = sub {
2585         my ($conf, $pending) = @_;
2586
2587         my $raw = '';
2588
2589         # add description as comment to top of file
2590         if (defined(my $descr = $conf->{description})) {
2591             if ($descr) {
2592                 foreach my $cl (split(/\n/, $descr)) {
2593                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2594                 }
2595             } else {
2596                 $raw .= "#\n" if $pending;
2597             }
2598         }
2599
2600         foreach my $key (sort keys %$conf) {
2601             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2602             $raw .= "$key: $conf->{$key}\n";
2603         }
2604         return $raw;
2605     };
2606
2607     my $raw = &$generate_raw_config($conf);
2608
2609     if (scalar(keys %{$conf->{pending}})){
2610         $raw .= "\n[PENDING]\n";
2611         $raw .= &$generate_raw_config($conf->{pending}, 1);
2612     }
2613
2614     if (scalar(keys %{$conf->{cloudinit}})){
2615         $raw .= "\n[special:cloudinit]\n";
2616         $raw .= &$generate_raw_config($conf->{cloudinit});
2617     }
2618
2619     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2620         $raw .= "\n[$snapname]\n";
2621         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2622     }
2623
2624     return $raw;
2625 }
2626
2627 sub load_defaults {
2628
2629     my $res = {};
2630
2631     # we use static defaults from our JSON schema configuration
2632     foreach my $key (keys %$confdesc) {
2633         if (defined(my $default = $confdesc->{$key}->{default})) {
2634             $res->{$key} = $default;
2635         }
2636     }
2637
2638     return $res;
2639 }
2640
2641 sub config_list {
2642     my $vmlist = PVE::Cluster::get_vmlist();
2643     my $res = {};
2644     return $res if !$vmlist || !$vmlist->{ids};
2645     my $ids = $vmlist->{ids};
2646     my $nodename = nodename();
2647
2648     foreach my $vmid (keys %$ids) {
2649         my $d = $ids->{$vmid};
2650         next if !$d->{node} || $d->{node} ne $nodename;
2651         next if !$d->{type} || $d->{type} ne 'qemu';
2652         $res->{$vmid}->{exists} = 1;
2653     }
2654     return $res;
2655 }
2656
2657 # test if VM uses local resources (to prevent migration)
2658 sub check_local_resources {
2659     my ($conf, $noerr) = @_;
2660
2661     my @loc_res = ();
2662
2663     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2664     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2665
2666     push @loc_res, "ivshmem" if $conf->{ivshmem};
2667
2668     foreach my $k (keys %$conf) {
2669         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2670         # sockets are safe: they will recreated be on the target side post-migrate
2671         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2672         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2673     }
2674
2675     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2676
2677     return \@loc_res;
2678 }
2679
2680 # check if used storages are available on all nodes (use by migrate)
2681 sub check_storage_availability {
2682     my ($storecfg, $conf, $node) = @_;
2683
2684     PVE::QemuConfig->foreach_volume($conf, sub {
2685         my ($ds, $drive) = @_;
2686
2687         my $volid = $drive->{file};
2688         return if !$volid;
2689
2690         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2691         return if !$sid;
2692
2693         # check if storage is available on both nodes
2694         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2695         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2696
2697         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2698
2699         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2700             if !$scfg->{content}->{$vtype};
2701    });
2702 }
2703
2704 # list nodes where all VM images are available (used by has_feature API)
2705 sub shared_nodes {
2706     my ($conf, $storecfg) = @_;
2707
2708     my $nodelist = PVE::Cluster::get_nodelist();
2709     my $nodehash = { map { $_ => 1 } @$nodelist };
2710     my $nodename = nodename();
2711
2712     PVE::QemuConfig->foreach_volume($conf, sub {
2713         my ($ds, $drive) = @_;
2714
2715         my $volid = $drive->{file};
2716         return if !$volid;
2717
2718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2719         if ($storeid) {
2720             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2721             if ($scfg->{disable}) {
2722                 $nodehash = {};
2723             } elsif (my $avail = $scfg->{nodes}) {
2724                 foreach my $node (keys %$nodehash) {
2725                     delete $nodehash->{$node} if !$avail->{$node};
2726                 }
2727             } elsif (!$scfg->{shared}) {
2728                 foreach my $node (keys %$nodehash) {
2729                     delete $nodehash->{$node} if $node ne $nodename
2730                 }
2731             }
2732         }
2733     });
2734
2735     return $nodehash
2736 }
2737
2738 sub check_local_storage_availability {
2739     my ($conf, $storecfg) = @_;
2740
2741     my $nodelist = PVE::Cluster::get_nodelist();
2742     my $nodehash = { map { $_ => {} } @$nodelist };
2743
2744     PVE::QemuConfig->foreach_volume($conf, sub {
2745         my ($ds, $drive) = @_;
2746
2747         my $volid = $drive->{file};
2748         return if !$volid;
2749
2750         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2751         if ($storeid) {
2752             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2753
2754             if ($scfg->{disable}) {
2755                 foreach my $node (keys %$nodehash) {
2756                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2757                 }
2758             } elsif (my $avail = $scfg->{nodes}) {
2759                 foreach my $node (keys %$nodehash) {
2760                     if (!$avail->{$node}) {
2761                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2762                     }
2763                 }
2764             }
2765         }
2766     });
2767
2768     foreach my $node (values %$nodehash) {
2769         if (my $unavail = $node->{unavailable_storages}) {
2770             $node->{unavailable_storages} = [ sort keys %$unavail ];
2771         }
2772     }
2773
2774     return $nodehash
2775 }
2776
2777 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2778 sub check_running {
2779     my ($vmid, $nocheck, $node) = @_;
2780
2781     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2782     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2783 }
2784
2785 sub vzlist {
2786
2787     my $vzlist = config_list();
2788
2789     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2790
2791     while (defined(my $de = $fd->read)) {
2792         next if $de !~ m/^(\d+)\.pid$/;
2793         my $vmid = $1;
2794         next if !defined($vzlist->{$vmid});
2795         if (my $pid = check_running($vmid)) {
2796             $vzlist->{$vmid}->{pid} = $pid;
2797         }
2798     }
2799
2800     return $vzlist;
2801 }
2802
2803 our $vmstatus_return_properties = {
2804     vmid => get_standard_option('pve-vmid'),
2805     status => {
2806         description => "Qemu process status.",
2807         type => 'string',
2808         enum => ['stopped', 'running'],
2809     },
2810     maxmem => {
2811         description => "Maximum memory in bytes.",
2812         type => 'integer',
2813         optional => 1,
2814         renderer => 'bytes',
2815     },
2816     maxdisk => {
2817         description => "Root disk size in bytes.",
2818         type => 'integer',
2819         optional => 1,
2820         renderer => 'bytes',
2821     },
2822     name => {
2823         description => "VM name.",
2824         type => 'string',
2825         optional => 1,
2826     },
2827     qmpstatus => {
2828         description => "Qemu QMP agent status.",
2829         type => 'string',
2830         optional => 1,
2831     },
2832     pid => {
2833         description => "PID of running qemu process.",
2834         type => 'integer',
2835         optional => 1,
2836     },
2837     uptime => {
2838         description => "Uptime.",
2839         type => 'integer',
2840         optional => 1,
2841         renderer => 'duration',
2842     },
2843     cpus => {
2844         description => "Maximum usable CPUs.",
2845         type => 'number',
2846         optional => 1,
2847     },
2848     lock => {
2849         description => "The current config lock, if any.",
2850         type => 'string',
2851         optional => 1,
2852     },
2853     tags => {
2854         description  => "The current configured tags, if any",
2855         type => 'string',
2856         optional => 1,
2857     },
2858     'running-machine' => {
2859         description => "The currently running machine type (if running).",
2860         type => 'string',
2861         optional => 1,
2862     },
2863     'running-qemu' => {
2864         description => "The currently running QEMU version (if running).",
2865         type => 'string',
2866         optional => 1,
2867     },
2868 };
2869
2870 my $last_proc_pid_stat;
2871
2872 # get VM status information
2873 # This must be fast and should not block ($full == false)
2874 # We only query KVM using QMP if $full == true (this can be slow)
2875 sub vmstatus {
2876     my ($opt_vmid, $full) = @_;
2877
2878     my $res = {};
2879
2880     my $storecfg = PVE::Storage::config();
2881
2882     my $list = vzlist();
2883     my $defaults = load_defaults();
2884
2885     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2886
2887     my $cpucount = $cpuinfo->{cpus} || 1;
2888
2889     foreach my $vmid (keys %$list) {
2890         next if $opt_vmid && ($vmid ne $opt_vmid);
2891
2892         my $conf = PVE::QemuConfig->load_config($vmid);
2893
2894         my $d = { vmid => int($vmid) };
2895         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2896
2897         # fixme: better status?
2898         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2899
2900         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2901         if (defined($size)) {
2902             $d->{disk} = 0; # no info available
2903             $d->{maxdisk} = $size;
2904         } else {
2905             $d->{disk} = 0;
2906             $d->{maxdisk} = 0;
2907         }
2908
2909         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2910             * ($conf->{cores} || $defaults->{cores});
2911         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2912         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2913
2914         $d->{name} = $conf->{name} || "VM $vmid";
2915         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2916             : $defaults->{memory}*(1024*1024);
2917
2918         if ($conf->{balloon}) {
2919             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2920             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2921                 : $defaults->{shares};
2922         }
2923
2924         $d->{uptime} = 0;
2925         $d->{cpu} = 0;
2926         $d->{mem} = 0;
2927
2928         $d->{netout} = 0;
2929         $d->{netin} = 0;
2930
2931         $d->{diskread} = 0;
2932         $d->{diskwrite} = 0;
2933
2934         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2935
2936         $d->{serial} = 1 if conf_has_serial($conf);
2937         $d->{lock} = $conf->{lock} if $conf->{lock};
2938         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2939
2940         $res->{$vmid} = $d;
2941     }
2942
2943     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2944     foreach my $dev (keys %$netdev) {
2945         next if $dev !~ m/^tap([1-9]\d*)i/;
2946         my $vmid = $1;
2947         my $d = $res->{$vmid};
2948         next if !$d;
2949
2950         $d->{netout} += $netdev->{$dev}->{receive};
2951         $d->{netin} += $netdev->{$dev}->{transmit};
2952
2953         if ($full) {
2954             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2955             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2956         }
2957
2958     }
2959
2960     my $ctime = gettimeofday;
2961
2962     foreach my $vmid (keys %$list) {
2963
2964         my $d = $res->{$vmid};
2965         my $pid = $d->{pid};
2966         next if !$pid;
2967
2968         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2969         next if !$pstat; # not running
2970
2971         my $used = $pstat->{utime} + $pstat->{stime};
2972
2973         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2974
2975         if ($pstat->{vsize}) {
2976             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2977         }
2978
2979         my $old = $last_proc_pid_stat->{$pid};
2980         if (!$old) {
2981             $last_proc_pid_stat->{$pid} = {
2982                 time => $ctime,
2983                 used => $used,
2984                 cpu => 0,
2985             };
2986             next;
2987         }
2988
2989         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2990
2991         if ($dtime > 1000) {
2992             my $dutime = $used -  $old->{used};
2993
2994             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2995             $last_proc_pid_stat->{$pid} = {
2996                 time => $ctime,
2997                 used => $used,
2998                 cpu => $d->{cpu},
2999             };
3000         } else {
3001             $d->{cpu} = $old->{cpu};
3002         }
3003     }
3004
3005     return $res if !$full;
3006
3007     my $qmpclient = PVE::QMPClient->new();
3008
3009     my $ballooncb = sub {
3010         my ($vmid, $resp) = @_;
3011
3012         my $info = $resp->{'return'};
3013         return if !$info->{max_mem};
3014
3015         my $d = $res->{$vmid};
3016
3017         # use memory assigned to VM
3018         $d->{maxmem} = $info->{max_mem};
3019         $d->{balloon} = $info->{actual};
3020
3021         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3022             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3023             $d->{freemem} = $info->{free_mem};
3024         }
3025
3026         $d->{ballooninfo} = $info;
3027     };
3028
3029     my $blockstatscb = sub {
3030         my ($vmid, $resp) = @_;
3031         my $data = $resp->{'return'} || [];
3032         my $totalrdbytes = 0;
3033         my $totalwrbytes = 0;
3034
3035         for my $blockstat (@$data) {
3036             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3037             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3038
3039             $blockstat->{device} =~ s/drive-//;
3040             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3041         }
3042         $res->{$vmid}->{diskread} = $totalrdbytes;
3043         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3044     };
3045
3046     my $machinecb = sub {
3047         my ($vmid, $resp) = @_;
3048         my $data = $resp->{'return'} || [];
3049
3050         $res->{$vmid}->{'running-machine'} =
3051             PVE::QemuServer::Machine::current_from_query_machines($data);
3052     };
3053
3054     my $versioncb = sub {
3055         my ($vmid, $resp) = @_;
3056         my $data = $resp->{'return'} // {};
3057         my $version = 'unknown';
3058
3059         if (my $v = $data->{qemu}) {
3060             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3061         }
3062
3063         $res->{$vmid}->{'running-qemu'} = $version;
3064     };
3065
3066     my $statuscb = sub {
3067         my ($vmid, $resp) = @_;
3068
3069         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3070         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3071         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3072         # this fails if ballon driver is not loaded, so this must be
3073         # the last commnand (following command are aborted if this fails).
3074         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3075
3076         my $status = 'unknown';
3077         if (!defined($status = $resp->{'return'}->{status})) {
3078             warn "unable to get VM status\n";
3079             return;
3080         }
3081
3082         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3083     };
3084
3085     foreach my $vmid (keys %$list) {
3086         next if $opt_vmid && ($vmid ne $opt_vmid);
3087         next if !$res->{$vmid}->{pid}; # not running
3088         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3089     }
3090
3091     $qmpclient->queue_execute(undef, 2);
3092
3093     foreach my $vmid (keys %$list) {
3094         next if $opt_vmid && ($vmid ne $opt_vmid);
3095         next if !$res->{$vmid}->{pid}; #not running
3096
3097         # we can't use the $qmpclient since it might have already aborted on
3098         # 'query-balloon', but this might also fail for older versions...
3099         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3100         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3101     }
3102
3103     foreach my $vmid (keys %$list) {
3104         next if $opt_vmid && ($vmid ne $opt_vmid);
3105         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3106     }
3107
3108     return $res;
3109 }
3110
3111 sub conf_has_serial {
3112     my ($conf) = @_;
3113
3114     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3115         if ($conf->{"serial$i"}) {
3116             return 1;
3117         }
3118     }
3119
3120     return 0;
3121 }
3122
3123 sub conf_has_audio {
3124     my ($conf, $id) = @_;
3125
3126     $id //= 0;
3127     my $audio = $conf->{"audio$id"};
3128     return if !defined($audio);
3129
3130     my $audioproperties = parse_property_string($audio_fmt, $audio);
3131     my $audiodriver = $audioproperties->{driver} // 'spice';
3132
3133     return {
3134         dev => $audioproperties->{device},
3135         dev_id => "audiodev$id",
3136         backend => $audiodriver,
3137         backend_id => "$audiodriver-backend${id}",
3138     };
3139 }
3140
3141 sub audio_devs {
3142     my ($audio, $audiopciaddr, $machine_version) = @_;
3143
3144     my $devs = [];
3145
3146     my $id = $audio->{dev_id};
3147     my $audiodev = "";
3148     if (min_version($machine_version, 4, 2)) {
3149         $audiodev = ",audiodev=$audio->{backend_id}";
3150     }
3151
3152     if ($audio->{dev} eq 'AC97') {
3153         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3154     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3155         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3156         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3157         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3158     } else {
3159         die "unkown audio device '$audio->{dev}', implement me!";
3160     }
3161
3162     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3163
3164     return $devs;
3165 }
3166
3167 sub get_tpm_paths {
3168     my ($vmid) = @_;
3169     return {
3170         socket => "/var/run/qemu-server/$vmid.swtpm",
3171         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3172     };
3173 }
3174
3175 sub add_tpm_device {
3176     my ($vmid, $devices, $conf) = @_;
3177
3178     return if !$conf->{tpmstate0};
3179
3180     my $paths = get_tpm_paths($vmid);
3181
3182     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3183     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3184     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3185 }
3186
3187 sub start_swtpm {
3188     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3189
3190     return if !$tpmdrive;
3191
3192     my $state;
3193     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3194     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3195     if ($storeid) {
3196         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3197     } else {
3198         $state = $tpm->{file};
3199     }
3200
3201     my $paths = get_tpm_paths($vmid);
3202
3203     # during migration, we will get state from remote
3204     #
3205     if (!$migration) {
3206         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3207         my $setup_cmd = [
3208             "swtpm_setup",
3209             "--tpmstate",
3210             "file://$state",
3211             "--createek",
3212             "--create-ek-cert",
3213             "--create-platform-cert",
3214             "--lock-nvram",
3215             "--config",
3216             "/etc/swtpm_setup.conf", # do not use XDG configs
3217             "--runas",
3218             "0", # force creation as root, error if not possible
3219             "--not-overwrite", # ignore existing state, do not modify
3220         ];
3221
3222         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3223         # TPM 2.0 supports ECC crypto, use if possible
3224         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3225
3226         run_command($setup_cmd, outfunc => sub {
3227             print "swtpm_setup: $1\n";
3228         });
3229     }
3230
3231     my $emulator_cmd = [
3232         "swtpm",
3233         "socket",
3234         "--tpmstate",
3235         "backend-uri=file://$state,mode=0600",
3236         "--ctrl",
3237         "type=unixio,path=$paths->{socket},mode=0600",
3238         "--pid",
3239         "file=$paths->{pid}",
3240         "--terminate", # terminate on QEMU disconnect
3241         "--daemon",
3242     ];
3243     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3244     run_command($emulator_cmd, outfunc => sub { print $1; });
3245
3246     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3247     while (! -e $paths->{pid}) {
3248         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3249         usleep(50_000);
3250     }
3251
3252     # return untainted PID of swtpm daemon so it can be killed on error
3253     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3254     return $1;
3255 }
3256
3257 sub vga_conf_has_spice {
3258     my ($vga) = @_;
3259
3260     my $vgaconf = parse_vga($vga);
3261     my $vgatype = $vgaconf->{type};
3262     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3263
3264     return $1 || 1;
3265 }
3266
3267 sub is_native($) {
3268     my ($arch) = @_;
3269     return get_host_arch() eq $arch;
3270 }
3271
3272 sub get_vm_arch {
3273     my ($conf) = @_;
3274     return $conf->{arch} // get_host_arch();
3275 }
3276
3277 my $default_machines = {
3278     x86_64 => 'pc',
3279     aarch64 => 'virt',
3280 };
3281
3282 sub get_installed_machine_version {
3283     my ($kvmversion) = @_;
3284     $kvmversion = kvm_user_version() if !defined($kvmversion);
3285     $kvmversion =~ m/^(\d+\.\d+)/;
3286     return $1;
3287 }
3288
3289 sub windows_get_pinned_machine_version {
3290     my ($machine, $base_version, $kvmversion) = @_;
3291
3292     my $pin_version = $base_version;
3293     if (!defined($base_version) ||
3294         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3295     ) {
3296         $pin_version = get_installed_machine_version($kvmversion);
3297     }
3298     if (!$machine || $machine eq 'pc') {
3299         $machine = "pc-i440fx-$pin_version";
3300     } elsif ($machine eq 'q35') {
3301         $machine = "pc-q35-$pin_version";
3302     } elsif ($machine eq 'virt') {
3303         $machine = "virt-$pin_version";
3304     } else {
3305         warn "unknown machine type '$machine', not touching that!\n";
3306     }
3307
3308     return $machine;
3309 }
3310
3311 sub get_vm_machine {
3312     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3313
3314     my $machine = $forcemachine || $conf->{machine};
3315
3316     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3317         $kvmversion //= kvm_user_version();
3318         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3319         # layout which confuses windows quite a bit and may result in various regressions..
3320         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3321         if (windows_version($conf->{ostype})) {
3322             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3323         }
3324         $arch //= 'x86_64';
3325         $machine ||= $default_machines->{$arch};
3326         if ($add_pve_version) {
3327             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3328             $machine .= "+pve$pvever";
3329         }
3330     }
3331
3332     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3333         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3334         $machine = $1 if $is_pxe;
3335
3336         # for version-pinned machines that do not include a pve-version (e.g.
3337         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3338         $machine .= '+pve0';
3339
3340         $machine .= '.pxe' if $is_pxe;
3341     }
3342
3343     return $machine;
3344 }
3345
3346 sub get_ovmf_files($$$) {
3347     my ($arch, $efidisk, $smm) = @_;
3348
3349     my $types = $OVMF->{$arch}
3350         or die "no OVMF images known for architecture '$arch'\n";
3351
3352     my $type = 'default';
3353     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3354         $type = $smm ? "4m" : "4m-no-smm";
3355         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3356     }
3357
3358     return $types->{$type}->@*;
3359 }
3360
3361 my $Arch2Qemu = {
3362     aarch64 => '/usr/bin/qemu-system-aarch64',
3363     x86_64 => '/usr/bin/qemu-system-x86_64',
3364 };
3365 sub get_command_for_arch($) {
3366     my ($arch) = @_;
3367     return '/usr/bin/kvm' if is_native($arch);
3368
3369     my $cmd = $Arch2Qemu->{$arch}
3370         or die "don't know how to emulate architecture '$arch'\n";
3371     return $cmd;
3372 }
3373
3374 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3375 # to use in a QEMU command line (-cpu element), first array_intersect the result
3376 # of query_supported_ with query_understood_. This is necessary because:
3377 #
3378 # a) query_understood_ returns flags the host cannot use and
3379 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3380 #    flags, but CPU settings - with most of them being flags. Those settings
3381 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3382 #
3383 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3384 # expensive. If you need the value returned from this, you can get it much
3385 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3386 # $accel being 'kvm' or 'tcg'.
3387 #
3388 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3389 # changes, automatically populating pmxcfs.
3390 #
3391 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3392 # since kvm and tcg machines support different flags
3393 #
3394 sub query_supported_cpu_flags {
3395     my ($arch) = @_;
3396
3397     $arch //= get_host_arch();
3398     my $default_machine = $default_machines->{$arch};
3399
3400     my $flags = {};
3401
3402     # FIXME: Once this is merged, the code below should work for ARM as well:
3403     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3404     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3405         $arch eq "aarch64";
3406
3407     my $kvm_supported = defined(kvm_version());
3408     my $qemu_cmd = get_command_for_arch($arch);
3409     my $fakevmid = -1;
3410     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3411
3412     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3413     my $query_supported_run_qemu = sub {
3414         my ($kvm) = @_;
3415
3416         my $flags = {};
3417         my $cmd = [
3418             $qemu_cmd,
3419             '-machine', $default_machine,
3420             '-display', 'none',
3421             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3422             '-mon', 'chardev=qmp,mode=control',
3423             '-pidfile', $pidfile,
3424             '-S', '-daemonize'
3425         ];
3426
3427         if (!$kvm) {
3428             push @$cmd, '-accel', 'tcg';
3429         }
3430
3431         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3432         die "QEMU flag querying VM exited with code " . $rc if $rc;
3433
3434         eval {
3435             my $cmd_result = mon_cmd(
3436                 $fakevmid,
3437                 'query-cpu-model-expansion',
3438                 type => 'full',
3439                 model => { name => 'host' }
3440             );
3441
3442             my $props = $cmd_result->{model}->{props};
3443             foreach my $prop (keys %$props) {
3444                 next if $props->{$prop} ne '1';
3445                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3446                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3447                 # We only keep those with underscores, to match /proc/cpuinfo
3448                 $prop =~ s/\.|-/_/g;
3449                 $flags->{$prop} = 1;
3450             }
3451         };
3452         my $err = $@;
3453
3454         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3455         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3456
3457         die $err if $err;
3458
3459         return [ sort keys %$flags ];
3460     };
3461
3462     # We need to query QEMU twice, since KVM and TCG have different supported flags
3463     PVE::QemuConfig->lock_config($fakevmid, sub {
3464         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3465         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3466
3467         if ($kvm_supported) {
3468             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3469             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3470         }
3471     });
3472
3473     return $flags;
3474 }
3475
3476 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3477 my $understood_cpu_flag_dir = "/usr/share/kvm";
3478 sub query_understood_cpu_flags {
3479     my $arch = get_host_arch();
3480     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3481
3482     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3483         if ! -e $filepath;
3484
3485     my $raw = file_get_contents($filepath);
3486     $raw =~ s/^\s+|\s+$//g;
3487     my @flags = split(/\s+/, $raw);
3488
3489     return \@flags;
3490 }
3491
3492 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3493 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3494 my sub should_disable_smm {
3495     my ($conf, $vga) = @_;
3496
3497     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3498         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3499 }
3500
3501 sub config_to_command {
3502     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3503         $pbs_backing) = @_;
3504
3505     my $cmd = [];
3506     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3507     my $devices = [];
3508     my $bridges = {};
3509     my $ostype = $conf->{ostype};
3510     my $winversion = windows_version($ostype);
3511     my $kvm = $conf->{kvm};
3512     my $nodename = nodename();
3513
3514     my $arch = get_vm_arch($conf);
3515     my $kvm_binary = get_command_for_arch($arch);
3516     my $kvmver = kvm_user_version($kvm_binary);
3517
3518     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3519         $kvmver //= "undefined";
3520         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3521     }
3522
3523     my $add_pve_version = min_version($kvmver, 4, 1);
3524
3525     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3526     my $machine_version = extract_version($machine_type, $kvmver);
3527     $kvm //= 1 if is_native($arch);
3528
3529     $machine_version =~ m/(\d+)\.(\d+)/;
3530     my ($machine_major, $machine_minor) = ($1, $2);
3531
3532     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3533         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3534     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3535         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3536             ." please upgrade node '$nodename'\n"
3537     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3538         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3539         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3540             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3541             ." node '$nodename'\n";
3542     }
3543
3544     # if a specific +pve version is required for a feature, use $version_guard
3545     # instead of min_version to allow machines to be run with the minimum
3546     # required version
3547     my $required_pve_version = 0;
3548     my $version_guard = sub {
3549         my ($major, $minor, $pve) = @_;
3550         return 0 if !min_version($machine_version, $major, $minor, $pve);
3551         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3552         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3553         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3554         return 1;
3555     };
3556
3557     if ($kvm && !defined kvm_version()) {
3558         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3559             ." or enable in BIOS.\n";
3560     }
3561
3562     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3563     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3564     my $use_old_bios_files = undef;
3565     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3566
3567     if ($conf->{affinity}) {
3568         push @$cmd, "/usr/bin/taskset";
3569         push @$cmd, "--cpu-list";
3570         push @$cmd, "--all-tasks";
3571         push @$cmd, $conf->{affinity};
3572     }
3573
3574     push @$cmd, $kvm_binary;
3575
3576     push @$cmd, '-id', $vmid;
3577
3578     my $vmname = $conf->{name} || "vm$vmid";
3579
3580     push @$cmd, '-name', "$vmname,debug-threads=on";
3581
3582     push @$cmd, '-no-shutdown';
3583
3584     my $use_virtio = 0;
3585
3586     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3587     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3588     push @$cmd, '-mon', "chardev=qmp,mode=control";
3589
3590     if (min_version($machine_version, 2, 12)) {
3591         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3592         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3593     }
3594
3595     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3596
3597     push @$cmd, '-daemonize';
3598
3599     if ($conf->{smbios1}) {
3600         my $smbios_conf = parse_smbios1($conf->{smbios1});
3601         if ($smbios_conf->{base64}) {
3602             # Do not pass base64 flag to qemu
3603             delete $smbios_conf->{base64};
3604             my $smbios_string = "";
3605             foreach my $key (keys %$smbios_conf) {
3606                 my $value;
3607                 if ($key eq "uuid") {
3608                     $value = $smbios_conf->{uuid}
3609                 } else {
3610                     $value = decode_base64($smbios_conf->{$key});
3611                 }
3612                 # qemu accepts any binary data, only commas need escaping by double comma
3613                 $value =~ s/,/,,/g;
3614                 $smbios_string .= "," . $key . "=" . $value if $value;
3615             }
3616             push @$cmd, '-smbios', "type=1" . $smbios_string;
3617         } else {
3618             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3619         }
3620     }
3621
3622     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3623         my $d;
3624         if (my $efidisk = $conf->{efidisk0}) {
3625             $d = parse_drive('efidisk0', $efidisk);
3626         }
3627
3628         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3629         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3630
3631         my ($path, $format);
3632         my $read_only_str = '';
3633         if ($d) {
3634             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3635             $format = $d->{format};
3636             if ($storeid) {
3637                 $path = PVE::Storage::path($storecfg, $d->{file});
3638                 if (!defined($format)) {
3639                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3640                     $format = qemu_img_format($scfg, $volname);
3641                 }
3642             } else {
3643                 $path = $d->{file};
3644                 die "efidisk format must be specified\n"
3645                     if !defined($format);
3646             }
3647
3648             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3649         } else {
3650             log_warn("no efidisk configured! Using temporary efivars disk.");
3651             $path = "/tmp/$vmid-ovmf.fd";
3652             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3653             $format = 'raw';
3654         }
3655
3656         my $size_str = "";
3657
3658         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3659             $size_str = ",size=" . (-s $ovmf_vars);
3660         }
3661
3662         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3663         my $cache = "";
3664         if ($path =~ m/^rbd:/) {
3665                 $cache = ',cache=writeback';
3666                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3667         }
3668
3669         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3670         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3671     }
3672
3673     if ($q35) { # tell QEMU to load q35 config early
3674         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3675         if (min_version($machine_version, 4, 0)) {
3676             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3677         } else {
3678             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3679         }
3680     }
3681
3682     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3683         push @$cmd, $fixups->@*;
3684     }
3685
3686     if ($conf->{vmgenid}) {
3687         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3688     }
3689
3690     # add usb controllers
3691     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3692         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3693     push @$devices, @usbcontrollers if @usbcontrollers;
3694     my $vga = parse_vga($conf->{vga});
3695
3696     my $qxlnum = vga_conf_has_spice($conf->{vga});
3697     $vga->{type} = 'qxl' if $qxlnum;
3698
3699     if (!$vga->{type}) {
3700         if ($arch eq 'aarch64') {
3701             $vga->{type} = 'virtio';
3702         } elsif (min_version($machine_version, 2, 9)) {
3703             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3704         } else {
3705             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3706         }
3707     }
3708
3709     # enable absolute mouse coordinates (needed by vnc)
3710     my $tablet = $conf->{tablet};
3711     if (!defined($tablet)) {
3712         $tablet = $defaults->{tablet};
3713         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3714         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3715     }
3716
3717     if ($tablet) {
3718         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3719         my $kbd = print_keyboarddevice_full($conf, $arch);
3720         push @$devices, '-device', $kbd if defined($kbd);
3721     }
3722
3723     my $bootorder = device_bootorder($conf);
3724
3725     # host pci device passthrough
3726     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3727         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3728
3729     # usb devices
3730     my $usb_dev_features = {};
3731     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3732
3733     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3734         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3735     push @$devices, @usbdevices if @usbdevices;
3736
3737     # serial devices
3738     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3739         my $path = $conf->{"serial$i"} or next;
3740         if ($path eq 'socket') {
3741             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3742             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3743             # On aarch64, serial0 is the UART device. Qemu only allows
3744             # connecting UART devices via the '-serial' command line, as
3745             # the device has a fixed slot on the hardware...
3746             if ($arch eq 'aarch64' && $i == 0) {
3747                 push @$devices, '-serial', "chardev:serial$i";
3748             } else {
3749                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3750             }
3751         } else {
3752             die "no such serial device\n" if ! -c $path;
3753             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3754             push @$devices, '-device', "isa-serial,chardev=serial$i";
3755         }
3756     }
3757
3758     # parallel devices
3759     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3760         if (my $path = $conf->{"parallel$i"}) {
3761             die "no such parallel device\n" if ! -c $path;
3762             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3763             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3764             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3765         }
3766     }
3767
3768     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3769         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3770         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3771         push @$devices, @$audio_devs;
3772     }
3773
3774     add_tpm_device($vmid, $devices, $conf);
3775
3776     my $sockets = 1;
3777     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3778     $sockets = $conf->{sockets} if  $conf->{sockets};
3779
3780     my $cores = $conf->{cores} || 1;
3781
3782     my $maxcpus = $sockets * $cores;
3783
3784     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3785
3786     my $allowed_vcpus = $cpuinfo->{cpus};
3787
3788     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3789
3790     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3791         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3792         for (my $i = 2; $i <= $vcpus; $i++)  {
3793             my $cpustr = print_cpu_device($conf,$i);
3794             push @$cmd, '-device', $cpustr;
3795         }
3796
3797     } else {
3798
3799         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3800     }
3801     push @$cmd, '-nodefaults';
3802
3803     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3804
3805     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3806
3807     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3808
3809     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3810         push @$devices, '-device', print_vga_device(
3811             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3812
3813         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3814
3815         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3816         push @$cmd,  '-vnc', "unix:$socket,password=on";
3817     } else {
3818         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3819         push @$cmd, '-nographic';
3820     }
3821
3822     # time drift fix
3823     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3824     my $useLocaltime = $conf->{localtime};
3825
3826     if ($winversion >= 5) { # windows
3827         $useLocaltime = 1 if !defined($conf->{localtime});
3828
3829         # use time drift fix when acpi is enabled
3830         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3831             $tdf = 1 if !defined($conf->{tdf});
3832         }
3833     }
3834
3835     if ($winversion >= 6) {
3836         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3837         push @$cmd, '-no-hpet';
3838     }
3839
3840     push @$rtcFlags, 'driftfix=slew' if $tdf;
3841
3842     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3843         push @$rtcFlags, "base=$conf->{startdate}";
3844     } elsif ($useLocaltime) {
3845         push @$rtcFlags, 'base=localtime';
3846     }
3847
3848     if ($forcecpu) {
3849         push @$cmd, '-cpu', $forcecpu;
3850     } else {
3851         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3852     }
3853
3854     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3855
3856     push @$cmd, '-S' if $conf->{freeze};
3857
3858     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3859
3860     my $guest_agent = parse_guest_agent($conf);
3861
3862     if ($guest_agent->{enabled}) {
3863         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3864         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3865
3866         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3867             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3868             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3869             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3870         } elsif ($guest_agent->{type} eq 'isa') {
3871             push @$devices, '-device', "isa-serial,chardev=qga0";
3872         }
3873     }
3874
3875     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3876     if ($rng && $version_guard->(4, 1, 2)) {
3877         check_rng_source($rng->{source});
3878
3879         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3880         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3881         my $limiter_str = "";
3882         if ($max_bytes) {
3883             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3884         }
3885
3886         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3887         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3888         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3889     }
3890
3891     my $spice_port;
3892
3893     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3894         if ($qxlnum > 1) {
3895             if ($winversion){
3896                 for (my $i = 1; $i < $qxlnum; $i++){
3897                     push @$devices, '-device', print_vga_device(
3898                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3899                 }
3900             } else {
3901                 # assume other OS works like Linux
3902                 my ($ram, $vram) = ("134217728", "67108864");
3903                 if ($vga->{memory}) {
3904                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3905                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3906                 }
3907                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3908                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3909             }
3910         }
3911
3912         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3913
3914         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3915         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3916         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3917
3918         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3919         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3920         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3921
3922         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3923         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3924
3925         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3926         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3927         if ($spice_enhancement->{foldersharing}) {
3928             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3929             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3930         }
3931
3932         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3933         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3934             if $spice_enhancement->{videostreaming};
3935
3936         push @$devices, '-spice', "$spice_opts";
3937     }
3938
3939     # enable balloon by default, unless explicitly disabled
3940     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3941         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3942         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3943         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3944         push @$devices, '-device', $ballooncmd;
3945     }
3946
3947     if ($conf->{watchdog}) {
3948         my $wdopts = parse_watchdog($conf->{watchdog});
3949         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3950         my $watchdog = $wdopts->{model} || 'i6300esb';
3951         push @$devices, '-device', "$watchdog$pciaddr";
3952         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3953     }
3954
3955     my $vollist = [];
3956     my $scsicontroller = {};
3957     my $ahcicontroller = {};
3958     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3959
3960     # Add iscsi initiator name if available
3961     if (my $initiator = get_initiator_name()) {
3962         push @$devices, '-iscsi', "initiator-name=$initiator";
3963     }
3964
3965     PVE::QemuConfig->foreach_volume($conf, sub {
3966         my ($ds, $drive) = @_;
3967
3968         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3969             check_volume_storage_type($storecfg, $drive->{file});
3970             push @$vollist, $drive->{file};
3971         }
3972
3973         # ignore efidisk here, already added in bios/fw handling code above
3974         return if $drive->{interface} eq 'efidisk';
3975         # similar for TPM
3976         return if $drive->{interface} eq 'tpmstate';
3977
3978         $use_virtio = 1 if $ds =~ m/^virtio/;
3979
3980         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3981
3982         if ($drive->{interface} eq 'virtio'){
3983            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3984         }
3985
3986         if ($drive->{interface} eq 'scsi') {
3987
3988             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3989
3990             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3991                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3992
3993             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3994             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3995
3996             my $iothread = '';
3997             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3998                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3999                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4000             } elsif ($drive->{iothread}) {
4001                 log_warn(
4002                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4003                 );
4004             }
4005
4006             my $queues = '';
4007             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4008                 $queues = ",num_queues=$drive->{queues}";
4009             }
4010
4011             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4012                 if !$scsicontroller->{$controller};
4013             $scsicontroller->{$controller}=1;
4014         }
4015
4016         if ($drive->{interface} eq 'sata') {
4017             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4018             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4019             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4020                 if !$ahcicontroller->{$controller};
4021             $ahcicontroller->{$controller}=1;
4022         }
4023
4024         my $pbs_conf = $pbs_backing->{$ds};
4025         my $pbs_name = undef;
4026         if ($pbs_conf) {
4027             $pbs_name = "drive-$ds-pbs";
4028             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4029         }
4030
4031         my $drive_cmd = print_drive_commandline_full(
4032             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4033
4034         # extra protection for templates, but SATA and IDE don't support it..
4035         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4036
4037         push @$devices, '-drive',$drive_cmd;
4038         push @$devices, '-device', print_drivedevice_full(
4039             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4040     });
4041
4042     for (my $i = 0; $i < $MAX_NETS; $i++) {
4043         my $netname = "net$i";
4044
4045         next if !$conf->{$netname};
4046         my $d = parse_net($conf->{$netname});
4047         next if !$d;
4048         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4049
4050         $use_virtio = 1 if $d->{model} eq 'virtio';
4051
4052         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4053
4054         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4055         push @$devices, '-netdev', $netdevfull;
4056
4057         my $netdevicefull = print_netdevice_full(
4058             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4059
4060         push @$devices, '-device', $netdevicefull;
4061     }
4062
4063     if ($conf->{ivshmem}) {
4064         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4065
4066         my $bus;
4067         if ($q35) {
4068             $bus = print_pcie_addr("ivshmem");
4069         } else {
4070             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4071         }
4072
4073         my $ivshmem_name = $ivshmem->{name} // $vmid;
4074         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4075
4076         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4077         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4078             .",size=$ivshmem->{size}M";
4079     }
4080
4081     # pci.4 is nested in pci.1
4082     $bridges->{1} = 1 if $bridges->{4};
4083
4084     if (!$q35) { # add pci bridges
4085         if (min_version($machine_version, 2, 3)) {
4086            $bridges->{1} = 1;
4087            $bridges->{2} = 1;
4088         }
4089         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4090     }
4091
4092     for my $k (sort {$b cmp $a} keys %$bridges) {
4093         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4094
4095         my $k_name = $k;
4096         if ($k == 2 && $legacy_igd) {
4097             $k_name = "$k-igd";
4098         }
4099         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4100         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4101
4102         if ($q35) { # add after -readconfig pve-q35.cfg
4103             splice @$devices, 2, 0, '-device', $devstr;
4104         } else {
4105             unshift @$devices, '-device', $devstr if $k > 0;
4106         }
4107     }
4108
4109     if (!$kvm) {
4110         push @$machineFlags, 'accel=tcg';
4111     }
4112
4113     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4114
4115     my $machine_type_min = $machine_type;
4116     if ($add_pve_version) {
4117         $machine_type_min =~ s/\+pve\d+$//;
4118         $machine_type_min .= "+pve$required_pve_version";
4119     }
4120     push @$machineFlags, "type=${machine_type_min}";
4121
4122     push @$cmd, @$devices;
4123     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4124     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4125     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4126
4127     if (my $vmstate = $conf->{vmstate}) {
4128         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4129         push @$vollist, $vmstate;
4130         push @$cmd, '-loadstate', $statepath;
4131         print "activating and using '$vmstate' as vmstate\n";
4132     }
4133
4134     if (PVE::QemuConfig->is_template($conf)) {
4135         # needed to workaround base volumes being read-only
4136         push @$cmd, '-snapshot';
4137     }
4138
4139     # add custom args
4140     if ($conf->{args}) {
4141         my $aa = PVE::Tools::split_args($conf->{args});
4142         push @$cmd, @$aa;
4143     }
4144
4145     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4146 }
4147
4148 sub check_rng_source {
4149     my ($source) = @_;
4150
4151     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4152     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4153         if ! -e $source;
4154
4155     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4156     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4157         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4158         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4159         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4160             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4161             ." to the host.\n";
4162     }
4163 }
4164
4165 sub spice_port {
4166     my ($vmid) = @_;
4167
4168     my $res = mon_cmd($vmid, 'query-spice');
4169
4170     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4171 }
4172
4173 sub vm_devices_list {
4174     my ($vmid) = @_;
4175
4176     my $res = mon_cmd($vmid, 'query-pci');
4177     my $devices_to_check = [];
4178     my $devices = {};
4179     foreach my $pcibus (@$res) {
4180         push @$devices_to_check, @{$pcibus->{devices}},
4181     }
4182
4183     while (@$devices_to_check) {
4184         my $to_check = [];
4185         for my $d (@$devices_to_check) {
4186             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4187             next if !$d->{'pci_bridge'};
4188
4189             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4190             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4191         }
4192         $devices_to_check = $to_check;
4193     }
4194
4195     my $resblock = mon_cmd($vmid, 'query-block');
4196     foreach my $block (@$resblock) {
4197         if($block->{device} =~ m/^drive-(\S+)/){
4198                 $devices->{$1} = 1;
4199         }
4200     }
4201
4202     my $resmice = mon_cmd($vmid, 'query-mice');
4203     foreach my $mice (@$resmice) {
4204         if ($mice->{name} eq 'QEMU HID Tablet') {
4205             $devices->{tablet} = 1;
4206             last;
4207         }
4208     }
4209
4210     # for usb devices there is no query-usb
4211     # but we can iterate over the entries in
4212     # qom-list path=/machine/peripheral
4213     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4214     foreach my $per (@$resperipheral) {
4215         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4216             $devices->{$per->{name}} = 1;
4217         }
4218     }
4219
4220     return $devices;
4221 }
4222
4223 sub vm_deviceplug {
4224     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4225
4226     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4227
4228     my $devices_list = vm_devices_list($vmid);
4229     return 1 if defined($devices_list->{$deviceid});
4230
4231     # add PCI bridge if we need it for the device
4232     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4233
4234     if ($deviceid eq 'tablet') {
4235         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4236     } elsif ($deviceid eq 'keyboard') {
4237         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4238     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4239         my $id = $1;
4240         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4241         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4242     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4243         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4244     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4245         qemu_iothread_add($vmid, $deviceid, $device);
4246
4247         qemu_driveadd($storecfg, $vmid, $device);
4248         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4249
4250         qemu_deviceadd($vmid, $devicefull);
4251         eval { qemu_deviceaddverify($vmid, $deviceid); };
4252         if (my $err = $@) {
4253             eval { qemu_drivedel($vmid, $deviceid); };
4254             warn $@ if $@;
4255             die $err;
4256         }
4257     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4258         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4259         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4260         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4261
4262         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4263
4264         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4265             qemu_iothread_add($vmid, $deviceid, $device);
4266             $devicefull .= ",iothread=iothread-$deviceid";
4267         }
4268
4269         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4270             $devicefull .= ",num_queues=$device->{queues}";
4271         }
4272
4273         qemu_deviceadd($vmid, $devicefull);
4274         qemu_deviceaddverify($vmid, $deviceid);
4275     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4276         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4277         qemu_driveadd($storecfg, $vmid, $device);
4278
4279         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4280         eval { qemu_deviceadd($vmid, $devicefull); };
4281         if (my $err = $@) {
4282             eval { qemu_drivedel($vmid, $deviceid); };
4283             warn $@ if $@;
4284             die $err;
4285         }
4286     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4287         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4288
4289         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4290         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4291         my $use_old_bios_files = undef;
4292         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4293
4294         my $netdevicefull = print_netdevice_full(
4295             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4296         qemu_deviceadd($vmid, $netdevicefull);
4297         eval {
4298             qemu_deviceaddverify($vmid, $deviceid);
4299             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4300         };
4301         if (my $err = $@) {
4302             eval { qemu_netdevdel($vmid, $deviceid); };
4303             warn $@ if $@;
4304             die $err;
4305         }
4306     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4307         my $bridgeid = $2;
4308         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4309         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4310
4311         qemu_deviceadd($vmid, $devicefull);
4312         qemu_deviceaddverify($vmid, $deviceid);
4313     } else {
4314         die "can't hotplug device '$deviceid'\n";
4315     }
4316
4317     return 1;
4318 }
4319
4320 # fixme: this should raise exceptions on error!
4321 sub vm_deviceunplug {
4322     my ($vmid, $conf, $deviceid) = @_;
4323
4324     my $devices_list = vm_devices_list($vmid);
4325     return 1 if !defined($devices_list->{$deviceid});
4326
4327     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4328     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4329
4330     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4331         qemu_devicedel($vmid, $deviceid);
4332     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4333         qemu_devicedel($vmid, $deviceid);
4334         qemu_devicedelverify($vmid, $deviceid);
4335     } elsif ($deviceid =~ m/^usb\d+$/) {
4336         qemu_devicedel($vmid, $deviceid);
4337         qemu_devicedelverify($vmid, $deviceid);
4338     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4339         my $device = parse_drive($deviceid, $conf->{$deviceid});
4340
4341         qemu_devicedel($vmid, $deviceid);
4342         qemu_devicedelverify($vmid, $deviceid);
4343         qemu_drivedel($vmid, $deviceid);
4344         qemu_iothread_del($vmid, $deviceid, $device);
4345     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4349         my $device = parse_drive($deviceid, $conf->{$deviceid});
4350
4351         qemu_devicedel($vmid, $deviceid);
4352         qemu_devicedelverify($vmid, $deviceid);
4353         qemu_drivedel($vmid, $deviceid);
4354         qemu_deletescsihw($conf, $vmid, $deviceid);
4355
4356         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4357             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4358     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4359         qemu_devicedel($vmid, $deviceid);
4360         qemu_devicedelverify($vmid, $deviceid);
4361         qemu_netdevdel($vmid, $deviceid);
4362     } else {
4363         die "can't unplug device '$deviceid'\n";
4364     }
4365
4366     return 1;
4367 }
4368
4369 sub qemu_spice_usbredir_chardev_add {
4370     my ($vmid, $id) = @_;
4371
4372     mon_cmd($vmid, "chardev-add" , (
4373         id => $id,
4374         backend => {
4375             type => 'spicevmc',
4376             data => {
4377                 type => "usbredir",
4378             },
4379         },
4380     ));
4381 }
4382
4383 sub qemu_deviceadd {
4384     my ($vmid, $devicefull) = @_;
4385
4386     $devicefull = "driver=".$devicefull;
4387     my %options =  split(/[=,]/, $devicefull);
4388
4389     mon_cmd($vmid, "device_add" , %options);
4390 }
4391
4392 sub qemu_devicedel {
4393     my ($vmid, $deviceid) = @_;
4394
4395     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4396 }
4397
4398 sub qemu_iothread_add {
4399     my ($vmid, $deviceid, $device) = @_;
4400
4401     if ($device->{iothread}) {
4402         my $iothreads = vm_iothreads_list($vmid);
4403         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4404     }
4405 }
4406
4407 sub qemu_iothread_del {
4408     my ($vmid, $deviceid, $device) = @_;
4409
4410     if ($device->{iothread}) {
4411         my $iothreads = vm_iothreads_list($vmid);
4412         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4413     }
4414 }
4415
4416 sub qemu_objectadd {
4417     my ($vmid, $objectid, $qomtype) = @_;
4418
4419     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4420
4421     return 1;
4422 }
4423
4424 sub qemu_objectdel {
4425     my ($vmid, $objectid) = @_;
4426
4427     mon_cmd($vmid, "object-del", id => $objectid);
4428
4429     return 1;
4430 }
4431
4432 sub qemu_driveadd {
4433     my ($storecfg, $vmid, $device) = @_;
4434
4435     my $kvmver = get_running_qemu_version($vmid);
4436     my $io_uring = min_version($kvmver, 6, 0);
4437     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4438     $drive =~ s/\\/\\\\/g;
4439     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4440
4441     # If the command succeeds qemu prints: "OK"
4442     return 1 if $ret =~ m/OK/s;
4443
4444     die "adding drive failed: $ret\n";
4445 }
4446
4447 sub qemu_drivedel {
4448     my ($vmid, $deviceid) = @_;
4449
4450     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4451     $ret =~ s/^\s+//;
4452
4453     return 1 if $ret eq "";
4454
4455     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4456     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4457
4458     die "deleting drive $deviceid failed : $ret\n";
4459 }
4460
4461 sub qemu_deviceaddverify {
4462     my ($vmid, $deviceid) = @_;
4463
4464     for (my $i = 0; $i <= 5; $i++) {
4465          my $devices_list = vm_devices_list($vmid);
4466          return 1 if defined($devices_list->{$deviceid});
4467          sleep 1;
4468     }
4469
4470     die "error on hotplug device '$deviceid'\n";
4471 }
4472
4473
4474 sub qemu_devicedelverify {
4475     my ($vmid, $deviceid) = @_;
4476
4477     # need to verify that the device is correctly removed as device_del
4478     # is async and empty return is not reliable
4479
4480     for (my $i = 0; $i <= 5; $i++) {
4481          my $devices_list = vm_devices_list($vmid);
4482          return 1 if !defined($devices_list->{$deviceid});
4483          sleep 1;
4484     }
4485
4486     die "error on hot-unplugging device '$deviceid'\n";
4487 }
4488
4489 sub qemu_findorcreatescsihw {
4490     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4491
4492     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4493
4494     my $scsihwid="$controller_prefix$controller";
4495     my $devices_list = vm_devices_list($vmid);
4496
4497     if (!defined($devices_list->{$scsihwid})) {
4498         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4499     }
4500
4501     return 1;
4502 }
4503
4504 sub qemu_deletescsihw {
4505     my ($conf, $vmid, $opt) = @_;
4506
4507     my $device = parse_drive($opt, $conf->{$opt});
4508
4509     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4510         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4511         return 1;
4512     }
4513
4514     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4515
4516     my $devices_list = vm_devices_list($vmid);
4517     foreach my $opt (keys %{$devices_list}) {
4518         if (is_valid_drivename($opt)) {
4519             my $drive = parse_drive($opt, $conf->{$opt});
4520             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4521                 return 1;
4522             }
4523         }
4524     }
4525
4526     my $scsihwid="scsihw$controller";
4527
4528     vm_deviceunplug($vmid, $conf, $scsihwid);
4529
4530     return 1;
4531 }
4532
4533 sub qemu_add_pci_bridge {
4534     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4535
4536     my $bridges = {};
4537
4538     my $bridgeid;
4539
4540     print_pci_addr($device, $bridges, $arch, $machine_type);
4541
4542     while (my ($k, $v) = each %$bridges) {
4543         $bridgeid = $k;
4544     }
4545     return 1 if !defined($bridgeid) || $bridgeid < 1;
4546
4547     my $bridge = "pci.$bridgeid";
4548     my $devices_list = vm_devices_list($vmid);
4549
4550     if (!defined($devices_list->{$bridge})) {
4551         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4552     }
4553
4554     return 1;
4555 }
4556
4557 sub qemu_set_link_status {
4558     my ($vmid, $device, $up) = @_;
4559
4560     mon_cmd($vmid, "set_link", name => $device,
4561                up => $up ? JSON::true : JSON::false);
4562 }
4563
4564 sub qemu_netdevadd {
4565     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4566
4567     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4568     my %options =  split(/[=,]/, $netdev);
4569
4570     if (defined(my $vhost = $options{vhost})) {
4571         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4572     }
4573
4574     if (defined(my $queues = $options{queues})) {
4575         $options{queues} = $queues + 0;
4576     }
4577
4578     mon_cmd($vmid, "netdev_add",  %options);
4579     return 1;
4580 }
4581
4582 sub qemu_netdevdel {
4583     my ($vmid, $deviceid) = @_;
4584
4585     mon_cmd($vmid, "netdev_del", id => $deviceid);
4586 }
4587
4588 sub qemu_usb_hotplug {
4589     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4590
4591     return if !$device;
4592
4593     # remove the old one first
4594     vm_deviceunplug($vmid, $conf, $deviceid);
4595
4596     # check if xhci controller is necessary and available
4597     my $devicelist = vm_devices_list($vmid);
4598
4599     if (!$devicelist->{xhci}) {
4600         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4601         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4602     }
4603
4604     # print_usbdevice_full expects the parsed device
4605     my $d = parse_usb_device($device->{host});
4606     $d->{usb3} = $device->{usb3};
4607
4608     # add the new one
4609     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4610 }
4611
4612 sub qemu_cpu_hotplug {
4613     my ($vmid, $conf, $vcpus) = @_;
4614
4615     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4616
4617     my $sockets = 1;
4618     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4619     $sockets = $conf->{sockets} if  $conf->{sockets};
4620     my $cores = $conf->{cores} || 1;
4621     my $maxcpus = $sockets * $cores;
4622
4623     $vcpus = $maxcpus if !$vcpus;
4624
4625     die "you can't add more vcpus than maxcpus\n"
4626         if $vcpus > $maxcpus;
4627
4628     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4629
4630     if ($vcpus < $currentvcpus) {
4631
4632         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4633
4634             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4635                 qemu_devicedel($vmid, "cpu$i");
4636                 my $retry = 0;
4637                 my $currentrunningvcpus = undef;
4638                 while (1) {
4639                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4640                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4641                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4642                     $retry++;
4643                     sleep 1;
4644                 }
4645                 #update conf after each succesfull cpu unplug
4646                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4647                 PVE::QemuConfig->write_config($vmid, $conf);
4648             }
4649         } else {
4650             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4651         }
4652
4653         return;
4654     }
4655
4656     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4657     die "vcpus in running vm does not match its configuration\n"
4658         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4659
4660     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4661
4662         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4663             my $cpustr = print_cpu_device($conf, $i);
4664             qemu_deviceadd($vmid, $cpustr);
4665
4666             my $retry = 0;
4667             my $currentrunningvcpus = undef;
4668             while (1) {
4669                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4670                 last if scalar(@{$currentrunningvcpus}) == $i;
4671                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4672                 sleep 1;
4673                 $retry++;
4674             }
4675             #update conf after each succesfull cpu hotplug
4676             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4677             PVE::QemuConfig->write_config($vmid, $conf);
4678         }
4679     } else {
4680
4681         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4682             mon_cmd($vmid, "cpu-add", id => int($i));
4683         }
4684     }
4685 }
4686
4687 sub qemu_block_set_io_throttle {
4688     my ($vmid, $deviceid,
4689         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4690         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4691         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4692         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4693
4694     return if !check_running($vmid) ;
4695
4696     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4697         bps => int($bps),
4698         bps_rd => int($bps_rd),
4699         bps_wr => int($bps_wr),
4700         iops => int($iops),
4701         iops_rd => int($iops_rd),
4702         iops_wr => int($iops_wr),
4703         bps_max => int($bps_max),
4704         bps_rd_max => int($bps_rd_max),
4705         bps_wr_max => int($bps_wr_max),
4706         iops_max => int($iops_max),
4707         iops_rd_max => int($iops_rd_max),
4708         iops_wr_max => int($iops_wr_max),
4709         bps_max_length => int($bps_max_length),
4710         bps_rd_max_length => int($bps_rd_max_length),
4711         bps_wr_max_length => int($bps_wr_max_length),
4712         iops_max_length => int($iops_max_length),
4713         iops_rd_max_length => int($iops_rd_max_length),
4714         iops_wr_max_length => int($iops_wr_max_length),
4715     );
4716
4717 }
4718
4719 sub qemu_block_resize {
4720     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4721
4722     my $running = check_running($vmid);
4723
4724     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4725
4726     return if !$running;
4727
4728     my $padding = (1024 - $size % 1024) % 1024;
4729     $size = $size + $padding;
4730
4731     mon_cmd(
4732         $vmid,
4733         "block_resize",
4734         device => $deviceid,
4735         size => int($size),
4736         timeout => 60,
4737     );
4738 }
4739
4740 sub qemu_volume_snapshot {
4741     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4742
4743     my $running = check_running($vmid);
4744
4745     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4746         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4747     } else {
4748         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4749     }
4750 }
4751
4752 sub qemu_volume_snapshot_delete {
4753     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4754
4755     my $running = check_running($vmid);
4756
4757     if($running) {
4758
4759         $running = undef;
4760         my $conf = PVE::QemuConfig->load_config($vmid);
4761         PVE::QemuConfig->foreach_volume($conf, sub {
4762             my ($ds, $drive) = @_;
4763             $running = 1 if $drive->{file} eq $volid;
4764         });
4765     }
4766
4767     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4768         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4769     } else {
4770         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4771     }
4772 }
4773
4774 sub set_migration_caps {
4775     my ($vmid, $savevm) = @_;
4776
4777     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4778
4779     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4780     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4781
4782     my $cap_ref = [];
4783
4784     my $enabled_cap = {
4785         "auto-converge" => 1,
4786         "xbzrle" => 1,
4787         "x-rdma-pin-all" => 0,
4788         "zero-blocks" => 0,
4789         "compress" => 0,
4790         "dirty-bitmaps" => $dirty_bitmaps,
4791     };
4792
4793     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4794
4795     for my $supported_capability (@$supported_capabilities) {
4796         push @$cap_ref, {
4797             capability => $supported_capability->{capability},
4798             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4799         };
4800     }
4801
4802     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4803 }
4804
4805 sub foreach_volid {
4806     my ($conf, $func, @param) = @_;
4807
4808     my $volhash = {};
4809
4810     my $test_volid = sub {
4811         my ($key, $drive, $snapname) = @_;
4812
4813         my $volid = $drive->{file};
4814         return if !$volid;
4815
4816         $volhash->{$volid}->{cdrom} //= 1;
4817         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4818
4819         my $replicate = $drive->{replicate} // 1;
4820         $volhash->{$volid}->{replicate} //= 0;
4821         $volhash->{$volid}->{replicate} = 1 if $replicate;
4822
4823         $volhash->{$volid}->{shared} //= 0;
4824         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4825
4826         $volhash->{$volid}->{referenced_in_config} //= 0;
4827         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4828
4829         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4830             if defined($snapname);
4831
4832         my $size = $drive->{size};
4833         $volhash->{$volid}->{size} //= $size if $size;
4834
4835         $volhash->{$volid}->{is_vmstate} //= 0;
4836         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4837
4838         $volhash->{$volid}->{is_tpmstate} //= 0;
4839         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4840
4841         $volhash->{$volid}->{is_unused} //= 0;
4842         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4843
4844         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4845     };
4846
4847     my $include_opts = {
4848         extra_keys => ['vmstate'],
4849         include_unused => 1,
4850     };
4851
4852     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4853     foreach my $snapname (keys %{$conf->{snapshots}}) {
4854         my $snap = $conf->{snapshots}->{$snapname};
4855         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4856     }
4857
4858     foreach my $volid (keys %$volhash) {
4859         &$func($volid, $volhash->{$volid}, @param);
4860     }
4861 }
4862
4863 my $fast_plug_option = {
4864     'lock' => 1,
4865     'name' => 1,
4866     'onboot' => 1,
4867     'shares' => 1,
4868     'startup' => 1,
4869     'description' => 1,
4870     'protection' => 1,
4871     'vmstatestorage' => 1,
4872     'hookscript' => 1,
4873     'tags' => 1,
4874 };
4875
4876 for my $opt (keys %$confdesc_cloudinit) {
4877     $fast_plug_option->{$opt} = 1;
4878 };
4879
4880 # hotplug changes in [PENDING]
4881 # $selection hash can be used to only apply specified options, for
4882 # example: { cores => 1 } (only apply changed 'cores')
4883 # $errors ref is used to return error messages
4884 sub vmconfig_hotplug_pending {
4885     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4886
4887     my $defaults = load_defaults();
4888     my $arch = get_vm_arch($conf);
4889     my $machine_type = get_vm_machine($conf, undef, $arch);
4890
4891     # commit values which do not have any impact on running VM first
4892     # Note: those option cannot raise errors, we we do not care about
4893     # $selection and always apply them.
4894
4895     my $add_error = sub {
4896         my ($opt, $msg) = @_;
4897         $errors->{$opt} = "hotplug problem - $msg";
4898     };
4899
4900     my $changes = 0;
4901     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4902         if ($fast_plug_option->{$opt}) {
4903             $conf->{$opt} = $conf->{pending}->{$opt};
4904             delete $conf->{pending}->{$opt};
4905             $changes = 1;
4906         }
4907     }
4908
4909     if ($changes) {
4910         PVE::QemuConfig->write_config($vmid, $conf);
4911     }
4912
4913     my $ostype = $conf->{ostype};
4914     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4915     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4916     my $usb_hotplug = $hotplug_features->{usb}
4917         && min_version($version, 7, 1)
4918         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4919
4920     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4921     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4922     foreach my $opt (sort keys %$pending_delete_hash) {
4923         next if $selection && !$selection->{$opt};
4924         my $force = $pending_delete_hash->{$opt}->{force};
4925         eval {
4926             if ($opt eq 'hotplug') {
4927                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4928             } elsif ($opt eq 'tablet') {
4929                 die "skip\n" if !$hotplug_features->{usb};
4930                 if ($defaults->{tablet}) {
4931                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4932                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4933                         if $arch eq 'aarch64';
4934                 } else {
4935                     vm_deviceunplug($vmid, $conf, 'tablet');
4936                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4937                 }
4938             } elsif ($opt =~ m/^usb(\d+)$/) {
4939                 my $index = $1;
4940                 die "skip\n" if !$usb_hotplug;
4941                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4942                 vm_deviceunplug($vmid, $conf, $opt);
4943             } elsif ($opt eq 'vcpus') {
4944                 die "skip\n" if !$hotplug_features->{cpu};
4945                 qemu_cpu_hotplug($vmid, $conf, undef);
4946             } elsif ($opt eq 'balloon') {
4947                 # enable balloon device is not hotpluggable
4948                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4949                 # here we reset the ballooning value to memory
4950                 my $balloon = $conf->{memory} || $defaults->{memory};
4951                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4952             } elsif ($fast_plug_option->{$opt}) {
4953                 # do nothing
4954             } elsif ($opt =~ m/^net(\d+)$/) {
4955                 die "skip\n" if !$hotplug_features->{network};
4956                 vm_deviceunplug($vmid, $conf, $opt);
4957             } elsif (is_valid_drivename($opt)) {
4958                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4959                 vm_deviceunplug($vmid, $conf, $opt);
4960                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4961             } elsif ($opt =~ m/^memory$/) {
4962                 die "skip\n" if !$hotplug_features->{memory};
4963                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4964             } elsif ($opt eq 'cpuunits') {
4965                 $cgroup->change_cpu_shares(undef);
4966             } elsif ($opt eq 'cpulimit') {
4967                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4968             } else {
4969                 die "skip\n";
4970             }
4971         };
4972         if (my $err = $@) {
4973             &$add_error($opt, $err) if $err ne "skip\n";
4974         } else {
4975             delete $conf->{$opt};
4976             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4977         }
4978     }
4979
4980     foreach my $opt (keys %{$conf->{pending}}) {
4981         next if $selection && !$selection->{$opt};
4982         my $value = $conf->{pending}->{$opt};
4983         eval {
4984             if ($opt eq 'hotplug') {
4985                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4986             } elsif ($opt eq 'tablet') {
4987                 die "skip\n" if !$hotplug_features->{usb};
4988                 if ($value == 1) {
4989                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4990                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4991                         if $arch eq 'aarch64';
4992                 } elsif ($value == 0) {
4993                     vm_deviceunplug($vmid, $conf, 'tablet');
4994                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4995                 }
4996             } elsif ($opt =~ m/^usb(\d+)$/) {
4997                 my $index = $1;
4998                 die "skip\n" if !$usb_hotplug;
4999                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5000                 my $id = $opt;
5001                 if ($d->{host} eq 'spice')  {
5002                     $id = "usbredirdev$index";
5003                 }
5004                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5005             } elsif ($opt eq 'vcpus') {
5006                 die "skip\n" if !$hotplug_features->{cpu};
5007                 qemu_cpu_hotplug($vmid, $conf, $value);
5008             } elsif ($opt eq 'balloon') {
5009                 # enable/disable balloning device is not hotpluggable
5010                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5011                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5012                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5013
5014                 # allow manual ballooning if shares is set to zero
5015                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5016                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5017                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5018                 }
5019             } elsif ($opt =~ m/^net(\d+)$/) {
5020                 # some changes can be done without hotplug
5021                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5022                                     $vmid, $opt, $value, $arch, $machine_type);
5023             } elsif (is_valid_drivename($opt)) {
5024                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5025                 # some changes can be done without hotplug
5026                 my $drive = parse_drive($opt, $value);
5027                 if (drive_is_cloudinit($drive)) {
5028                     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5029                 }
5030                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5031                                      $vmid, $opt, $value, $arch, $machine_type);
5032             } elsif ($opt =~ m/^memory$/) { #dimms
5033                 die "skip\n" if !$hotplug_features->{memory};
5034                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5035             } elsif ($opt eq 'cpuunits') {
5036                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5037                 $cgroup->change_cpu_shares($new_cpuunits);
5038             } elsif ($opt eq 'cpulimit') {
5039                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5040                 $cgroup->change_cpu_quota($cpulimit, 100000);
5041             } elsif ($opt eq 'agent') {
5042                 vmconfig_update_agent($conf, $opt, $value);
5043             } else {
5044                 die "skip\n";  # skip non-hot-pluggable options
5045             }
5046         };
5047         if (my $err = $@) {
5048             &$add_error($opt, $err) if $err ne "skip\n";
5049         } else {
5050             $conf->{$opt} = $value;
5051             delete $conf->{pending}->{$opt};
5052         }
5053     }
5054
5055     # unplug xhci controller if no usb device is left
5056     if ($usb_hotplug) {
5057         my $has_usb = 0;
5058         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5059             next if !defined($conf->{"usb$i"});
5060             $has_usb = 1;
5061             last;
5062         }
5063         if (!$has_usb) {
5064             vm_deviceunplug($vmid, $conf, 'xhci');
5065         }
5066     }
5067
5068     PVE::QemuConfig->write_config($vmid, $conf);
5069
5070     if($hotplug_features->{cloudinit}) {
5071         my $pending = PVE::QemuServer::Cloudinit::get_pending_config($conf, $vmid);
5072         my $regenerate = undef;
5073         for my $item (@$pending) {
5074             $regenerate = 1 if defined($item->{delete}) or defined($item->{pending});
5075         }
5076         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid) if $regenerate;
5077     }
5078 }
5079
5080 sub try_deallocate_drive {
5081     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5082
5083     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5084         my $volid = $drive->{file};
5085         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5086             my $sid = PVE::Storage::parse_volume_id($volid);
5087             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5088
5089             # check if the disk is really unused
5090             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5091                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5092             PVE::Storage::vdisk_free($storecfg, $volid);
5093             return 1;
5094         } else {
5095             # If vm is not owner of this disk remove from config
5096             return 1;
5097         }
5098     }
5099
5100     return;
5101 }
5102
5103 sub vmconfig_delete_or_detach_drive {
5104     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5105
5106     my $drive = parse_drive($opt, $conf->{$opt});
5107
5108     my $rpcenv = PVE::RPCEnvironment::get();
5109     my $authuser = $rpcenv->get_user();
5110
5111     if ($force) {
5112         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5113         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5114     } else {
5115         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5116     }
5117 }
5118
5119
5120
5121 sub vmconfig_apply_pending {
5122     my ($vmid, $conf, $storecfg, $errors) = @_;
5123
5124     return if !scalar(keys %{$conf->{pending}});
5125
5126     my $add_apply_error = sub {
5127         my ($opt, $msg) = @_;
5128         my $err_msg = "unable to apply pending change $opt : $msg";
5129         $errors->{$opt} = $err_msg;
5130         warn $err_msg;
5131     };
5132
5133     # cold plug
5134
5135     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5136     foreach my $opt (sort keys %$pending_delete_hash) {
5137         my $force = $pending_delete_hash->{$opt}->{force};
5138         eval {
5139             if ($opt =~ m/^unused/) {
5140                 die "internal error";
5141             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5142                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5143             }
5144         };
5145         if (my $err = $@) {
5146             $add_apply_error->($opt, $err);
5147         } else {
5148             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5149             delete $conf->{$opt};
5150         }
5151     }
5152
5153     PVE::QemuConfig->cleanup_pending($conf);
5154
5155     my $generate_cloudnit = undef;
5156
5157     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5158         next if $opt eq 'delete'; # just to be sure
5159         eval {
5160             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5161                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5162             }
5163         };
5164         if (my $err = $@) {
5165             $add_apply_error->($opt, $err);
5166         } else {
5167
5168             if (is_valid_drivename($opt)) {
5169                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5170                 $generate_cloudnit = 1 if drive_is_cloudinit($drive);
5171             }
5172
5173             $conf->{$opt} = delete $conf->{pending}->{$opt};
5174         }
5175     }
5176
5177     # write all changes at once to avoid unnecessary i/o
5178     PVE::QemuConfig->write_config($vmid, $conf);
5179     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if $generate_cloudnit;
5180 }
5181
5182 sub vmconfig_update_net {
5183     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5184
5185     my $newnet = parse_net($value);
5186
5187     if ($conf->{$opt}) {
5188         my $oldnet = parse_net($conf->{$opt});
5189
5190         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5191             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5192             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5193             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5194
5195             # for non online change, we try to hot-unplug
5196             die "skip\n" if !$hotplug;
5197             vm_deviceunplug($vmid, $conf, $opt);
5198         } else {
5199
5200             die "internal error" if $opt !~ m/net(\d+)/;
5201             my $iface = "tap${vmid}i$1";
5202
5203             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5204                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5205                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5206                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5207                 PVE::Network::tap_unplug($iface);
5208
5209                 if ($have_sdn) {
5210                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5211                 } else {
5212                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5213                 }
5214             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5215                 # Rate can be applied on its own but any change above needs to
5216                 # include the rate in tap_plug since OVS resets everything.
5217                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5218             }
5219
5220             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5221                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5222             }
5223
5224             return 1;
5225         }
5226     }
5227
5228     if ($hotplug) {
5229         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5230     } else {
5231         die "skip\n";
5232     }
5233 }
5234
5235 sub vmconfig_update_agent {
5236     my ($conf, $opt, $value) = @_;
5237
5238     die "skip\n" if !$conf->{$opt};
5239
5240     my $hotplug_options = { fstrim_cloned_disks => 1 };
5241
5242     my $old_agent = parse_guest_agent($conf);
5243     my $agent = parse_guest_agent({$opt => $value});
5244
5245     for my $option (keys %$agent) { # added/changed options
5246         next if defined($hotplug_options->{$option});
5247         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5248     }
5249
5250     for my $option (keys %$old_agent) { # removed options
5251         next if defined($hotplug_options->{$option});
5252         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5253     }
5254
5255     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5256 }
5257
5258 sub vmconfig_update_disk {
5259     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5260
5261     my $drive = parse_drive($opt, $value);
5262
5263     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5264         my $media = $drive->{media} || 'disk';
5265         my $oldmedia = $old_drive->{media} || 'disk';
5266         die "unable to change media type\n" if $media ne $oldmedia;
5267
5268         if (!drive_is_cdrom($old_drive)) {
5269
5270             if ($drive->{file} ne $old_drive->{file}) {
5271
5272                 die "skip\n" if !$hotplug;
5273
5274                 # unplug and register as unused
5275                 vm_deviceunplug($vmid, $conf, $opt);
5276                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5277
5278             } else {
5279                 # update existing disk
5280
5281                 # skip non hotpluggable value
5282                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5283                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5284                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5285                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5286                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5287                     die "skip\n";
5288                 }
5289
5290                 # apply throttle
5291                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5292                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5293                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5294                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5295                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5296                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5297                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5298                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5299                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5300                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5301                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5302                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5303                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5304                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5305                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5306                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5307                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5308                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5309
5310                     qemu_block_set_io_throttle(
5311                         $vmid,"drive-$opt",
5312                         ($drive->{mbps} || 0)*1024*1024,
5313                         ($drive->{mbps_rd} || 0)*1024*1024,
5314                         ($drive->{mbps_wr} || 0)*1024*1024,
5315                          $drive->{iops} || 0,
5316                          $drive->{iops_rd} || 0,
5317                          $drive->{iops_wr} || 0,
5318                         ($drive->{mbps_max} || 0)*1024*1024,
5319                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5320                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5321                          $drive->{iops_max} || 0,
5322                          $drive->{iops_rd_max} || 0,
5323                          $drive->{iops_wr_max} || 0,
5324                          $drive->{bps_max_length} || 1,
5325                          $drive->{bps_rd_max_length} || 1,
5326                          $drive->{bps_wr_max_length} || 1,
5327                          $drive->{iops_max_length} || 1,
5328                          $drive->{iops_rd_max_length} || 1,
5329                          $drive->{iops_wr_max_length} || 1,
5330                     );
5331
5332                 }
5333
5334                 return 1;
5335             }
5336
5337         } else { # cdrom
5338
5339             if ($drive->{file} eq 'none') {
5340                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5341                 if (drive_is_cloudinit($old_drive)) {
5342                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5343                 }
5344             } else {
5345                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5346
5347                 # force eject if locked
5348                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5349
5350                 if ($path) {
5351                     mon_cmd($vmid, "blockdev-change-medium",
5352                         id => "$opt", filename => "$path");
5353                 }
5354             }
5355
5356             return 1;
5357         }
5358     }
5359
5360     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5361     # hotplug new disks
5362     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5363     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5364 }
5365
5366 sub vmconfig_update_cloudinit_drive {
5367     my ($storecfg, $conf, $vmid) = @_;
5368
5369     my $cloudinit_ds = undef;
5370     my $cloudinit_drive = undef;
5371
5372     PVE::QemuConfig->foreach_volume($conf, sub {
5373         my ($ds, $drive) = @_;
5374         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5375             $cloudinit_ds = $ds;
5376             $cloudinit_drive = $drive;
5377         }
5378     });
5379
5380     return if !$cloudinit_drive;
5381
5382     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5383     my $running = PVE::QemuServer::check_running($vmid);
5384
5385     if ($running) {
5386         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5387         if ($path) {
5388             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5389             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5390         }
5391     }
5392 }
5393
5394 # called in locked context by incoming migration
5395 sub vm_migrate_get_nbd_disks {
5396     my ($storecfg, $conf, $replicated_volumes) = @_;
5397
5398     my $local_volumes = {};
5399     PVE::QemuConfig->foreach_volume($conf, sub {
5400         my ($ds, $drive) = @_;
5401
5402         return if drive_is_cdrom($drive);
5403         return if $ds eq 'tpmstate0';
5404
5405         my $volid = $drive->{file};
5406
5407         return if !$volid;
5408
5409         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5410
5411         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5412         return if $scfg->{shared};
5413
5414         # replicated disks re-use existing state via bitmap
5415         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5416         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5417     });
5418     return $local_volumes;
5419 }
5420
5421 # called in locked context by incoming migration
5422 sub vm_migrate_alloc_nbd_disks {
5423     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5424
5425     my $nbd = {};
5426     foreach my $opt (sort keys %$source_volumes) {
5427         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5428
5429         if ($use_existing) {
5430             $nbd->{$opt}->{drivestr} = print_drive($drive);
5431             $nbd->{$opt}->{volid} = $volid;
5432             $nbd->{$opt}->{replicated} = 1;
5433             next;
5434         }
5435
5436         # storage mapping + volname = regular migration
5437         # storage mapping + format = remote migration
5438         # order of precedence, filtered by whether storage supports it:
5439         # 1. explicit requested format
5440         # 2. format of current volume
5441         # 3. default format of storage
5442         if (!$storagemap->{identity}) {
5443             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5444             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5445             if (!$format || !grep { $format eq $_ } @$validFormats) {
5446                 if ($volname) {
5447                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5448                     my $fileFormat = qemu_img_format($scfg, $volname);
5449                     $format = $fileFormat
5450                         if grep { $fileFormat eq $_ } @$validFormats;
5451                 }
5452                 $format //= $defFormat;
5453             }
5454         } else {
5455             # can't happen for remote migration, so $volname is always defined
5456             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5457             $format = qemu_img_format($scfg, $volname);
5458         }
5459
5460         my $size = $drive->{size} / 1024;
5461         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5462         my $newdrive = $drive;
5463         $newdrive->{format} = $format;
5464         $newdrive->{file} = $newvolid;
5465         my $drivestr = print_drive($newdrive);
5466         $nbd->{$opt}->{drivestr} = $drivestr;
5467         $nbd->{$opt}->{volid} = $newvolid;
5468     }
5469
5470     return $nbd;
5471 }
5472
5473 # see vm_start_nolock for parameters, additionally:
5474 # migrate_opts:
5475 #   storagemap = parsed storage map for allocating NBD disks
5476 sub vm_start {
5477     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5478
5479     return PVE::QemuConfig->lock_config($vmid, sub {
5480         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5481
5482         die "you can't start a vm if it's a template\n"
5483             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5484
5485         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5486         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5487
5488         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5489
5490         if ($has_backup_lock && $running) {
5491             # a backup is currently running, attempt to start the guest in the
5492             # existing QEMU instance
5493             return vm_resume($vmid);
5494         }
5495
5496         PVE::QemuConfig->check_lock($conf)
5497             if !($params->{skiplock} || $has_suspended_lock);
5498
5499         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5500
5501         die "VM $vmid already running\n" if $running;
5502
5503         if (my $storagemap = $migrate_opts->{storagemap}) {
5504             my $replicated = $migrate_opts->{replicated_volumes};
5505             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5506             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5507
5508             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5509                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5510             }
5511         }
5512
5513         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5514     });
5515 }
5516
5517
5518 # params:
5519 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5520 #   skiplock => 0/1, skip checking for config lock
5521 #   skiptemplate => 0/1, skip checking whether VM is template
5522 #   forcemachine => to force Qemu machine (rollback/migration)
5523 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5524 #   timeout => in seconds
5525 #   paused => start VM in paused state (backup)
5526 #   resume => resume from hibernation
5527 #   pbs-backing => {
5528 #      sata0 => {
5529 #         repository
5530 #         snapshot
5531 #         keyfile
5532 #         archive
5533 #      },
5534 #      virtio2 => ...
5535 #   }
5536 # migrate_opts:
5537 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5538 #   migratedfrom => source node
5539 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5540 #   network => CIDR of migration network
5541 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5542 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5543 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5544 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5545 #       contained in config
5546 sub vm_start_nolock {
5547     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5548
5549     my $statefile = $params->{statefile};
5550     my $resume = $params->{resume};
5551
5552     my $migratedfrom = $migrate_opts->{migratedfrom};
5553     my $migration_type = $migrate_opts->{type};
5554
5555     my $res = {};
5556
5557     # clean up leftover reboot request files
5558     eval { clear_reboot_request($vmid); };
5559     warn $@ if $@;
5560
5561     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5562         vmconfig_apply_pending($vmid, $conf, $storecfg);
5563         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5564     }
5565
5566     # don't regenerate the ISO if the VM is started as part of a live migration
5567     # this way we can reuse the old ISO with the correct config
5568     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5569
5570     # override offline migrated volumes, conf is out of date still
5571     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5572         for my $key (sort keys $offline_volumes->%*) {
5573             my $parsed = parse_drive($key, $conf->{$key});
5574             $parsed->{file} = $offline_volumes->{$key};
5575             $conf->{$key} = print_drive($parsed);
5576         }
5577     }
5578
5579     my $defaults = load_defaults();
5580
5581     # set environment variable useful inside network script
5582     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5583
5584     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5585
5586     my $forcemachine = $params->{forcemachine};
5587     my $forcecpu = $params->{forcecpu};
5588     if ($resume) {
5589         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5590         $forcemachine = $conf->{runningmachine};
5591         $forcecpu = $conf->{runningcpu};
5592         print "Resuming suspended VM\n";
5593     }
5594
5595     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5596         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5597
5598     my $migration_ip;
5599     my $get_migration_ip = sub {
5600         my ($nodename) = @_;
5601
5602         return $migration_ip if defined($migration_ip);
5603
5604         my $cidr = $migrate_opts->{network};
5605
5606         if (!defined($cidr)) {
5607             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5608             $cidr = $dc_conf->{migration}->{network};
5609         }
5610
5611         if (defined($cidr)) {
5612             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5613
5614             die "could not get IP: no address configured on local " .
5615                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5616
5617             die "could not get IP: multiple addresses configured on local " .
5618                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5619
5620             $migration_ip = @$ips[0];
5621         }
5622
5623         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5624             if !defined($migration_ip);
5625
5626         return $migration_ip;
5627     };
5628
5629     my $migrate_uri;
5630     if ($statefile) {
5631         if ($statefile eq 'tcp') {
5632             my $localip = "localhost";
5633             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5634             my $nodename = nodename();
5635
5636             if (!defined($migration_type)) {
5637                 if (defined($datacenterconf->{migration}->{type})) {
5638                     $migration_type = $datacenterconf->{migration}->{type};
5639                 } else {
5640                     $migration_type = 'secure';
5641                 }
5642             }
5643
5644             if ($migration_type eq 'insecure') {
5645                 $localip = $get_migration_ip->($nodename);
5646                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5647             }
5648
5649             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5650             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5651             $migrate_uri = "tcp:${localip}:${migrate_port}";
5652             push @$cmd, '-incoming', $migrate_uri;
5653             push @$cmd, '-S';
5654
5655         } elsif ($statefile eq 'unix') {
5656             # should be default for secure migrations as a ssh TCP forward
5657             # tunnel is not deterministic reliable ready and fails regurarly
5658             # to set up in time, so use UNIX socket forwards
5659             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5660             unlink $socket_addr;
5661
5662             $migrate_uri = "unix:$socket_addr";
5663
5664             push @$cmd, '-incoming', $migrate_uri;
5665             push @$cmd, '-S';
5666
5667         } elsif (-e $statefile) {
5668             push @$cmd, '-loadstate', $statefile;
5669         } else {
5670             my $statepath = PVE::Storage::path($storecfg, $statefile);
5671             push @$vollist, $statefile;
5672             push @$cmd, '-loadstate', $statepath;
5673         }
5674     } elsif ($params->{paused}) {
5675         push @$cmd, '-S';
5676     }
5677
5678     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5679
5680     my $pci_devices = {}; # host pci devices
5681     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5682         my $dev = $conf->{"hostpci$i"} or next;
5683         $pci_devices->{$i} = parse_hostpci($dev);
5684     }
5685
5686     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5687     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5688
5689     # map to a flat list of pci ids
5690     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5691
5692     # reserve all PCI IDs before actually doing anything with them
5693     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5694
5695     eval {
5696         my $uuid;
5697         for my $id (sort keys %$pci_devices) {
5698             my $d = $pci_devices->{$id};
5699             for my $dev ($d->{pciid}->@*) {
5700                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5701
5702                 # nvidia grid needs the uuid of the mdev as qemu parameter
5703                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5704                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5705                 }
5706             }
5707         }
5708         push @$cmd, '-uuid', $uuid if defined($uuid);
5709     };
5710     if (my $err = $@) {
5711         eval { cleanup_pci_devices($vmid, $conf) };
5712         warn $@ if $@;
5713         die $err;
5714     }
5715
5716     PVE::Storage::activate_volumes($storecfg, $vollist);
5717
5718     eval {
5719         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5720     };
5721     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5722     # timeout should be more than enough here...
5723     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5724
5725     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5726
5727     my %run_params = (
5728         timeout => $statefile ? undef : $start_timeout,
5729         umask => 0077,
5730         noerr => 1,
5731     );
5732
5733     # when migrating, prefix QEMU output so other side can pick up any
5734     # errors that might occur and show the user
5735     if ($migratedfrom) {
5736         $run_params{quiet} = 1;
5737         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5738     }
5739
5740     my %systemd_properties = (
5741         Slice => 'qemu.slice',
5742         KillMode => 'process',
5743         SendSIGKILL => 0,
5744         TimeoutStopUSec => ULONG_MAX, # infinity
5745     );
5746
5747     if (PVE::CGroup::cgroup_mode() == 2) {
5748         $systemd_properties{CPUWeight} = $cpuunits;
5749     } else {
5750         $systemd_properties{CPUShares} = $cpuunits;
5751     }
5752
5753     if (my $cpulimit = $conf->{cpulimit}) {
5754         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5755     }
5756     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5757
5758     my $run_qemu = sub {
5759         PVE::Tools::run_fork sub {
5760             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5761
5762             my $tpmpid;
5763             if (my $tpm = $conf->{tpmstate0}) {
5764                 # start the TPM emulator so QEMU can connect on start
5765                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5766             }
5767
5768             my $exitcode = run_command($cmd, %run_params);
5769             if ($exitcode) {
5770                 if ($tpmpid) {
5771                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5772                     kill 'TERM', $tpmpid;
5773                 }
5774                 die "QEMU exited with code $exitcode\n";
5775             }
5776         };
5777     };
5778
5779     if ($conf->{hugepages}) {
5780
5781         my $code = sub {
5782             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5783             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5784
5785             PVE::QemuServer::Memory::hugepages_mount();
5786             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5787
5788             eval { $run_qemu->() };
5789             if (my $err = $@) {
5790                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5791                     if !$conf->{keephugepages};
5792                 die $err;
5793             }
5794
5795             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5796                 if !$conf->{keephugepages};
5797         };
5798         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5799
5800     } else {
5801         eval { $run_qemu->() };
5802     }
5803
5804     if (my $err = $@) {
5805         # deactivate volumes if start fails
5806         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5807         warn $@ if $@;
5808         eval { cleanup_pci_devices($vmid, $conf) };
5809         warn $@ if $@;
5810
5811         die "start failed: $err";
5812     }
5813
5814     # re-reserve all PCI IDs now that we can know the actual VM PID
5815     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5816     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5817     warn $@ if $@;
5818
5819     print "migration listens on $migrate_uri\n" if $migrate_uri;
5820     $res->{migrate_uri} = $migrate_uri;
5821
5822     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5823         eval { mon_cmd($vmid, "cont"); };
5824         warn $@ if $@;
5825     }
5826
5827     #start nbd server for storage migration
5828     if (my $nbd = $migrate_opts->{nbd}) {
5829         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5830
5831         my $migrate_storage_uri;
5832         # nbd_protocol_version > 0 for unix socket support
5833         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5834             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5835             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5836             $migrate_storage_uri = "nbd:unix:$socket_path";
5837         } else {
5838             my $nodename = nodename();
5839             my $localip = $get_migration_ip->($nodename);
5840             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5841             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5842
5843             mon_cmd($vmid, "nbd-server-start", addr => {
5844                 type => 'inet',
5845                 data => {
5846                     host => "${localip}",
5847                     port => "${storage_migrate_port}",
5848                 },
5849             });
5850             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5851             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5852         }
5853
5854         $res->{migrate_storage_uri} = $migrate_storage_uri;
5855
5856         foreach my $opt (sort keys %$nbd) {
5857             my $drivestr = $nbd->{$opt}->{drivestr};
5858             my $volid = $nbd->{$opt}->{volid};
5859             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5860             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5861             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5862             print "re-using replicated volume: $opt - $volid\n"
5863                 if $nbd->{$opt}->{replicated};
5864
5865             $res->{drives}->{$opt} = $nbd->{$opt};
5866             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5867         }
5868     }
5869
5870     if ($migratedfrom) {
5871         eval {
5872             set_migration_caps($vmid);
5873         };
5874         warn $@ if $@;
5875
5876         if ($spice_port) {
5877             print "spice listens on port $spice_port\n";
5878             $res->{spice_port} = $spice_port;
5879             if ($migrate_opts->{spice_ticket}) {
5880                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5881                     $migrate_opts->{spice_ticket});
5882                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5883             }
5884         }
5885
5886     } else {
5887         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5888             if !$statefile && $conf->{balloon};
5889
5890         foreach my $opt (keys %$conf) {
5891             next if $opt !~  m/^net\d+$/;
5892             my $nicconf = parse_net($conf->{$opt});
5893             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5894         }
5895         add_nets_bridge_fdb($conf, $vmid);
5896     }
5897
5898     mon_cmd($vmid, 'qom-set',
5899                 path => "machine/peripheral/balloon0",
5900                 property => "guest-stats-polling-interval",
5901                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5902
5903     if ($resume) {
5904         print "Resumed VM, removing state\n";
5905         if (my $vmstate = $conf->{vmstate}) {
5906             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5907             PVE::Storage::vdisk_free($storecfg, $vmstate);
5908         }
5909         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5910         PVE::QemuConfig->write_config($vmid, $conf);
5911     }
5912
5913     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5914
5915     return $res;
5916 }
5917
5918 sub vm_commandline {
5919     my ($storecfg, $vmid, $snapname) = @_;
5920
5921     my $conf = PVE::QemuConfig->load_config($vmid);
5922
5923     my ($forcemachine, $forcecpu);
5924     if ($snapname) {
5925         my $snapshot = $conf->{snapshots}->{$snapname};
5926         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5927
5928         # check for machine or CPU overrides in snapshot
5929         $forcemachine = $snapshot->{runningmachine};
5930         $forcecpu = $snapshot->{runningcpu};
5931
5932         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5933
5934         $conf = $snapshot;
5935     }
5936
5937     my $defaults = load_defaults();
5938
5939     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5940
5941     return PVE::Tools::cmd2string($cmd);
5942 }
5943
5944 sub vm_reset {
5945     my ($vmid, $skiplock) = @_;
5946
5947     PVE::QemuConfig->lock_config($vmid, sub {
5948
5949         my $conf = PVE::QemuConfig->load_config($vmid);
5950
5951         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5952
5953         mon_cmd($vmid, "system_reset");
5954     });
5955 }
5956
5957 sub get_vm_volumes {
5958     my ($conf) = @_;
5959
5960     my $vollist = [];
5961     foreach_volid($conf, sub {
5962         my ($volid, $attr) = @_;
5963
5964         return if $volid =~ m|^/|;
5965
5966         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5967         return if !$sid;
5968
5969         push @$vollist, $volid;
5970     });
5971
5972     return $vollist;
5973 }
5974
5975 sub cleanup_pci_devices {
5976     my ($vmid, $conf) = @_;
5977
5978     foreach my $key (keys %$conf) {
5979         next if $key !~ m/^hostpci(\d+)$/;
5980         my $hostpciindex = $1;
5981         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5982         my $d = parse_hostpci($conf->{$key});
5983         if ($d->{mdev}) {
5984             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
5985             # don't want to break ABI just for this two liner
5986             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
5987             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
5988         }
5989     }
5990     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
5991 }
5992
5993 sub vm_stop_cleanup {
5994     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5995
5996     eval {
5997
5998         if (!$keepActive) {
5999             my $vollist = get_vm_volumes($conf);
6000             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6001
6002             if (my $tpmdrive = $conf->{tpmstate0}) {
6003                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6004                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6005                 if ($storeid) {
6006                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6007                 }
6008             }
6009         }
6010
6011         foreach my $ext (qw(mon qmp pid vnc qga)) {
6012             unlink "/var/run/qemu-server/${vmid}.$ext";
6013         }
6014
6015         if ($conf->{ivshmem}) {
6016             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6017             # just delete it for now, VMs which have this already open do not
6018             # are affected, but new VMs will get a separated one. If this
6019             # becomes an issue we either add some sort of ref-counting or just
6020             # add a "don't delete on stop" flag to the ivshmem format.
6021             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6022         }
6023
6024         cleanup_pci_devices($vmid, $conf);
6025
6026         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6027     };
6028     warn $@ if $@; # avoid errors - just warn
6029 }
6030
6031 # call only in locked context
6032 sub _do_vm_stop {
6033     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6034
6035     my $pid = check_running($vmid, $nocheck);
6036     return if !$pid;
6037
6038     my $conf;
6039     if (!$nocheck) {
6040         $conf = PVE::QemuConfig->load_config($vmid);
6041         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6042         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6043             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6044             $timeout = $opts->{down} if $opts->{down};
6045         }
6046         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6047     }
6048
6049     eval {
6050         if ($shutdown) {
6051             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6052                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6053             } else {
6054                 mon_cmd($vmid, "system_powerdown");
6055             }
6056         } else {
6057             mon_cmd($vmid, "quit");
6058         }
6059     };
6060     my $err = $@;
6061
6062     if (!$err) {
6063         $timeout = 60 if !defined($timeout);
6064
6065         my $count = 0;
6066         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6067             $count++;
6068             sleep 1;
6069         }
6070
6071         if ($count >= $timeout) {
6072             if ($force) {
6073                 warn "VM still running - terminating now with SIGTERM\n";
6074                 kill 15, $pid;
6075             } else {
6076                 die "VM quit/powerdown failed - got timeout\n";
6077             }
6078         } else {
6079             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6080             return;
6081         }
6082     } else {
6083        if (!check_running($vmid, $nocheck)) {
6084             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6085             return;
6086        }
6087        if ($force) {
6088             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6089             kill 15, $pid;
6090         } else {
6091             die "VM quit/powerdown failed\n";
6092         }
6093     }
6094
6095     # wait again
6096     $timeout = 10;
6097
6098     my $count = 0;
6099     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6100         $count++;
6101         sleep 1;
6102     }
6103
6104     if ($count >= $timeout) {
6105         warn "VM still running - terminating now with SIGKILL\n";
6106         kill 9, $pid;
6107         sleep 1;
6108     }
6109
6110     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6111 }
6112
6113 # Note: use $nocheck to skip tests if VM configuration file exists.
6114 # We need that when migration VMs to other nodes (files already moved)
6115 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6116 sub vm_stop {
6117     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6118
6119     $force = 1 if !defined($force) && !$shutdown;
6120
6121     if ($migratedfrom){
6122         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6123         kill 15, $pid if $pid;
6124         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6125         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6126         return;
6127     }
6128
6129     PVE::QemuConfig->lock_config($vmid, sub {
6130         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6131    });
6132 }
6133
6134 sub vm_reboot {
6135     my ($vmid, $timeout) = @_;
6136
6137     PVE::QemuConfig->lock_config($vmid, sub {
6138         eval {
6139
6140             # only reboot if running, as qmeventd starts it again on a stop event
6141             return if !check_running($vmid);
6142
6143             create_reboot_request($vmid);
6144
6145             my $storecfg = PVE::Storage::config();
6146             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6147
6148         };
6149         if (my $err = $@) {
6150             # avoid that the next normal shutdown will be confused for a reboot
6151             clear_reboot_request($vmid);
6152             die $err;
6153         }
6154    });
6155 }
6156
6157 # note: if using the statestorage parameter, the caller has to check privileges
6158 sub vm_suspend {
6159     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6160
6161     my $conf;
6162     my $path;
6163     my $storecfg;
6164     my $vmstate;
6165
6166     PVE::QemuConfig->lock_config($vmid, sub {
6167
6168         $conf = PVE::QemuConfig->load_config($vmid);
6169
6170         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6171         PVE::QemuConfig->check_lock($conf)
6172             if !($skiplock || $is_backing_up);
6173
6174         die "cannot suspend to disk during backup\n"
6175             if $is_backing_up && $includestate;
6176
6177         if ($includestate) {
6178             $conf->{lock} = 'suspending';
6179             my $date = strftime("%Y-%m-%d", localtime(time()));
6180             $storecfg = PVE::Storage::config();
6181             if (!$statestorage) {
6182                 $statestorage = find_vmstate_storage($conf, $storecfg);
6183                 # check permissions for the storage
6184                 my $rpcenv = PVE::RPCEnvironment::get();
6185                 if ($rpcenv->{type} ne 'cli') {
6186                     my $authuser = $rpcenv->get_user();
6187                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6188                 }
6189             }
6190
6191
6192             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6193                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6194             $path = PVE::Storage::path($storecfg, $vmstate);
6195             PVE::QemuConfig->write_config($vmid, $conf);
6196         } else {
6197             mon_cmd($vmid, "stop");
6198         }
6199     });
6200
6201     if ($includestate) {
6202         # save vm state
6203         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6204
6205         eval {
6206             set_migration_caps($vmid, 1);
6207             mon_cmd($vmid, "savevm-start", statefile => $path);
6208             for(;;) {
6209                 my $state = mon_cmd($vmid, "query-savevm");
6210                 if (!$state->{status}) {
6211                     die "savevm not active\n";
6212                 } elsif ($state->{status} eq 'active') {
6213                     sleep(1);
6214                     next;
6215                 } elsif ($state->{status} eq 'completed') {
6216                     print "State saved, quitting\n";
6217                     last;
6218                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6219                     die "query-savevm failed with error '$state->{error}'\n"
6220                 } else {
6221                     die "query-savevm returned status '$state->{status}'\n";
6222                 }
6223             }
6224         };
6225         my $err = $@;
6226
6227         PVE::QemuConfig->lock_config($vmid, sub {
6228             $conf = PVE::QemuConfig->load_config($vmid);
6229             if ($err) {
6230                 # cleanup, but leave suspending lock, to indicate something went wrong
6231                 eval {
6232                     mon_cmd($vmid, "savevm-end");
6233                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6234                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6235                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6236                     PVE::QemuConfig->write_config($vmid, $conf);
6237                 };
6238                 warn $@ if $@;
6239                 die $err;
6240             }
6241
6242             die "lock changed unexpectedly\n"
6243                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6244
6245             mon_cmd($vmid, "quit");
6246             $conf->{lock} = 'suspended';
6247             PVE::QemuConfig->write_config($vmid, $conf);
6248         });
6249     }
6250 }
6251
6252 sub vm_resume {
6253     my ($vmid, $skiplock, $nocheck) = @_;
6254
6255     PVE::QemuConfig->lock_config($vmid, sub {
6256         my $res = mon_cmd($vmid, 'query-status');
6257         my $resume_cmd = 'cont';
6258         my $reset = 0;
6259         my $conf = PVE::QemuConfig->load_config($vmid);
6260
6261         if ($res->{status}) {
6262             return if $res->{status} eq 'running'; # job done, go home
6263             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6264             $reset = 1 if $res->{status} eq 'shutdown';
6265         }
6266
6267         if (!$nocheck) {
6268
6269             PVE::QemuConfig->check_lock($conf)
6270                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6271         }
6272
6273         if ($reset) {
6274             # required if a VM shuts down during a backup and we get a resume
6275             # request before the backup finishes for example
6276             mon_cmd($vmid, "system_reset");
6277         }
6278
6279         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6280
6281         mon_cmd($vmid, $resume_cmd);
6282     });
6283 }
6284
6285 sub vm_sendkey {
6286     my ($vmid, $skiplock, $key) = @_;
6287
6288     PVE::QemuConfig->lock_config($vmid, sub {
6289
6290         my $conf = PVE::QemuConfig->load_config($vmid);
6291
6292         # there is no qmp command, so we use the human monitor command
6293         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6294         die $res if $res ne '';
6295     });
6296 }
6297
6298 # vzdump restore implementaion
6299
6300 sub tar_archive_read_firstfile {
6301     my $archive = shift;
6302
6303     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6304
6305     # try to detect archive type first
6306     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6307         die "unable to open file '$archive'\n";
6308     my $firstfile = <$fh>;
6309     kill 15, $pid;
6310     close $fh;
6311
6312     die "ERROR: archive contaions no data\n" if !$firstfile;
6313     chomp $firstfile;
6314
6315     return $firstfile;
6316 }
6317
6318 sub tar_restore_cleanup {
6319     my ($storecfg, $statfile) = @_;
6320
6321     print STDERR "starting cleanup\n";
6322
6323     if (my $fd = IO::File->new($statfile, "r")) {
6324         while (defined(my $line = <$fd>)) {
6325             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6326                 my $volid = $2;
6327                 eval {
6328                     if ($volid =~ m|^/|) {
6329                         unlink $volid || die 'unlink failed\n';
6330                     } else {
6331                         PVE::Storage::vdisk_free($storecfg, $volid);
6332                     }
6333                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6334                 };
6335                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6336             } else {
6337                 print STDERR "unable to parse line in statfile - $line";
6338             }
6339         }
6340         $fd->close();
6341     }
6342 }
6343
6344 sub restore_file_archive {
6345     my ($archive, $vmid, $user, $opts) = @_;
6346
6347     return restore_vma_archive($archive, $vmid, $user, $opts)
6348         if $archive eq '-';
6349
6350     my $info = PVE::Storage::archive_info($archive);
6351     my $format = $opts->{format} // $info->{format};
6352     my $comp = $info->{compression};
6353
6354     # try to detect archive format
6355     if ($format eq 'tar') {
6356         return restore_tar_archive($archive, $vmid, $user, $opts);
6357     } else {
6358         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6359     }
6360 }
6361
6362 # hepler to remove disks that will not be used after restore
6363 my $restore_cleanup_oldconf = sub {
6364     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6365
6366     my $kept_disks = {};
6367
6368     PVE::QemuConfig->foreach_volume($oldconf, sub {
6369         my ($ds, $drive) = @_;
6370
6371         return if drive_is_cdrom($drive, 1);
6372
6373         my $volid = $drive->{file};
6374         return if !$volid || $volid =~ m|^/|;
6375
6376         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6377         return if !$path || !$owner || ($owner != $vmid);
6378
6379         # Note: only delete disk we want to restore
6380         # other volumes will become unused
6381         if ($virtdev_hash->{$ds}) {
6382             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6383             if (my $err = $@) {
6384                 warn $err;
6385             }
6386         } else {
6387             $kept_disks->{$volid} = 1;
6388         }
6389     });
6390
6391     # after the restore we have no snapshots anymore
6392     for my $snapname (keys $oldconf->{snapshots}->%*) {
6393         my $snap = $oldconf->{snapshots}->{$snapname};
6394         if ($snap->{vmstate}) {
6395             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6396             if (my $err = $@) {
6397                 warn $err;
6398             }
6399         }
6400
6401         for my $volid (keys $kept_disks->%*) {
6402             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6403             warn $@ if $@;
6404         }
6405     }
6406 };
6407
6408 # Helper to parse vzdump backup device hints
6409 #
6410 # $rpcenv: Environment, used to ckeck storage permissions
6411 # $user: User ID, to check storage permissions
6412 # $storecfg: Storage configuration
6413 # $fh: the file handle for reading the configuration
6414 # $devinfo: should contain device sizes for all backu-up'ed devices
6415 # $options: backup options (pool, default storage)
6416 #
6417 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6418 my $parse_backup_hints = sub {
6419     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6420
6421     my $check_storage = sub { # assert if an image can be allocate
6422         my ($storeid, $scfg) = @_;
6423         die "Content type 'images' is not available on storage '$storeid'\n"
6424             if !$scfg->{content}->{images};
6425         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6426             if $user ne 'root@pam';
6427     };
6428
6429     my $virtdev_hash = {};
6430     while (defined(my $line = <$fh>)) {
6431         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6432             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6433             die "archive does not contain data for drive '$virtdev'\n"
6434                 if !$devinfo->{$devname};
6435
6436             if (defined($options->{storage})) {
6437                 $storeid = $options->{storage} || 'local';
6438             } elsif (!$storeid) {
6439                 $storeid = 'local';
6440             }
6441             $format = 'raw' if !$format;
6442             $devinfo->{$devname}->{devname} = $devname;
6443             $devinfo->{$devname}->{virtdev} = $virtdev;
6444             $devinfo->{$devname}->{format} = $format;
6445             $devinfo->{$devname}->{storeid} = $storeid;
6446
6447             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6448             $check_storage->($storeid, $scfg); # permission and content type check
6449
6450             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6451         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6452             my $virtdev = $1;
6453             my $drive = parse_drive($virtdev, $2);
6454
6455             if (drive_is_cloudinit($drive)) {
6456                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6457                 $storeid = $options->{storage} if defined ($options->{storage});
6458                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6459                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6460
6461                 $check_storage->($storeid, $scfg); # permission and content type check
6462
6463                 $virtdev_hash->{$virtdev} = {
6464                     format => $format,
6465                     storeid => $storeid,
6466                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6467                     is_cloudinit => 1,
6468                 };
6469             }
6470         }
6471     }
6472
6473     return $virtdev_hash;
6474 };
6475
6476 # Helper to allocate and activate all volumes required for a restore
6477 #
6478 # $storecfg: Storage configuration
6479 # $virtdev_hash: as returned by parse_backup_hints()
6480 #
6481 # Returns: { $virtdev => $volid }
6482 my $restore_allocate_devices = sub {
6483     my ($storecfg, $virtdev_hash, $vmid) = @_;
6484
6485     my $map = {};
6486     foreach my $virtdev (sort keys %$virtdev_hash) {
6487         my $d = $virtdev_hash->{$virtdev};
6488         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6489         my $storeid = $d->{storeid};
6490         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6491
6492         # test if requested format is supported
6493         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6494         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6495         $d->{format} = $defFormat if !$supported;
6496
6497         my $name;
6498         if ($d->{is_cloudinit}) {
6499             $name = "vm-$vmid-cloudinit";
6500             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6501             if ($scfg->{path}) {
6502                 $name .= ".$d->{format}";
6503             }
6504         }
6505
6506         my $volid = PVE::Storage::vdisk_alloc(
6507             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6508
6509         print STDERR "new volume ID is '$volid'\n";
6510         $d->{volid} = $volid;
6511
6512         PVE::Storage::activate_volumes($storecfg, [$volid]);
6513
6514         $map->{$virtdev} = $volid;
6515     }
6516
6517     return $map;
6518 };
6519
6520 sub restore_update_config_line {
6521     my ($cookie, $map, $line, $unique) = @_;
6522
6523     return '' if $line =~ m/^\#qmdump\#/;
6524     return '' if $line =~ m/^\#vzdump\#/;
6525     return '' if $line =~ m/^lock:/;
6526     return '' if $line =~ m/^unused\d+:/;
6527     return '' if $line =~ m/^parent:/;
6528
6529     my $res = '';
6530
6531     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6532     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6533         # try to convert old 1.X settings
6534         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6535         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6536             my ($model, $macaddr) = split(/\=/, $devconfig);
6537             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6538             my $net = {
6539                 model => $model,
6540                 bridge => "vmbr$ind",
6541                 macaddr => $macaddr,
6542             };
6543             my $netstr = print_net($net);
6544
6545             $res .= "net$cookie->{netcount}: $netstr\n";
6546             $cookie->{netcount}++;
6547         }
6548     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6549         my ($id, $netstr) = ($1, $2);
6550         my $net = parse_net($netstr);
6551         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6552         $netstr = print_net($net);
6553         $res .= "$id: $netstr\n";
6554     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6555         my $virtdev = $1;
6556         my $value = $3;
6557         my $di = parse_drive($virtdev, $value);
6558         if (defined($di->{backup}) && !$di->{backup}) {
6559             $res .= "#$line";
6560         } elsif ($map->{$virtdev}) {
6561             delete $di->{format}; # format can change on restore
6562             $di->{file} = $map->{$virtdev};
6563             $value = print_drive($di);
6564             $res .= "$virtdev: $value\n";
6565         } else {
6566             $res .= $line;
6567         }
6568     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6569         my $vmgenid = $1;
6570         if ($vmgenid ne '0') {
6571             # always generate a new vmgenid if there was a valid one setup
6572             $vmgenid = generate_uuid();
6573         }
6574         $res .= "vmgenid: $vmgenid\n";
6575     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6576         my ($uuid, $uuid_str);
6577         UUID::generate($uuid);
6578         UUID::unparse($uuid, $uuid_str);
6579         my $smbios1 = parse_smbios1($2);
6580         $smbios1->{uuid} = $uuid_str;
6581         $res .= $1.print_smbios1($smbios1)."\n";
6582     } else {
6583         $res .= $line;
6584     }
6585
6586     return $res;
6587 }
6588
6589 my $restore_deactivate_volumes = sub {
6590     my ($storecfg, $virtdev_hash) = @_;
6591
6592     my $vollist = [];
6593     for my $dev (values $virtdev_hash->%*) {
6594         push $vollist->@*, $dev->{volid} if $dev->{volid};
6595     }
6596
6597     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6598     print STDERR $@ if $@;
6599 };
6600
6601 my $restore_destroy_volumes = sub {
6602     my ($storecfg, $virtdev_hash) = @_;
6603
6604     for my $dev (values $virtdev_hash->%*) {
6605         my $volid = $dev->{volid} or next;
6606         eval {
6607             PVE::Storage::vdisk_free($storecfg, $volid);
6608             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6609         };
6610         print STDERR "unable to cleanup '$volid' - $@" if $@;
6611     }
6612 };
6613
6614 my $restore_merge_config = sub {
6615     my ($filename, $backup_conf_raw, $override_conf) = @_;
6616
6617     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6618     for my $key (keys $override_conf->%*) {
6619         $backup_conf->{$key} = $override_conf->{$key};
6620     }
6621
6622     return $backup_conf;
6623 };
6624
6625 sub scan_volids {
6626     my ($cfg, $vmid) = @_;
6627
6628     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6629
6630     my $volid_hash = {};
6631     foreach my $storeid (keys %$info) {
6632         foreach my $item (@{$info->{$storeid}}) {
6633             next if !($item->{volid} && $item->{size});
6634             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6635             $volid_hash->{$item->{volid}} = $item;
6636         }
6637     }
6638
6639     return $volid_hash;
6640 }
6641
6642 sub update_disk_config {
6643     my ($vmid, $conf, $volid_hash) = @_;
6644
6645     my $changes;
6646     my $prefix = "VM $vmid";
6647
6648     # used and unused disks
6649     my $referenced = {};
6650
6651     # Note: it is allowed to define multiple storages with same path (alias), so
6652     # we need to check both 'volid' and real 'path' (two different volid can point
6653     # to the same path).
6654
6655     my $referencedpath = {};
6656
6657     # update size info
6658     PVE::QemuConfig->foreach_volume($conf, sub {
6659         my ($opt, $drive) = @_;
6660
6661         my $volid = $drive->{file};
6662         return if !$volid;
6663         my $volume = $volid_hash->{$volid};
6664
6665         # mark volid as "in-use" for next step
6666         $referenced->{$volid} = 1;
6667         if ($volume && (my $path = $volume->{path})) {
6668             $referencedpath->{$path} = 1;
6669         }
6670
6671         return if drive_is_cdrom($drive);
6672         return if !$volume;
6673
6674         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6675         if (defined($updated)) {
6676             $changes = 1;
6677             $conf->{$opt} = print_drive($updated);
6678             print "$prefix ($opt): $msg\n";
6679         }
6680     });
6681
6682     # remove 'unusedX' entry if volume is used
6683     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6684         my ($opt, $drive) = @_;
6685
6686         my $volid = $drive->{file};
6687         return if !$volid;
6688
6689         my $path;
6690         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6691         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6692             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6693             $changes = 1;
6694             delete $conf->{$opt};
6695         }
6696
6697         $referenced->{$volid} = 1;
6698         $referencedpath->{$path} = 1 if $path;
6699     });
6700
6701     foreach my $volid (sort keys %$volid_hash) {
6702         next if $volid =~ m/vm-$vmid-state-/;
6703         next if $referenced->{$volid};
6704         my $path = $volid_hash->{$volid}->{path};
6705         next if !$path; # just to be sure
6706         next if $referencedpath->{$path};
6707         $changes = 1;
6708         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6709         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6710         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6711     }
6712
6713     return $changes;
6714 }
6715
6716 sub rescan {
6717     my ($vmid, $nolock, $dryrun) = @_;
6718
6719     my $cfg = PVE::Storage::config();
6720
6721     print "rescan volumes...\n";
6722     my $volid_hash = scan_volids($cfg, $vmid);
6723
6724     my $updatefn =  sub {
6725         my ($vmid) = @_;
6726
6727         my $conf = PVE::QemuConfig->load_config($vmid);
6728
6729         PVE::QemuConfig->check_lock($conf);
6730
6731         my $vm_volids = {};
6732         foreach my $volid (keys %$volid_hash) {
6733             my $info = $volid_hash->{$volid};
6734             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6735         }
6736
6737         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6738
6739         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6740     };
6741
6742     if (defined($vmid)) {
6743         if ($nolock) {
6744             &$updatefn($vmid);
6745         } else {
6746             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6747         }
6748     } else {
6749         my $vmlist = config_list();
6750         foreach my $vmid (keys %$vmlist) {
6751             if ($nolock) {
6752                 &$updatefn($vmid);
6753             } else {
6754                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6755             }
6756         }
6757     }
6758 }
6759
6760 sub restore_proxmox_backup_archive {
6761     my ($archive, $vmid, $user, $options) = @_;
6762
6763     my $storecfg = PVE::Storage::config();
6764
6765     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6766     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6767
6768     my $fingerprint = $scfg->{fingerprint};
6769     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6770
6771     my $repo = PVE::PBSClient::get_repository($scfg);
6772     my $namespace = $scfg->{namespace};
6773
6774     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6775     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6776     local $ENV{PBS_PASSWORD} = $password;
6777     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6778
6779     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6780         PVE::Storage::parse_volname($storecfg, $archive);
6781
6782     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6783
6784     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6785
6786     my $tmpdir = "/var/tmp/vzdumptmp$$";
6787     rmtree $tmpdir;
6788     mkpath $tmpdir;
6789
6790     my $conffile = PVE::QemuConfig->config_file($vmid);
6791      # disable interrupts (always do cleanups)
6792     local $SIG{INT} =
6793         local $SIG{TERM} =
6794         local $SIG{QUIT} =
6795         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6796
6797     # Note: $oldconf is undef if VM does not exists
6798     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6799     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6800     my $new_conf_raw = '';
6801
6802     my $rpcenv = PVE::RPCEnvironment::get();
6803     my $devinfo = {}; # info about drives included in backup
6804     my $virtdev_hash = {}; # info about allocated drives
6805
6806     eval {
6807         # enable interrupts
6808         local $SIG{INT} =
6809             local $SIG{TERM} =
6810             local $SIG{QUIT} =
6811             local $SIG{HUP} =
6812             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6813
6814         my $cfgfn = "$tmpdir/qemu-server.conf";
6815         my $firewall_config_fn = "$tmpdir/fw.conf";
6816         my $index_fn = "$tmpdir/index.json";
6817
6818         my $cmd = "restore";
6819
6820         my $param = [$pbs_backup_name, "index.json", $index_fn];
6821         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6822         my $index = PVE::Tools::file_get_contents($index_fn);
6823         $index = decode_json($index);
6824
6825         foreach my $info (@{$index->{files}}) {
6826             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6827                 my $devname = $1;
6828                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6829                     $devinfo->{$devname}->{size} = $1;
6830                 } else {
6831                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6832                 }
6833             }
6834         }
6835
6836         my $is_qemu_server_backup = scalar(
6837             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6838         );
6839         if (!$is_qemu_server_backup) {
6840             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6841         }
6842         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6843
6844         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6845         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6846
6847         if ($has_firewall_config) {
6848             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6849             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6850
6851             my $pve_firewall_dir = '/etc/pve/firewall';
6852             mkdir $pve_firewall_dir; # make sure the dir exists
6853             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6854         }
6855
6856         my $fh = IO::File->new($cfgfn, "r") ||
6857             die "unable to read qemu-server.conf - $!\n";
6858
6859         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6860
6861         # fixme: rate limit?
6862
6863         # create empty/temp config
6864         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6865
6866         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6867
6868         # allocate volumes
6869         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6870
6871         foreach my $virtdev (sort keys %$virtdev_hash) {
6872             my $d = $virtdev_hash->{$virtdev};
6873             next if $d->{is_cloudinit}; # no need to restore cloudinit
6874
6875             # this fails if storage is unavailable
6876             my $volid = $d->{volid};
6877             my $path = PVE::Storage::path($storecfg, $volid);
6878
6879             # for live-restore we only want to preload the efidisk and TPM state
6880             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6881
6882             my @ns_arg;
6883             if (defined(my $ns = $scfg->{namespace})) {
6884                 @ns_arg = ('--ns', $ns);
6885             }
6886
6887             my $pbs_restore_cmd = [
6888                 '/usr/bin/pbs-restore',
6889                 '--repository', $repo,
6890                 @ns_arg,
6891                 $pbs_backup_name,
6892                 "$d->{devname}.img.fidx",
6893                 $path,
6894                 '--verbose',
6895                 ];
6896
6897             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6898             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6899
6900             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6901                 push @$pbs_restore_cmd, '--skip-zero';
6902             }
6903
6904             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6905             print "restore proxmox backup image: $dbg_cmdstring\n";
6906             run_command($pbs_restore_cmd);
6907         }
6908
6909         $fh->seek(0, 0) || die "seek failed - $!\n";
6910
6911         my $cookie = { netcount => 0 };
6912         while (defined(my $line = <$fh>)) {
6913             $new_conf_raw .= restore_update_config_line(
6914                 $cookie,
6915                 $map,
6916                 $line,
6917                 $options->{unique},
6918             );
6919         }
6920
6921         $fh->close();
6922     };
6923     my $err = $@;
6924
6925     if ($err || !$options->{live}) {
6926         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6927     }
6928
6929     rmtree $tmpdir;
6930
6931     if ($err) {
6932         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6933         die $err;
6934     }
6935
6936     if ($options->{live}) {
6937         # keep lock during live-restore
6938         $new_conf_raw .= "\nlock: create";
6939     }
6940
6941     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6942     PVE::QemuConfig->write_config($vmid, $new_conf);
6943
6944     eval { rescan($vmid, 1); };
6945     warn $@ if $@;
6946
6947     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6948
6949     if ($options->{live}) {
6950         # enable interrupts
6951         local $SIG{INT} =
6952             local $SIG{TERM} =
6953             local $SIG{QUIT} =
6954             local $SIG{HUP} =
6955             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6956
6957         my $conf = PVE::QemuConfig->load_config($vmid);
6958         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6959
6960         # these special drives are already restored before start
6961         delete $devinfo->{'drive-efidisk0'};
6962         delete $devinfo->{'drive-tpmstate0-backup'};
6963
6964         my $pbs_opts = {
6965             repo => $repo,
6966             keyfile => $keyfile,
6967             snapshot => $pbs_backup_name,
6968             namespace => $namespace,
6969         };
6970         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6971
6972         PVE::QemuConfig->remove_lock($vmid, "create");
6973     }
6974 }
6975
6976 sub pbs_live_restore {
6977     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6978
6979     print "starting VM for live-restore\n";
6980     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6981
6982     my $pbs_backing = {};
6983     for my $ds (keys %$restored_disks) {
6984         $ds =~ m/^drive-(.*)$/;
6985         my $confname = $1;
6986         $pbs_backing->{$confname} = {
6987             repository => $opts->{repo},
6988             snapshot => $opts->{snapshot},
6989             archive => "$ds.img.fidx",
6990         };
6991         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6992         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6993
6994         my $drive = parse_drive($confname, $conf->{$confname});
6995         print "restoring '$ds' to '$drive->{file}'\n";
6996     }
6997
6998     my $drives_streamed = 0;
6999     eval {
7000         # make sure HA doesn't interrupt our restore by stopping the VM
7001         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7002             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7003         }
7004
7005         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7006         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7007         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7008
7009         my $qmeventd_fd = register_qmeventd_handle($vmid);
7010
7011         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7012         # this will effectively collapse the backing image chain consisting of
7013         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7014         # removes itself once all backing images vanish with 'auto-remove=on')
7015         my $jobs = {};
7016         for my $ds (sort keys %$restored_disks) {
7017             my $job_id = "restore-$ds";
7018             mon_cmd($vmid, 'block-stream',
7019                 'job-id' => $job_id,
7020                 device => "$ds",
7021             );
7022             $jobs->{$job_id} = {};
7023         }
7024
7025         mon_cmd($vmid, 'cont');
7026         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7027
7028         print "restore-drive jobs finished successfully, removing all tracking block devices"
7029             ." to disconnect from Proxmox Backup Server\n";
7030
7031         for my $ds (sort keys %$restored_disks) {
7032             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7033         }
7034
7035         close($qmeventd_fd);
7036     };
7037
7038     my $err = $@;
7039
7040     if ($err) {
7041         warn "An error occurred during live-restore: $err\n";
7042         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7043         die "live-restore failed\n";
7044     }
7045 }
7046
7047 sub restore_vma_archive {
7048     my ($archive, $vmid, $user, $opts, $comp) = @_;
7049
7050     my $readfrom = $archive;
7051
7052     my $cfg = PVE::Storage::config();
7053     my $commands = [];
7054     my $bwlimit = $opts->{bwlimit};
7055
7056     my $dbg_cmdstring = '';
7057     my $add_pipe = sub {
7058         my ($cmd) = @_;
7059         push @$commands, $cmd;
7060         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7061         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7062         $readfrom = '-';
7063     };
7064
7065     my $input = undef;
7066     if ($archive eq '-') {
7067         $input = '<&STDIN';
7068     } else {
7069         # If we use a backup from a PVE defined storage we also consider that
7070         # storage's rate limit:
7071         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7072         if (defined($volid)) {
7073             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7074             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7075             if ($readlimit) {
7076                 print STDERR "applying read rate limit: $readlimit\n";
7077                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7078                 $add_pipe->($cstream);
7079             }
7080         }
7081     }
7082
7083     if ($comp) {
7084         my $info = PVE::Storage::decompressor_info('vma', $comp);
7085         my $cmd = $info->{decompressor};
7086         push @$cmd, $readfrom;
7087         $add_pipe->($cmd);
7088     }
7089
7090     my $tmpdir = "/var/tmp/vzdumptmp$$";
7091     rmtree $tmpdir;
7092
7093     # disable interrupts (always do cleanups)
7094     local $SIG{INT} =
7095         local $SIG{TERM} =
7096         local $SIG{QUIT} =
7097         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7098
7099     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7100     POSIX::mkfifo($mapfifo, 0600);
7101     my $fifofh;
7102     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7103
7104     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7105
7106     my $oldtimeout;
7107     my $timeout = 5;
7108
7109     my $devinfo = {}; # info about drives included in backup
7110     my $virtdev_hash = {}; # info about allocated drives
7111
7112     my $rpcenv = PVE::RPCEnvironment::get();
7113
7114     my $conffile = PVE::QemuConfig->config_file($vmid);
7115
7116     # Note: $oldconf is undef if VM does not exist
7117     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7118     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7119     my $new_conf_raw = '';
7120
7121     my %storage_limits;
7122
7123     my $print_devmap = sub {
7124         my $cfgfn = "$tmpdir/qemu-server.conf";
7125
7126         # we can read the config - that is already extracted
7127         my $fh = IO::File->new($cfgfn, "r") ||
7128             die "unable to read qemu-server.conf - $!\n";
7129
7130         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7131         if (-f $fwcfgfn) {
7132             my $pve_firewall_dir = '/etc/pve/firewall';
7133             mkdir $pve_firewall_dir; # make sure the dir exists
7134             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7135         }
7136
7137         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7138
7139         foreach my $info (values %{$virtdev_hash}) {
7140             my $storeid = $info->{storeid};
7141             next if defined($storage_limits{$storeid});
7142
7143             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7144             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7145             $storage_limits{$storeid} = $limit * 1024;
7146         }
7147
7148         foreach my $devname (keys %$devinfo) {
7149             die "found no device mapping information for device '$devname'\n"
7150                 if !$devinfo->{$devname}->{virtdev};
7151         }
7152
7153         # create empty/temp config
7154         if ($oldconf) {
7155             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7156             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7157         }
7158
7159         # allocate volumes
7160         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7161
7162         # print restore information to $fifofh
7163         foreach my $virtdev (sort keys %$virtdev_hash) {
7164             my $d = $virtdev_hash->{$virtdev};
7165             next if $d->{is_cloudinit}; # no need to restore cloudinit
7166
7167             my $storeid = $d->{storeid};
7168             my $volid = $d->{volid};
7169
7170             my $map_opts = '';
7171             if (my $limit = $storage_limits{$storeid}) {
7172                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7173             }
7174
7175             my $write_zeros = 1;
7176             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7177                 $write_zeros = 0;
7178             }
7179
7180             my $path = PVE::Storage::path($cfg, $volid);
7181
7182             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7183
7184             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7185         }
7186
7187         $fh->seek(0, 0) || die "seek failed - $!\n";
7188
7189         my $cookie = { netcount => 0 };
7190         while (defined(my $line = <$fh>)) {
7191             $new_conf_raw .= restore_update_config_line(
7192                 $cookie,
7193                 $map,
7194                 $line,
7195                 $opts->{unique},
7196             );
7197         }
7198
7199         $fh->close();
7200     };
7201
7202     eval {
7203         # enable interrupts
7204         local $SIG{INT} =
7205             local $SIG{TERM} =
7206             local $SIG{QUIT} =
7207             local $SIG{HUP} =
7208             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7209         local $SIG{ALRM} = sub { die "got timeout\n"; };
7210
7211         $oldtimeout = alarm($timeout);
7212
7213         my $parser = sub {
7214             my $line = shift;
7215
7216             print "$line\n";
7217
7218             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7219                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7220                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7221             } elsif ($line =~ m/^CTIME: /) {
7222                 # we correctly received the vma config, so we can disable
7223                 # the timeout now for disk allocation (set to 10 minutes, so
7224                 # that we always timeout if something goes wrong)
7225                 alarm(600);
7226                 &$print_devmap();
7227                 print $fifofh "done\n";
7228                 my $tmp = $oldtimeout || 0;
7229                 $oldtimeout = undef;
7230                 alarm($tmp);
7231                 close($fifofh);
7232                 $fifofh = undef;
7233             }
7234         };
7235
7236         print "restore vma archive: $dbg_cmdstring\n";
7237         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7238     };
7239     my $err = $@;
7240
7241     alarm($oldtimeout) if $oldtimeout;
7242
7243     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7244
7245     close($fifofh) if $fifofh;
7246     unlink $mapfifo;
7247     rmtree $tmpdir;
7248
7249     if ($err) {
7250         $restore_destroy_volumes->($cfg, $virtdev_hash);
7251         die $err;
7252     }
7253
7254     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7255     PVE::QemuConfig->write_config($vmid, $new_conf);
7256
7257     eval { rescan($vmid, 1); };
7258     warn $@ if $@;
7259
7260     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7261 }
7262
7263 sub restore_tar_archive {
7264     my ($archive, $vmid, $user, $opts) = @_;
7265
7266     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7267         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7268         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7269     }
7270
7271     if ($archive ne '-') {
7272         my $firstfile = tar_archive_read_firstfile($archive);
7273         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7274             if $firstfile ne 'qemu-server.conf';
7275     }
7276
7277     my $storecfg = PVE::Storage::config();
7278
7279     # avoid zombie disks when restoring over an existing VM -> cleanup first
7280     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7281     # skiplock=1 because qmrestore has set the 'create' lock itself already
7282     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7283     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7284
7285     my $tocmd = "/usr/lib/qemu-server/qmextract";
7286
7287     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7288     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7289     $tocmd .= ' --prealloc' if $opts->{prealloc};
7290     $tocmd .= ' --info' if $opts->{info};
7291
7292     # tar option "xf" does not autodetect compression when read from STDIN,
7293     # so we pipe to zcat
7294     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7295         PVE::Tools::shellquote("--to-command=$tocmd");
7296
7297     my $tmpdir = "/var/tmp/vzdumptmp$$";
7298     mkpath $tmpdir;
7299
7300     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7301     local $ENV{VZDUMP_VMID} = $vmid;
7302     local $ENV{VZDUMP_USER} = $user;
7303
7304     my $conffile = PVE::QemuConfig->config_file($vmid);
7305     my $new_conf_raw = '';
7306
7307     # disable interrupts (always do cleanups)
7308     local $SIG{INT} =
7309         local $SIG{TERM} =
7310         local $SIG{QUIT} =
7311         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7312
7313     eval {
7314         # enable interrupts
7315         local $SIG{INT} =
7316             local $SIG{TERM} =
7317             local $SIG{QUIT} =
7318             local $SIG{HUP} =
7319             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7320
7321         if ($archive eq '-') {
7322             print "extracting archive from STDIN\n";
7323             run_command($cmd, input => "<&STDIN");
7324         } else {
7325             print "extracting archive '$archive'\n";
7326             run_command($cmd);
7327         }
7328
7329         return if $opts->{info};
7330
7331         # read new mapping
7332         my $map = {};
7333         my $statfile = "$tmpdir/qmrestore.stat";
7334         if (my $fd = IO::File->new($statfile, "r")) {
7335             while (defined (my $line = <$fd>)) {
7336                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7337                     $map->{$1} = $2 if $1;
7338                 } else {
7339                     print STDERR "unable to parse line in statfile - $line\n";
7340                 }
7341             }
7342             $fd->close();
7343         }
7344
7345         my $confsrc = "$tmpdir/qemu-server.conf";
7346
7347         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7348
7349         my $cookie = { netcount => 0 };
7350         while (defined (my $line = <$srcfd>)) {
7351             $new_conf_raw .= restore_update_config_line(
7352                 $cookie,
7353                 $map,
7354                 $line,
7355                 $opts->{unique},
7356             );
7357         }
7358
7359         $srcfd->close();
7360     };
7361     if (my $err = $@) {
7362         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7363         die $err;
7364     }
7365
7366     rmtree $tmpdir;
7367
7368     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7369
7370     PVE::Cluster::cfs_update(); # make sure we read new file
7371
7372     eval { rescan($vmid, 1); };
7373     warn $@ if $@;
7374 };
7375
7376 sub foreach_storage_used_by_vm {
7377     my ($conf, $func) = @_;
7378
7379     my $sidhash = {};
7380
7381     PVE::QemuConfig->foreach_volume($conf, sub {
7382         my ($ds, $drive) = @_;
7383         return if drive_is_cdrom($drive);
7384
7385         my $volid = $drive->{file};
7386
7387         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7388         $sidhash->{$sid} = $sid if $sid;
7389     });
7390
7391     foreach my $sid (sort keys %$sidhash) {
7392         &$func($sid);
7393     }
7394 }
7395
7396 my $qemu_snap_storage = {
7397     rbd => 1,
7398 };
7399 sub do_snapshots_with_qemu {
7400     my ($storecfg, $volid, $deviceid) = @_;
7401
7402     return if $deviceid =~ m/tpmstate0/;
7403
7404     my $storage_name = PVE::Storage::parse_volume_id($volid);
7405     my $scfg = $storecfg->{ids}->{$storage_name};
7406     die "could not find storage '$storage_name'\n" if !defined($scfg);
7407
7408     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7409         return 1;
7410     }
7411
7412     if ($volid =~ m/\.(qcow2|qed)$/){
7413         return 1;
7414     }
7415
7416     return;
7417 }
7418
7419 sub qga_check_running {
7420     my ($vmid, $nowarn) = @_;
7421
7422     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7423     if ($@) {
7424         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7425         return 0;
7426     }
7427     return 1;
7428 }
7429
7430 sub template_create {
7431     my ($vmid, $conf, $disk) = @_;
7432
7433     my $storecfg = PVE::Storage::config();
7434
7435     PVE::QemuConfig->foreach_volume($conf, sub {
7436         my ($ds, $drive) = @_;
7437
7438         return if drive_is_cdrom($drive);
7439         return if $disk && $ds ne $disk;
7440
7441         my $volid = $drive->{file};
7442         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7443
7444         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7445         $drive->{file} = $voliddst;
7446         $conf->{$ds} = print_drive($drive);
7447         PVE::QemuConfig->write_config($vmid, $conf);
7448     });
7449 }
7450
7451 sub convert_iscsi_path {
7452     my ($path) = @_;
7453
7454     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7455         my $portal = $1;
7456         my $target = $2;
7457         my $lun = $3;
7458
7459         my $initiator_name = get_initiator_name();
7460
7461         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7462                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7463     }
7464
7465     die "cannot convert iscsi path '$path', unkown format\n";
7466 }
7467
7468 sub qemu_img_convert {
7469     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7470
7471     my $storecfg = PVE::Storage::config();
7472     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7473     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7474
7475     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7476
7477     my $cachemode;
7478     my $src_path;
7479     my $src_is_iscsi = 0;
7480     my $src_format;
7481
7482     if ($src_storeid) {
7483         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7484         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7485         $src_format = qemu_img_format($src_scfg, $src_volname);
7486         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7487         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7488         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7489     } elsif (-f $src_volid || -b $src_volid) {
7490         $src_path = $src_volid;
7491         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7492             $src_format = $1;
7493         }
7494     }
7495
7496     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7497
7498     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7499     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7500     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7501     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7502
7503     my $cmd = [];
7504     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7505     push @$cmd, '-l', "snapshot.name=$snapname"
7506         if $snapname && $src_format && $src_format eq "qcow2";
7507     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7508     push @$cmd, '-T', $cachemode if defined($cachemode);
7509
7510     if ($src_is_iscsi) {
7511         push @$cmd, '--image-opts';
7512         $src_path = convert_iscsi_path($src_path);
7513     } elsif ($src_format) {
7514         push @$cmd, '-f', $src_format;
7515     }
7516
7517     if ($dst_is_iscsi) {
7518         push @$cmd, '--target-image-opts';
7519         $dst_path = convert_iscsi_path($dst_path);
7520     } else {
7521         push @$cmd, '-O', $dst_format;
7522     }
7523
7524     push @$cmd, $src_path;
7525
7526     if (!$dst_is_iscsi && $is_zero_initialized) {
7527         push @$cmd, "zeroinit:$dst_path";
7528     } else {
7529         push @$cmd, $dst_path;
7530     }
7531
7532     my $parser = sub {
7533         my $line = shift;
7534         if($line =~ m/\((\S+)\/100\%\)/){
7535             my $percent = $1;
7536             my $transferred = int($size * $percent / 100);
7537             my $total_h = render_bytes($size, 1);
7538             my $transferred_h = render_bytes($transferred, 1);
7539
7540             print "transferred $transferred_h of $total_h ($percent%)\n";
7541         }
7542
7543     };
7544
7545     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7546     my $err = $@;
7547     die "copy failed: $err" if $err;
7548 }
7549
7550 sub qemu_img_format {
7551     my ($scfg, $volname) = @_;
7552
7553     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7554         return $1;
7555     } else {
7556         return "raw";
7557     }
7558 }
7559
7560 sub qemu_drive_mirror {
7561     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7562
7563     $jobs = {} if !$jobs;
7564
7565     my $qemu_target;
7566     my $format;
7567     $jobs->{"drive-$drive"} = {};
7568
7569     if ($dst_volid =~ /^nbd:/) {
7570         $qemu_target = $dst_volid;
7571         $format = "nbd";
7572     } else {
7573         my $storecfg = PVE::Storage::config();
7574         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7575
7576         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7577
7578         $format = qemu_img_format($dst_scfg, $dst_volname);
7579
7580         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7581
7582         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7583     }
7584
7585     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7586     $opts->{format} = $format if $format;
7587
7588     if (defined($src_bitmap)) {
7589         $opts->{sync} = 'incremental';
7590         $opts->{bitmap} = $src_bitmap;
7591         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7592     }
7593
7594     if (defined($bwlimit)) {
7595         $opts->{speed} = $bwlimit * 1024;
7596         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7597     } else {
7598         print "drive mirror is starting for drive-$drive\n";
7599     }
7600
7601     # if a job already runs for this device we get an error, catch it for cleanup
7602     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7603     if (my $err = $@) {
7604         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7605         warn "$@\n" if $@;
7606         die "mirroring error: $err\n";
7607     }
7608
7609     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7610 }
7611
7612 # $completion can be either
7613 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7614 # 'cancel': wait until all jobs are ready, block-job-cancel them
7615 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7616 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7617 sub qemu_drive_mirror_monitor {
7618     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7619
7620     $completion //= 'complete';
7621     $op //= "mirror";
7622
7623     eval {
7624         my $err_complete = 0;
7625
7626         my $starttime = time ();
7627         while (1) {
7628             die "block job ('$op') timed out\n" if $err_complete > 300;
7629
7630             my $stats = mon_cmd($vmid, "query-block-jobs");
7631             my $ctime = time();
7632
7633             my $running_jobs = {};
7634             for my $stat (@$stats) {
7635                 next if $stat->{type} ne $op;
7636                 $running_jobs->{$stat->{device}} = $stat;
7637             }
7638
7639             my $readycounter = 0;
7640
7641             for my $job_id (sort keys %$jobs) {
7642                 my $job = $running_jobs->{$job_id};
7643
7644                 my $vanished = !defined($job);
7645                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7646                 if($complete || ($vanished && $completion eq 'auto')) {
7647                     print "$job_id: $op-job finished\n";
7648                     delete $jobs->{$job_id};
7649                     next;
7650                 }
7651
7652                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7653
7654                 my $busy = $job->{busy};
7655                 my $ready = $job->{ready};
7656                 if (my $total = $job->{len}) {
7657                     my $transferred = $job->{offset} || 0;
7658                     my $remaining = $total - $transferred;
7659                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7660
7661                     my $duration = $ctime - $starttime;
7662                     my $total_h = render_bytes($total, 1);
7663                     my $transferred_h = render_bytes($transferred, 1);
7664
7665                     my $status = sprintf(
7666                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7667                         render_duration($duration),
7668                     );
7669
7670                     if ($ready) {
7671                         if ($busy) {
7672                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7673                         } else {
7674                             $status .= ", ready";
7675                         }
7676                     }
7677                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7678                     $jobs->{$job_id}->{ready} = $ready;
7679                 }
7680
7681                 $readycounter++ if $job->{ready};
7682             }
7683
7684             last if scalar(keys %$jobs) == 0;
7685
7686             if ($readycounter == scalar(keys %$jobs)) {
7687                 print "all '$op' jobs are ready\n";
7688
7689                 # do the complete later (or has already been done)
7690                 last if $completion eq 'skip' || $completion eq 'auto';
7691
7692                 if ($vmiddst && $vmiddst != $vmid) {
7693                     my $agent_running = $qga && qga_check_running($vmid);
7694                     if ($agent_running) {
7695                         print "freeze filesystem\n";
7696                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7697                         warn $@ if $@;
7698                     } else {
7699                         print "suspend vm\n";
7700                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7701                         warn $@ if $@;
7702                     }
7703
7704                     # if we clone a disk for a new target vm, we don't switch the disk
7705                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7706
7707                     if ($agent_running) {
7708                         print "unfreeze filesystem\n";
7709                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7710                         warn $@ if $@;
7711                     } else {
7712                         print "resume vm\n";
7713                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7714                         warn $@ if $@;
7715                     }
7716
7717                     last;
7718                 } else {
7719
7720                     for my $job_id (sort keys %$jobs) {
7721                         # try to switch the disk if source and destination are on the same guest
7722                         print "$job_id: Completing block job_id...\n";
7723
7724                         my $op;
7725                         if ($completion eq 'complete') {
7726                             $op = 'block-job-complete';
7727                         } elsif ($completion eq 'cancel') {
7728                             $op = 'block-job-cancel';
7729                         } else {
7730                             die "invalid completion value: $completion\n";
7731                         }
7732                         eval { mon_cmd($vmid, $op, device => $job_id) };
7733                         if ($@ =~ m/cannot be completed/) {
7734                             print "$job_id: block job cannot be completed, trying again.\n";
7735                             $err_complete++;
7736                         }else {
7737                             print "$job_id: Completed successfully.\n";
7738                             $jobs->{$job_id}->{complete} = 1;
7739                         }
7740                     }
7741                 }
7742             }
7743             sleep 1;
7744         }
7745     };
7746     my $err = $@;
7747
7748     if ($err) {
7749         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7750         die "block job ($op) error: $err";
7751     }
7752 }
7753
7754 sub qemu_blockjobs_cancel {
7755     my ($vmid, $jobs) = @_;
7756
7757     foreach my $job (keys %$jobs) {
7758         print "$job: Cancelling block job\n";
7759         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7760         $jobs->{$job}->{cancel} = 1;
7761     }
7762
7763     while (1) {
7764         my $stats = mon_cmd($vmid, "query-block-jobs");
7765
7766         my $running_jobs = {};
7767         foreach my $stat (@$stats) {
7768             $running_jobs->{$stat->{device}} = $stat;
7769         }
7770
7771         foreach my $job (keys %$jobs) {
7772
7773             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7774                 print "$job: Done.\n";
7775                 delete $jobs->{$job};
7776             }
7777         }
7778
7779         last if scalar(keys %$jobs) == 0;
7780
7781         sleep 1;
7782     }
7783 }
7784
7785 sub clone_disk {
7786     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7787
7788     my ($vmid, $running) = $source->@{qw(vmid running)};
7789     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7790
7791     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7792     my ($storage, $format) = $dest->@{qw(storage format)};
7793
7794     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7795
7796     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7797         die "cloning from/to EFI disk requires EFI disk\n"
7798             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7799         die "cloning from/to TPM state requires TPM state\n"
7800             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7801
7802         # This would lead to two device nodes in QEMU pointing to the same backing image!
7803         die "cannot change drive name when cloning disk from/to the same VM\n"
7804             if $use_drive_mirror && $vmid == $newvmid;
7805     }
7806
7807     die "cannot move TPM state while VM is running\n"
7808         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7809
7810     my $newvolid;
7811
7812     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7813     print "$src_drivename " if $src_drivename;
7814     print "($drive->{file})\n";
7815
7816     if (!$full) {
7817         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7818         push @$newvollist, $newvolid;
7819     } else {
7820
7821         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7822         $storeid = $storage if $storage;
7823
7824         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7825
7826         my $name = undef;
7827         my $size = undef;
7828         if (drive_is_cloudinit($drive)) {
7829             $name = "vm-$newvmid-cloudinit";
7830             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7831             if ($scfg->{path}) {
7832                 $name .= ".$dst_format";
7833             }
7834             $snapname = undef;
7835             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7836         } elsif ($dst_drivename eq 'efidisk0') {
7837             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7838         } elsif ($dst_drivename eq 'tpmstate0') {
7839             $dst_format = 'raw';
7840             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7841         } else {
7842             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7843         }
7844         $newvolid = PVE::Storage::vdisk_alloc(
7845             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7846         );
7847         push @$newvollist, $newvolid;
7848
7849         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7850
7851         if (drive_is_cloudinit($drive)) {
7852             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7853             # if this is the case, we have to complete any block-jobs still there from
7854             # previous drive-mirrors
7855             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7856                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7857             }
7858             goto no_data_clone;
7859         }
7860
7861         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7862         if ($use_drive_mirror) {
7863             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7864                 $completion, $qga, $bwlimit);
7865         } else {
7866             # TODO: handle bwlimits
7867             if ($dst_drivename eq 'efidisk0') {
7868                 # the relevant data on the efidisk may be smaller than the source
7869                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7870                 # that is given by the OVMF_VARS.fd
7871                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7872                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7873
7874                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7875
7876                 # better for Ceph if block size is not too small, see bug #3324
7877                 my $bs = 1024*1024;
7878
7879                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7880
7881                 if ($src_format eq 'qcow2' && $snapname) {
7882                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7883                         if !min_version(kvm_user_version(), 6, 2);
7884                     push $cmd->@*, '-l', $snapname;
7885                 }
7886                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7887                 run_command($cmd);
7888             } else {
7889                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7890             }
7891         }
7892     }
7893
7894 no_data_clone:
7895     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7896
7897     my $disk = dclone($drive);
7898     delete $disk->{format};
7899     $disk->{file} = $newvolid;
7900     $disk->{size} = $size if defined($size);
7901
7902     return $disk;
7903 }
7904
7905 sub get_running_qemu_version {
7906     my ($vmid) = @_;
7907     my $res = mon_cmd($vmid, "query-version");
7908     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7909 }
7910
7911 sub qemu_use_old_bios_files {
7912     my ($machine_type) = @_;
7913
7914     return if !$machine_type;
7915
7916     my $use_old_bios_files = undef;
7917
7918     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7919         $machine_type = $1;
7920         $use_old_bios_files = 1;
7921     } else {
7922         my $version = extract_version($machine_type, kvm_user_version());
7923         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7924         # load new efi bios files on migration. So this hack is required to allow
7925         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7926         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7927         $use_old_bios_files = !min_version($version, 2, 4);
7928     }
7929
7930     return ($use_old_bios_files, $machine_type);
7931 }
7932
7933 sub get_efivars_size {
7934     my ($conf, $efidisk) = @_;
7935
7936     my $arch = get_vm_arch($conf);
7937     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7938     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7939     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7940     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7941     return -s $ovmf_vars;
7942 }
7943
7944 sub update_efidisk_size {
7945     my ($conf) = @_;
7946
7947     return if !defined($conf->{efidisk0});
7948
7949     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7950     $disk->{size} = get_efivars_size($conf);
7951     $conf->{efidisk0} = print_drive($disk);
7952
7953     return;
7954 }
7955
7956 sub update_tpmstate_size {
7957     my ($conf) = @_;
7958
7959     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7960     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7961     $conf->{tpmstate0} = print_drive($disk);
7962 }
7963
7964 sub create_efidisk($$$$$$$) {
7965     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7966
7967     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7968     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7969
7970     my $vars_size_b = -s $ovmf_vars;
7971     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7972     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7973     PVE::Storage::activate_volumes($storecfg, [$volid]);
7974
7975     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7976     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7977
7978     return ($volid, $size/1024);
7979 }
7980
7981 sub vm_iothreads_list {
7982     my ($vmid) = @_;
7983
7984     my $res = mon_cmd($vmid, 'query-iothreads');
7985
7986     my $iothreads = {};
7987     foreach my $iothread (@$res) {
7988         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7989     }
7990
7991     return $iothreads;
7992 }
7993
7994 sub scsihw_infos {
7995     my ($conf, $drive) = @_;
7996
7997     my $maxdev = 0;
7998
7999     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8000         $maxdev = 7;
8001     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8002         $maxdev = 1;
8003     } else {
8004         $maxdev = 256;
8005     }
8006
8007     my $controller = int($drive->{index} / $maxdev);
8008     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8009         ? "virtioscsi"
8010         : "scsihw";
8011
8012     return ($maxdev, $controller, $controller_prefix);
8013 }
8014
8015 sub resolve_dst_disk_format {
8016         my ($storecfg, $storeid, $src_volname, $format) = @_;
8017         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8018
8019         if (!$format) {
8020             # if no target format is specified, use the source disk format as hint
8021             if ($src_volname) {
8022                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8023                 $format = qemu_img_format($scfg, $src_volname);
8024             } else {
8025                 return $defFormat;
8026             }
8027         }
8028
8029         # test if requested format is supported - else use default
8030         my $supported = grep { $_ eq $format } @$validFormats;
8031         $format = $defFormat if !$supported;
8032         return $format;
8033 }
8034
8035 # NOTE: if this logic changes, please update docs & possibly gui logic
8036 sub find_vmstate_storage {
8037     my ($conf, $storecfg) = @_;
8038
8039     # first, return storage from conf if set
8040     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8041
8042     my ($target, $shared, $local);
8043
8044     foreach_storage_used_by_vm($conf, sub {
8045         my ($sid) = @_;
8046         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8047         my $dst = $scfg->{shared} ? \$shared : \$local;
8048         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8049     });
8050
8051     # second, use shared storage where VM has at least one disk
8052     # third, use local storage where VM has at least one disk
8053     # fall back to local storage
8054     $target = $shared // $local // 'local';
8055
8056     return $target;
8057 }
8058
8059 sub generate_uuid {
8060     my ($uuid, $uuid_str);
8061     UUID::generate($uuid);
8062     UUID::unparse($uuid, $uuid_str);
8063     return $uuid_str;
8064 }
8065
8066 sub generate_smbios1_uuid {
8067     return "uuid=".generate_uuid();
8068 }
8069
8070 sub nbd_stop {
8071     my ($vmid) = @_;
8072
8073     mon_cmd($vmid, 'nbd-server-stop');
8074 }
8075
8076 sub create_reboot_request {
8077     my ($vmid) = @_;
8078     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8079         or die "failed to create reboot trigger file: $!\n";
8080     close($fh);
8081 }
8082
8083 sub clear_reboot_request {
8084     my ($vmid) = @_;
8085     my $path = "/run/qemu-server/$vmid.reboot";
8086     my $res = 0;
8087
8088     $res = unlink($path);
8089     die "could not remove reboot request for $vmid: $!"
8090         if !$res && $! != POSIX::ENOENT;
8091
8092     return $res;
8093 }
8094
8095 sub bootorder_from_legacy {
8096     my ($conf, $bootcfg) = @_;
8097
8098     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8099     my $bootindex_hash = {};
8100     my $i = 1;
8101     foreach my $o (split(//, $boot)) {
8102         $bootindex_hash->{$o} = $i*100;
8103         $i++;
8104     }
8105
8106     my $bootorder = {};
8107
8108     PVE::QemuConfig->foreach_volume($conf, sub {
8109         my ($ds, $drive) = @_;
8110
8111         if (drive_is_cdrom ($drive, 1)) {
8112             if ($bootindex_hash->{d}) {
8113                 $bootorder->{$ds} = $bootindex_hash->{d};
8114                 $bootindex_hash->{d} += 1;
8115             }
8116         } elsif ($bootindex_hash->{c}) {
8117             $bootorder->{$ds} = $bootindex_hash->{c}
8118                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8119             $bootindex_hash->{c} += 1;
8120         }
8121     });
8122
8123     if ($bootindex_hash->{n}) {
8124         for (my $i = 0; $i < $MAX_NETS; $i++) {
8125             my $netname = "net$i";
8126             next if !$conf->{$netname};
8127             $bootorder->{$netname} = $bootindex_hash->{n};
8128             $bootindex_hash->{n} += 1;
8129         }
8130     }
8131
8132     return $bootorder;
8133 }
8134
8135 # Generate default device list for 'boot: order=' property. Matches legacy
8136 # default boot order, but with explicit device names. This is important, since
8137 # the fallback for when neither 'order' nor the old format is specified relies
8138 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8139 sub get_default_bootdevices {
8140     my ($conf) = @_;
8141
8142     my @ret = ();
8143
8144     # harddisk
8145     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8146     push @ret, $first if $first;
8147
8148     # cdrom
8149     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8150     push @ret, $first if $first;
8151
8152     # network
8153     for (my $i = 0; $i < $MAX_NETS; $i++) {
8154         my $netname = "net$i";
8155         next if !$conf->{$netname};
8156         push @ret, $netname;
8157         last;
8158     }
8159
8160     return \@ret;
8161 }
8162
8163 sub device_bootorder {
8164     my ($conf) = @_;
8165
8166     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8167
8168     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8169
8170     my $bootorder = {};
8171     if (!defined($boot) || $boot->{legacy}) {
8172         $bootorder = bootorder_from_legacy($conf, $boot);
8173     } elsif ($boot->{order}) {
8174         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8175         for my $dev (PVE::Tools::split_list($boot->{order})) {
8176             $bootorder->{$dev} = $i++;
8177         }
8178     }
8179
8180     return $bootorder;
8181 }
8182
8183 sub register_qmeventd_handle {
8184     my ($vmid) = @_;
8185
8186     my $fh;
8187     my $peer = "/var/run/qmeventd.sock";
8188     my $count = 0;
8189
8190     for (;;) {
8191         $count++;
8192         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8193         last if $fh;
8194         if ($! != EINTR && $! != EAGAIN) {
8195             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8196         }
8197         if ($count > 4) {
8198             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8199               . "after $count retries\n";
8200         }
8201         usleep(25000);
8202     }
8203
8204     # send handshake to mark VM as backing up
8205     print $fh to_json({vzdump => {vmid => "$vmid"}});
8206
8207     # return handle to be closed later when inhibit is no longer required
8208     return $fh;
8209 }
8210
8211 # bash completion helper
8212
8213 sub complete_backup_archives {
8214     my ($cmdname, $pname, $cvalue) = @_;
8215
8216     my $cfg = PVE::Storage::config();
8217
8218     my $storeid;
8219
8220     if ($cvalue =~ m/^([^:]+):/) {
8221         $storeid = $1;
8222     }
8223
8224     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8225
8226     my $res = [];
8227     foreach my $id (keys %$data) {
8228         foreach my $item (@{$data->{$id}}) {
8229             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8230             push @$res, $item->{volid} if defined($item->{volid});
8231         }
8232     }
8233
8234     return $res;
8235 }
8236
8237 my $complete_vmid_full = sub {
8238     my ($running) = @_;
8239
8240     my $idlist = vmstatus();
8241
8242     my $res = [];
8243
8244     foreach my $id (keys %$idlist) {
8245         my $d = $idlist->{$id};
8246         if (defined($running)) {
8247             next if $d->{template};
8248             next if $running && $d->{status} ne 'running';
8249             next if !$running && $d->{status} eq 'running';
8250         }
8251         push @$res, $id;
8252
8253     }
8254     return $res;
8255 };
8256
8257 sub complete_vmid {
8258     return &$complete_vmid_full();
8259 }
8260
8261 sub complete_vmid_stopped {
8262     return &$complete_vmid_full(0);
8263 }
8264
8265 sub complete_vmid_running {
8266     return &$complete_vmid_full(1);
8267 }
8268
8269 sub complete_storage {
8270
8271     my $cfg = PVE::Storage::config();
8272     my $ids = $cfg->{ids};
8273
8274     my $res = [];
8275     foreach my $sid (keys %$ids) {
8276         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8277         next if !$ids->{$sid}->{content}->{images};
8278         push @$res, $sid;
8279     }
8280
8281     return $res;
8282 }
8283
8284 sub complete_migration_storage {
8285     my ($cmd, $param, $current_value, $all_args) = @_;
8286
8287     my $targetnode = @$all_args[1];
8288
8289     my $cfg = PVE::Storage::config();
8290     my $ids = $cfg->{ids};
8291
8292     my $res = [];
8293     foreach my $sid (keys %$ids) {
8294         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8295         next if !$ids->{$sid}->{content}->{images};
8296         push @$res, $sid;
8297     }
8298
8299     return $res;
8300 }
8301
8302 sub vm_is_paused {
8303     my ($vmid) = @_;
8304     my $qmpstatus = eval {
8305         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8306         mon_cmd($vmid, "query-status");
8307     };
8308     warn "$@\n" if $@;
8309     return $qmpstatus && $qmpstatus->{status} eq "paused";
8310 }
8311
8312 sub check_volume_storage_type {
8313     my ($storecfg, $vol) = @_;
8314
8315     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8316     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8317     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8318
8319     die "storage '$storeid' does not support content-type '$vtype'\n"
8320         if !$scfg->{content}->{$vtype};
8321
8322     return 1;
8323 }
8324
8325 sub add_nets_bridge_fdb {
8326     my ($conf, $vmid) = @_;
8327
8328     for my $opt (keys %$conf) {
8329         next if $opt !~ m/^net(\d+)$/;
8330         my $iface = "tap${vmid}i$1";
8331         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8332         my $net = parse_net($conf->{$opt}, 1) or next;
8333
8334         my $mac = $net->{macaddr};
8335         if (!$mac) {
8336             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8337                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8338             next;
8339         }
8340
8341         if ($have_sdn) {
8342             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
8343         } else {
8344             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8345         }
8346     }
8347 }
8348
8349 sub del_nets_bridge_fdb {
8350     my ($conf, $vmid) = @_;
8351
8352     for my $opt (keys %$conf) {
8353         next if $opt !~ m/^net(\d+)$/;
8354         my $iface = "tap${vmid}i$1";
8355
8356         my $net = parse_net($conf->{$opt}) or next;
8357         my $mac = $net->{macaddr} or next;
8358
8359         if ($have_sdn) {
8360             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
8361         } else {
8362             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8363         }
8364     }
8365 }
8366
8367 1;