PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file('/qemu-server/',
 106                   \&parse_vm_config,
 107                   \&write_vm_config);
 108
 109 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 110     description => "Some command save/restore state from this location.",
 111     type => 'string',
 112     maxLength => 128,
 113     optional => 1,
 114 });
 115
 116 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 117         description => "Specifies the Qemu machine type.",
 118         type => 'string',
 119         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 120         maxLength => 40,
 121         optional => 1,
 122 });
 123
 124 #no warnings 'redefine';
 125
 126 my $nodename_cache;
 127 sub nodename {
 128     $nodename_cache //= PVE::INotify::nodename();
 129     return $nodename_cache;
 130 }
 131
 132 my $watchdog_fmt = {
 133     model => {
 134         default_key => 1,
 135         type => 'string',
 136         enum => [qw(i6300esb ib700)],
 137         description => "Watchdog type to emulate.",
 138         default => 'i6300esb',
 139         optional => 1,
 140     },
 141     action => {
 142         type => 'string',
 143         enum => [qw(reset shutdown poweroff pause debug none)],
 144         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 145         optional => 1,
 146     },
 147 };
 148 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 149
 150 my $agent_fmt = {
 151     enabled => {
 152         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 153         type => 'boolean',
 154         default => 0,
 155         default_key => 1,
 156     },
 157     fstrim_cloned_disks => {
 158         description => "Run fstrim after moving a disk or migrating the VM.",
 159         type => 'boolean',
 160         optional => 1,
 161         default => 0
 162     },
 163     type => {
 164         description => "Select the agent type",
 165         type => 'string',
 166         default => 'virtio',
 167         optional => 1,
 168         enum => [qw(virtio isa)],
 169     },
 170 };
 171
 172 my $vga_fmt = {
 173     type => {
 174         description => "Select the VGA type.",
 175         type => 'string',
 176         default => 'std',
 177         optional => 1,
 178         default_key => 1,
 179         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 180     },
 181     memory => {
 182         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 183         type => 'integer',
 184         optional => 1,
 185         minimum => 4,
 186         maximum => 512,
 187     },
 188 };
 189
 190 my $ivshmem_fmt = {
 191     size => {
 192         type => 'integer',
 193         minimum => 1,
 194         description => "The size of the file in MB.",
 195     },
 196     name => {
 197         type => 'string',
 198         pattern => '[a-zA-Z0-9\-]+',
 199         optional => 1,
 200         format_description => 'string',
 201         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 202     },
 203 };
 204
 205 my $audio_fmt = {
 206     device => {
 207         type => 'string',
 208         enum => [qw(ich9-intel-hda intel-hda AC97)],
 209         description =>  "Configure an audio device."
 210     },
 211     driver =>  {
 212         type => 'string',
 213         enum => ['spice', 'none'],
 214         default => 'spice',
 215         optional => 1,
 216         description => "Driver backend for the audio device."
 217     },
 218 };
 219
 220 my $spice_enhancements_fmt = {
 221     foldersharing => {
 222         type => 'boolean',
 223         optional => 1,
 224         default => '0',
 225         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 226     },
 227     videostreaming =>  {
 228         type => 'string',
 229         enum => ['off', 'all', 'filter'],
 230         default => 'off',
 231         optional => 1,
 232         description => "Enable video streaming. Uses compression for detected video streams."
 233     },
 234 };
 235
 236 my $rng_fmt = {
 237     source => {
 238         type => 'string',
 239         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 240         default_key => 1,
 241         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 242             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 243             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 244             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 245             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 246             ." a hardware RNG from the host.",
 247     },
 248     max_bytes => {
 249         type => 'integer',
 250         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 251             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 252             ." `0` to disable limiting (potentially dangerous!).",
 253         optional => 1,
 254
 255         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 256         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 257         # reading from /dev/urandom
 258         default => 1024,
 259     },
 260     period => {
 261         type => 'integer',
 262         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 263             ." the guest to retrieve another 'max_bytes' of entropy.",
 264         optional => 1,
 265         default => 1000,
 266     },
 267 };
 268
 269 my $meta_info_fmt = {
 270     'ctime' => {
 271         type => 'integer',
 272         description => "The guest creation timestamp as UNIX epoch time",
 273         minimum => 0,
 274         optional => 1,
 275     },
 276     'creation-qemu' => {
 277         type => 'string',
 278         description => "The QEMU (machine) version from the time this VM was created.",
 279         pattern => '\d+(\.\d+)+',
 280         optional => 1,
 281     },
 282 };
 283
 284 my $confdesc = {
 285     onboot => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Specifies whether a VM will be started during system bootup.",
 289         default => 0,
 290     },
 291     autostart => {
 292         optional => 1,
 293         type => 'boolean',
 294         description => "Automatic restart after crash (currently ignored).",
 295         default => 0,
 296     },
 297     hotplug => {
 298         optional => 1,
 299         type => 'string', format => 'pve-hotplug-features',
 300         description => "Selectively enable hotplug features. This is a comma separated list of"
 301             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 302             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 303         default => 'network,disk,usb',
 304     },
 305     reboot => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 309         default => 1,
 310     },
 311     lock => {
 312         optional => 1,
 313         type => 'string',
 314         description => "Lock/unlock the VM.",
 315         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 316     },
 317     cpulimit => {
 318         optional => 1,
 319         type => 'number',
 320         description => "Limit of CPU usage.",
 321         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 322             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 323         minimum => 0,
 324         maximum => 128,
 325         default => 0,
 326     },
 327     cpuunits => {
 328         optional => 1,
 329         type => 'integer',
 330         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 331         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 332             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 333             ." weights of all the other running VMs.",
 334         minimum => 1,
 335         maximum => 262144,
 336         default => 'cgroup v1: 1024, cgroup v2: 100',
 337     },
 338     memory => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 342             ." you use the balloon device.",
 343         minimum => 16,
 344         default => 512,
 345     },
 346     balloon => {
 347         optional => 1,
 348         type => 'integer',
 349         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 350         minimum => 0,
 351     },
 352     shares => {
 353         optional => 1,
 354         type => 'integer',
 355         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 356             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 357             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 358         minimum => 0,
 359         maximum => 50000,
 360         default => 1000,
 361     },
 362     keyboard => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Keyboard layout for VNC server. This option is generally not required and"
 366         ." is often better handled from within the guest OS.",
 367         enum => PVE::Tools::kvmkeymaplist(),
 368         default => undef,
 369     },
 370     name => {
 371         optional => 1,
 372         type => 'string', format => 'dns-name',
 373         description => "Set a name for the VM. Only used on the configuration web interface.",
 374     },
 375     scsihw => {
 376         optional => 1,
 377         type => 'string',
 378         description => "SCSI controller model",
 379         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 380         default => 'lsi',
 381     },
 382     description => {
 383         optional => 1,
 384         type => 'string',
 385         description => "Description for the VM. Shown in the web-interface VM's summary."
 386             ." This is saved as comment inside the configuration file.",
 387         maxLength => 1024 * 8,
 388     },
 389     ostype => {
 390         optional => 1,
 391         type => 'string',
 392         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 393         description => "Specify guest operating system.",
 394         verbose_description => <<EODESC,
 395 Specify guest operating system. This is used to enable special
 396 optimization/features for specific operating systems:
 397
 398 [horizontal]
 399 other;; unspecified OS
 400 wxp;; Microsoft Windows XP
 401 w2k;; Microsoft Windows 2000
 402 w2k3;; Microsoft Windows 2003
 403 w2k8;; Microsoft Windows 2008
 404 wvista;; Microsoft Windows Vista
 405 win7;; Microsoft Windows 7
 406 win8;; Microsoft Windows 8/2012/2012r2
 407 win10;; Microsoft Windows 10/2016/2019
 408 win11;; Microsoft Windows 11/2022
 409 l24;; Linux 2.4 Kernel
 410 l26;; Linux 2.6 - 5.X Kernel
 411 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 412 EODESC
 413     },
 414     boot => {
 415         optional => 1,
 416         type => 'string', format => 'pve-qm-boot',
 417         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 418             ." key or 'legacy=' is deprecated.",
 419     },
 420     bootdisk => {
 421         optional => 1,
 422         type => 'string', format => 'pve-qm-bootdisk',
 423         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 424         pattern => '(ide|sata|scsi|virtio)\d+',
 425     },
 426     smp => {
 427         optional => 1,
 428         type => 'integer',
 429         description => "The number of CPUs. Please use option -sockets instead.",
 430         minimum => 1,
 431         default => 1,
 432     },
 433     sockets => {
 434         optional => 1,
 435         type => 'integer',
 436         description => "The number of CPU sockets.",
 437         minimum => 1,
 438         default => 1,
 439     },
 440     cores => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "The number of cores per socket.",
 444         minimum => 1,
 445         default => 1,
 446     },
 447     numa => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Enable/disable NUMA.",
 451         default => 0,
 452     },
 453     hugepages => {
 454         optional => 1,
 455         type => 'string',
 456         description => "Enable/disable hugepages memory.",
 457         enum => [qw(any 2 1024)],
 458     },
 459     keephugepages => {
 460         optional => 1,
 461         type => 'boolean',
 462         default => 0,
 463         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 464             ." after VM shutdown and can be used for subsequent starts.",
 465     },
 466     vcpus => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "Number of hotplugged vcpus.",
 470         minimum => 1,
 471         default => 0,
 472     },
 473     acpi => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable ACPI.",
 477         default => 1,
 478     },
 479     agent => {
 480         optional => 1,
 481         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 482         type => 'string',
 483         format => $agent_fmt,
 484     },
 485     kvm => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Enable/disable KVM hardware virtualization.",
 489         default => 1,
 490     },
 491     tdf => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable time drift fix.",
 495         default => 0,
 496     },
 497     localtime => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 501             ." the `ostype` indicates a Microsoft Windows OS.",
 502     },
 503     freeze => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 507     },
 508     vga => {
 509         optional => 1,
 510         type => 'string', format => $vga_fmt,
 511         description => "Configure the VGA hardware.",
 512         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 513             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 514             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 515             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 516             ." display server. For win* OS you can select how many independent displays you want,"
 517             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 518             ." using a serial device as terminal.",
 519     },
 520     watchdog => {
 521         optional => 1,
 522         type => 'string', format => 'pve-qm-watchdog',
 523         description => "Create a virtual hardware watchdog device.",
 524         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 525             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 526             ." else the watchdog will reset the guest (or execute the respective action specified)",
 527     },
 528     startdate => {
 529         optional => 1,
 530         type => 'string',
 531         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 532         description => "Set the initial date of the real time clock. Valid format for date are:"
 533             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 534         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 535         default => 'now',
 536     },
 537     startup =>  get_standard_option('pve-startup-order'),
 538     template => {
 539         optional => 1,
 540         type => 'boolean',
 541         description => "Enable/disable Template.",
 542         default => 0,
 543     },
 544     args => {
 545         optional => 1,
 546         type => 'string',
 547         description => "Arbitrary arguments passed to kvm.",
 548         verbose_description => <<EODESCR,
 549 Arbitrary arguments passed to kvm, for example:
 550
 551 args: -no-reboot -no-hpet
 552
 553 NOTE: this option is for experts only.
 554 EODESCR
 555     },
 556     tablet => {
 557         optional => 1,
 558         type => 'boolean',
 559         default => 1,
 560         description => "Enable/disable the USB tablet device.",
 561         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 562             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 563             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 564             ." may consider disabling this to save some context switches. This is turned off by"
 565             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 566     },
 567     migrate_speed => {
 568         optional => 1,
 569         type => 'integer',
 570         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 571         minimum => 0,
 572         default => 0,
 573     },
 574     migrate_downtime => {
 575         optional => 1,
 576         type => 'number',
 577         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 578         minimum => 0,
 579         default => 0.1,
 580     },
 581     cdrom => {
 582         optional => 1,
 583         type => 'string', format => 'pve-qm-ide',
 584         typetext => '<volume>',
 585         description => "This is an alias for option -ide2",
 586     },
 587     cpu => {
 588         optional => 1,
 589         description => "Emulated CPU type.",
 590         type => 'string',
 591         format => 'pve-vm-cpu-conf',
 592     },
 593     parent => get_standard_option('pve-snapshot-name', {
 594         optional => 1,
 595         description => "Parent snapshot name. This is used internally, and should not be modified.",
 596     }),
 597     snaptime => {
 598         optional => 1,
 599         description => "Timestamp for snapshots.",
 600         type => 'integer',
 601         minimum => 0,
 602     },
 603     vmstate => {
 604         optional => 1,
 605         type => 'string', format => 'pve-volume-id',
 606         description => "Reference to a volume which stores the VM state. This is used internally"
 607             ." for snapshots.",
 608     },
 609     vmstatestorage => get_standard_option('pve-storage-id', {
 610         description => "Default storage for VM state volumes/files.",
 611         optional => 1,
 612     }),
 613     runningmachine => get_standard_option('pve-qemu-machine', {
 614         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 615             ." for snapshots.",
 616     }),
 617     runningcpu => {
 618         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 619             ." internally for snapshots.",
 620         optional => 1,
 621         type => 'string',
 622         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 623         format_description => 'QEMU -cpu parameter'
 624     },
 625     machine => get_standard_option('pve-qemu-machine'),
 626     arch => {
 627         description => "Virtual processor architecture. Defaults to the host.",
 628         optional => 1,
 629         type => 'string',
 630         enum => [qw(x86_64 aarch64)],
 631     },
 632     smbios1 => {
 633         description => "Specify SMBIOS type 1 fields.",
 634         type => 'string', format => 'pve-qm-smbios1',
 635         maxLength => 512,
 636         optional => 1,
 637     },
 638     protection => {
 639         optional => 1,
 640         type => 'boolean',
 641         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 642             ." remove disk operations.",
 643         default => 0,
 644     },
 645     bios => {
 646         optional => 1,
 647         type => 'string',
 648         enum => [ qw(seabios ovmf) ],
 649         description => "Select BIOS implementation.",
 650         default => 'seabios',
 651     },
 652     vmgenid => {
 653         type => 'string',
 654         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 655         format_description => 'UUID',
 656         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 657             ." to disable explicitly.",
 658         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 659             ." value identifier to the guest OS. This allows to notify the guest operating system"
 660             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 661             ." execution or creation from a template). The guest operating system notices the"
 662             ." change, and is then able to react as appropriate by marking its copies of"
 663             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 664             ."Note that auto-creation only works when done through API/CLI create or update methods"
 665             .", but not when manually editing the config file.",
 666         default => "1 (autogenerated)",
 667         optional => 1,
 668     },
 669     hookscript => {
 670         type => 'string',
 671         format => 'pve-volume-id',
 672         optional => 1,
 673         description => "Script that will be executed during various steps in the vms lifetime.",
 674     },
 675     ivshmem => {
 676         type => 'string',
 677         format => $ivshmem_fmt,
 678         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 679             ." the host.",
 680         optional => 1,
 681     },
 682     audio0 => {
 683         type => 'string',
 684         format => $audio_fmt,
 685         description => "Configure a audio device, useful in combination with QXL/Spice.",
 686         optional => 1
 687     },
 688     spice_enhancements => {
 689         type => 'string',
 690         format => $spice_enhancements_fmt,
 691         description => "Configure additional enhancements for SPICE.",
 692         optional => 1
 693     },
 694     tags => {
 695         type => 'string', format => 'pve-tag-list',
 696         description => 'Tags of the VM. This is only meta information.',
 697         optional => 1,
 698     },
 699     rng0 => {
 700         type => 'string',
 701         format => $rng_fmt,
 702         description => "Configure a VirtIO-based Random Number Generator.",
 703         optional => 1,
 704     },
 705     meta => {
 706         type => 'string',
 707         format => $meta_info_fmt,
 708         description => "Some (read-only) meta-information about this guest.",
 709         optional => 1,
 710     },
 711     affinity => {
 712         type => 'string', format => 'pve-cpuset',
 713         description => "List of host cores used to execute guest processes.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.',
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.',
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 14;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1036 sub pve_verify_cpuset {
1037     my ($set_text, $noerr) = @_;
1038
1039     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1040
1041     if ($@) {
1042         return if $noerr;
1043         die "unable to parse cpuset option\n";
1044     }
1045
1046     return PVE::CpuSet->new($members)->short_string();
1047 }
1048
1049 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1050 sub verify_volume_id_or_qm_path {
1051     my ($volid, $noerr) = @_;
1052
1053     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1054
1055     return verify_volume_id_or_absolute_path($volid, $noerr);
1056 }
1057
1058 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1059 sub verify_volume_id_or_absolute_path {
1060     my ($volid, $noerr) = @_;
1061
1062     return $volid if $volid =~ m|^/|;
1063
1064     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1065     if ($@) {
1066         return if $noerr;
1067         die $@;
1068     }
1069     return $volid;
1070 }
1071
1072 my $usb_fmt = {
1073     host => {
1074         default_key => 1,
1075         type => 'string', format => 'pve-qm-usb-device',
1076         format_description => 'HOSTUSBDEVICE|spice',
1077         description => <<EODESCR,
1078 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1079
1080  'bus-port(.port)*' (decimal numbers) or
1081  'vendor_id:product_id' (hexadeciaml numbers) or
1082  'spice'
1083
1084 You can use the 'lsusb -t' command to list existing usb devices.
1085
1086 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1087 machines - use with special care.
1088
1089 The value 'spice' can be used to add a usb redirection devices for spice.
1090 EODESCR
1091     },
1092     usb3 => {
1093         optional => 1,
1094         type => 'boolean',
1095         description => "Specifies whether if given host option is a USB3 device or port."
1096             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1097             ." is irrelevant (all devices are plugged into a xhci controller).",
1098         default => 0,
1099     },
1100 };
1101
1102 my $usbdesc = {
1103     optional => 1,
1104     type => 'string', format => $usb_fmt,
1105     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1106         ." l26 or windows > 7, n can be up to 14).",
1107 };
1108 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1109
1110 my $serialdesc = {
1111         optional => 1,
1112         type => 'string',
1113         pattern => '(/dev/.+|socket)',
1114         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1115         verbose_description =>  <<EODESCR,
1116 Create a serial device inside the VM (n is 0 to 3), and pass through a
1117 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1118 host side (use 'qm terminal' to open a terminal connection).
1119
1120 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1121 use with special care.
1122
1123 CAUTION: Experimental! User reported problems with this option.
1124 EODESCR
1125 };
1126
1127 my $paralleldesc= {
1128         optional => 1,
1129         type => 'string',
1130         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1131         description =>  "Map host parallel devices (n is 0 to 2).",
1132         verbose_description =>  <<EODESCR,
1133 Map host parallel devices (n is 0 to 2).
1134
1135 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1136 machines - use with special care.
1137
1138 CAUTION: Experimental! User reported problems with this option.
1139 EODESCR
1140 };
1141
1142 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1143     $confdesc->{"parallel$i"} = $paralleldesc;
1144 }
1145
1146 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1147     $confdesc->{"serial$i"} = $serialdesc;
1148 }
1149
1150 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1151     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1152 }
1153
1154 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1155     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1156 }
1157
1158 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1159     $confdesc->{"usb$i"} = $usbdesc;
1160 }
1161
1162 my $boot_fmt = {
1163     legacy => {
1164         optional => 1,
1165         default_key => 1,
1166         type => 'string',
1167         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1168                      . " Deprecated, use 'order=' instead.",
1169         pattern => '[acdn]{1,4}',
1170         format_description => "[acdn]{1,4}",
1171
1172         # note: this is also the fallback if boot: is not given at all
1173         default => 'cdn',
1174     },
1175     order => {
1176         optional => 1,
1177         type => 'string',
1178         format => 'pve-qm-bootdev-list',
1179         format_description => "device[;device...]",
1180         description => <<EODESC,
1181 The guest will attempt to boot from devices in the order they appear here.
1182
1183 Disks, optical drives and passed-through storage USB devices will be directly
1184 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1185 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1186
1187 Note that only devices in this list will be marked as bootable and thus loaded
1188 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1189 (e.g. software-raid), you need to specify all of them here.
1190
1191 Overrides the deprecated 'legacy=[acdn]*' value when given.
1192 EODESC
1193     },
1194 };
1195 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1196
1197 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1198 sub verify_bootdev {
1199     my ($dev, $noerr) = @_;
1200
1201     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1202     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1203
1204     my $check = sub {
1205         my ($base) = @_;
1206         return 0 if $dev !~ m/^$base\d+$/;
1207         return 0 if !$confdesc->{$dev};
1208         return 1;
1209     };
1210
1211     return $dev if $check->("net");
1212     return $dev if $check->("usb");
1213     return $dev if $check->("hostpci");
1214
1215     return if $noerr;
1216     die "invalid boot device '$dev'\n";
1217 }
1218
1219 sub print_bootorder {
1220     my ($devs) = @_;
1221     return "" if !@$devs;
1222     my $data = { order => join(';', @$devs) };
1223     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1224 }
1225
1226 my $kvm_api_version = 0;
1227
1228 sub kvm_version {
1229     return $kvm_api_version if $kvm_api_version;
1230
1231     open my $fh, '<', '/dev/kvm' or return;
1232
1233     # 0xae00 => KVM_GET_API_VERSION
1234     $kvm_api_version = ioctl($fh, 0xae00, 0);
1235     close($fh);
1236
1237     return $kvm_api_version;
1238 }
1239
1240 my $kvm_user_version = {};
1241 my $kvm_mtime = {};
1242
1243 sub kvm_user_version {
1244     my ($binary) = @_;
1245
1246     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1247     my $st = stat($binary);
1248
1249     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1250     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1251         $cachedmtime == $st->mtime;
1252
1253     $kvm_user_version->{$binary} = 'unknown';
1254     $kvm_mtime->{$binary} = $st->mtime;
1255
1256     my $code = sub {
1257         my $line = shift;
1258         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1259             $kvm_user_version->{$binary} = $2;
1260         }
1261     };
1262
1263     eval { run_command([$binary, '--version'], outfunc => $code); };
1264     warn $@ if $@;
1265
1266     return $kvm_user_version->{$binary};
1267
1268 }
1269 my sub extract_version {
1270     my ($machine_type, $version) = @_;
1271     $version = kvm_user_version() if !defined($version);
1272     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1273 }
1274
1275 sub kernel_has_vhost_net {
1276     return -c '/dev/vhost-net';
1277 }
1278
1279 sub option_exists {
1280     my $key = shift;
1281     return defined($confdesc->{$key});
1282 }
1283
1284 my $cdrom_path;
1285 sub get_cdrom_path {
1286
1287     return  $cdrom_path if $cdrom_path;
1288
1289     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1290     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1291     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1292 }
1293
1294 sub get_iso_path {
1295     my ($storecfg, $vmid, $cdrom) = @_;
1296
1297     if ($cdrom eq 'cdrom') {
1298         return get_cdrom_path();
1299     } elsif ($cdrom eq 'none') {
1300         return '';
1301     } elsif ($cdrom =~ m|^/|) {
1302         return $cdrom;
1303     } else {
1304         return PVE::Storage::path($storecfg, $cdrom);
1305     }
1306 }
1307
1308 # try to convert old style file names to volume IDs
1309 sub filename_to_volume_id {
1310     my ($vmid, $file, $media) = @_;
1311
1312      if (!($file eq 'none' || $file eq 'cdrom' ||
1313           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1314
1315         return if $file =~ m|/|;
1316
1317         if ($media && $media eq 'cdrom') {
1318             $file = "local:iso/$file";
1319         } else {
1320             $file = "local:$vmid/$file";
1321         }
1322     }
1323
1324     return $file;
1325 }
1326
1327 sub verify_media_type {
1328     my ($opt, $vtype, $media) = @_;
1329
1330     return if !$media;
1331
1332     my $etype;
1333     if ($media eq 'disk') {
1334         $etype = 'images';
1335     } elsif ($media eq 'cdrom') {
1336         $etype = 'iso';
1337     } else {
1338         die "internal error";
1339     }
1340
1341     return if ($vtype eq $etype);
1342
1343     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1344 }
1345
1346 sub cleanup_drive_path {
1347     my ($opt, $storecfg, $drive) = @_;
1348
1349     # try to convert filesystem paths to volume IDs
1350
1351     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1352         ($drive->{file} !~ m|^/dev/.+|) &&
1353         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1354         ($drive->{file} !~ m/^\d+$/)) {
1355         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1356         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1357             if !$vtype;
1358         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1359         verify_media_type($opt, $vtype, $drive->{media});
1360         $drive->{file} = $volid;
1361     }
1362
1363     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1364 }
1365
1366 sub parse_hotplug_features {
1367     my ($data) = @_;
1368
1369     my $res = {};
1370
1371     return $res if $data eq '0';
1372
1373     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1374
1375     foreach my $feature (PVE::Tools::split_list($data)) {
1376         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1377             $res->{$1} = 1;
1378         } else {
1379             die "invalid hotplug feature '$feature'\n";
1380         }
1381     }
1382     return $res;
1383 }
1384
1385 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1386 sub pve_verify_hotplug_features {
1387     my ($value, $noerr) = @_;
1388
1389     return $value if parse_hotplug_features($value);
1390
1391     return if $noerr;
1392
1393     die "unable to parse hotplug option\n";
1394 }
1395
1396 sub scsi_inquiry {
1397     my($fh, $noerr) = @_;
1398
1399     my $SG_IO = 0x2285;
1400     my $SG_GET_VERSION_NUM = 0x2282;
1401
1402     my $versionbuf = "\x00" x 8;
1403     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1404     if (!$ret) {
1405         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1406         return;
1407     }
1408     my $version = unpack("I", $versionbuf);
1409     if ($version < 30000) {
1410         die "scsi generic interface too old\n"  if !$noerr;
1411         return;
1412     }
1413
1414     my $buf = "\x00" x 36;
1415     my $sensebuf = "\x00" x 8;
1416     my $cmd = pack("C x3 C x1", 0x12, 36);
1417
1418     # see /usr/include/scsi/sg.h
1419     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1420
1421     my $packet = pack(
1422         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1423     );
1424
1425     $ret = ioctl($fh, $SG_IO, $packet);
1426     if (!$ret) {
1427         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1428         return;
1429     }
1430
1431     my @res = unpack($sg_io_hdr_t, $packet);
1432     if ($res[17] || $res[18]) {
1433         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1434         return;
1435     }
1436
1437     my $res = {};
1438     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1439
1440     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1441     $res->{type} &= 0x1F;
1442
1443     return $res;
1444 }
1445
1446 sub path_is_scsi {
1447     my ($path) = @_;
1448
1449     my $fh = IO::File->new("+<$path") || return;
1450     my $res = scsi_inquiry($fh, 1);
1451     close($fh);
1452
1453     return $res;
1454 }
1455
1456 sub print_tabletdevice_full {
1457     my ($conf, $arch) = @_;
1458
1459     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1460
1461     # we use uhci for old VMs because tablet driver was buggy in older qemu
1462     my $usbbus;
1463     if ($q35 || $arch eq 'aarch64') {
1464         $usbbus = 'ehci';
1465     } else {
1466         $usbbus = 'uhci';
1467     }
1468
1469     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1470 }
1471
1472 sub print_keyboarddevice_full {
1473     my ($conf, $arch) = @_;
1474
1475     return if $arch ne 'aarch64';
1476
1477     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1478 }
1479
1480 my sub get_drive_id {
1481     my ($drive) = @_;
1482     return "$drive->{interface}$drive->{index}";
1483 }
1484
1485 sub print_drivedevice_full {
1486     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1487
1488     my $device = '';
1489     my $maxdev = 0;
1490
1491     my $drive_id = get_drive_id($drive);
1492     if ($drive->{interface} eq 'virtio') {
1493         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1494         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1495         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1496     } elsif ($drive->{interface} eq 'scsi') {
1497
1498         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1499         my $unit = $drive->{index} % $maxdev;
1500         my $devicetype = 'hd';
1501         my $path = '';
1502         if (drive_is_cdrom($drive)) {
1503             $devicetype = 'cd';
1504         } else {
1505             if ($drive->{file} =~ m|^/|) {
1506                 $path = $drive->{file};
1507                 if (my $info = path_is_scsi($path)) {
1508                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1509                         $devicetype = 'block';
1510                     } elsif ($info->{type} == 1) { # tape
1511                         $devicetype = 'generic';
1512                     }
1513                 }
1514             } else {
1515                  $path = PVE::Storage::path($storecfg, $drive->{file});
1516             }
1517
1518             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1519             my $version = extract_version($machine_type, kvm_user_version());
1520             if ($path =~ m/^iscsi\:\/\// &&
1521                !min_version($version, 4, 1)) {
1522                 $devicetype = 'generic';
1523             }
1524         }
1525
1526         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1527             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1528         } else {
1529             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1530                 .",lun=$drive->{index}";
1531         }
1532         $device .= ",drive=drive-$drive_id,id=$drive_id";
1533
1534         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1535             $device .= ",rotation_rate=1";
1536         }
1537         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1538
1539     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1540         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1541         my $controller = int($drive->{index} / $maxdev);
1542         my $unit = $drive->{index} % $maxdev;
1543         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1544
1545         $device = "ide-$devicetype";
1546         if ($drive->{interface} eq 'ide') {
1547             $device .= ",bus=ide.$controller,unit=$unit";
1548         } else {
1549             $device .= ",bus=ahci$controller.$unit";
1550         }
1551         $device .= ",drive=drive-$drive_id,id=$drive_id";
1552
1553         if ($devicetype eq 'hd') {
1554             if (my $model = $drive->{model}) {
1555                 $model = URI::Escape::uri_unescape($model);
1556                 $device .= ",model=$model";
1557             }
1558             if ($drive->{ssd}) {
1559                 $device .= ",rotation_rate=1";
1560             }
1561         }
1562         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1563     } elsif ($drive->{interface} eq 'usb') {
1564         die "implement me";
1565         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1566     } else {
1567         die "unsupported interface type";
1568     }
1569
1570     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1571
1572     if (my $serial = $drive->{serial}) {
1573         $serial = URI::Escape::uri_unescape($serial);
1574         $device .= ",serial=$serial";
1575     }
1576
1577
1578     return $device;
1579 }
1580
1581 sub get_initiator_name {
1582     my $initiator;
1583
1584     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1585     while (defined(my $line = <$fh>)) {
1586         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1587         $initiator = $1;
1588         last;
1589     }
1590     $fh->close();
1591
1592     return $initiator;
1593 }
1594
1595 sub print_drive_commandline_full {
1596     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1597
1598     my $path;
1599     my $volid = $drive->{file};
1600     my $format = $drive->{format};
1601     my $drive_id = get_drive_id($drive);
1602
1603     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1604     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1605
1606     if (drive_is_cdrom($drive)) {
1607         $path = get_iso_path($storecfg, $vmid, $volid);
1608         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1609     } else {
1610         if ($storeid) {
1611             $path = PVE::Storage::path($storecfg, $volid);
1612             $format //= qemu_img_format($scfg, $volname);
1613         } else {
1614             $path = $volid;
1615             $format //= "raw";
1616         }
1617    }
1618
1619    my $is_rbd = $path =~ m/^rbd:/;
1620
1621     my $opts = '';
1622     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1623     foreach my $o (@qemu_drive_options) {
1624         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1625     }
1626
1627     # snapshot only accepts on|off
1628     if (defined($drive->{snapshot})) {
1629         my $v = $drive->{snapshot} ? 'on' : 'off';
1630         $opts .= ",snapshot=$v";
1631     }
1632
1633     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1634         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1635     }
1636
1637     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1638         my ($dir, $qmpname) = @$type;
1639         if (my $v = $drive->{"mbps$dir"}) {
1640             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1641         }
1642         if (my $v = $drive->{"mbps${dir}_max"}) {
1643             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1644         }
1645         if (my $v = $drive->{"bps${dir}_max_length"}) {
1646             $opts .= ",throttling.bps$qmpname-max-length=$v";
1647         }
1648         if (my $v = $drive->{"iops${dir}"}) {
1649             $opts .= ",throttling.iops$qmpname=$v";
1650         }
1651         if (my $v = $drive->{"iops${dir}_max"}) {
1652             $opts .= ",throttling.iops$qmpname-max=$v";
1653         }
1654         if (my $v = $drive->{"iops${dir}_max_length"}) {
1655             $opts .= ",throttling.iops$qmpname-max-length=$v";
1656         }
1657     }
1658
1659     if ($pbs_name) {
1660         $format = "rbd" if $is_rbd;
1661         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1662             if !$format;
1663         $opts .= ",format=alloc-track,file.driver=$format";
1664     } elsif ($format) {
1665         $opts .= ",format=$format";
1666     }
1667
1668     my $cache_direct = 0;
1669
1670     if (my $cache = $drive->{cache}) {
1671         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1672     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1673         $opts .= ",cache=none";
1674         $cache_direct = 1;
1675     }
1676
1677     # io_uring with cache mode writeback or writethrough on krbd will hang...
1678     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1679
1680     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1681     # sometimes, just plain disable...
1682     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1683
1684     # io_uring causes problems when used with CIFS since kernel 5.15
1685     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1686     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1687
1688     if (!$drive->{aio}) {
1689         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1690             # io_uring supports all cache modes
1691             $opts .= ",aio=io_uring";
1692         } else {
1693             # aio native works only with O_DIRECT
1694             if($cache_direct) {
1695                 $opts .= ",aio=native";
1696             } else {
1697                 $opts .= ",aio=threads";
1698             }
1699         }
1700     }
1701
1702     if (!drive_is_cdrom($drive)) {
1703         my $detectzeroes;
1704         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1705             $detectzeroes = 'off';
1706         } elsif ($drive->{discard}) {
1707             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1708         } else {
1709             # This used to be our default with discard not being specified:
1710             $detectzeroes = 'on';
1711         }
1712
1713         # note: 'detect-zeroes' works per blockdev and we want it to persist
1714         # after the alloc-track is removed, so put it on 'file' directly
1715         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1716         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1717     }
1718
1719     if ($pbs_name) {
1720         $opts .= ",backing=$pbs_name";
1721         $opts .= ",auto-remove=on";
1722     }
1723
1724     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1725     my $file_param = "file";
1726     if ($pbs_name) {
1727         # non-rbd drivers require the underlying file to be a seperate block
1728         # node, so add a second .file indirection
1729         $file_param .= ".file" if !$is_rbd;
1730         $file_param .= ".filename";
1731     }
1732     my $pathinfo = $path ? "$file_param=$path," : '';
1733
1734     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1735 }
1736
1737 sub print_pbs_blockdev {
1738     my ($pbs_conf, $pbs_name) = @_;
1739     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1740     $blockdev .= ",repository=$pbs_conf->{repository}";
1741     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1742     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1743     $blockdev .= ",archive=$pbs_conf->{archive}";
1744     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1745     return $blockdev;
1746 }
1747
1748 sub print_netdevice_full {
1749     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1750
1751     my $device = $net->{model};
1752     if ($net->{model} eq 'virtio') {
1753          $device = 'virtio-net-pci';
1754      };
1755
1756     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1757     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1758     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1759         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1760         # and out of each queue plus one config interrupt and control vector queue
1761         my $vectors = $net->{queues} * 2 + 2;
1762         $tmpstr .= ",vectors=$vectors,mq=on";
1763     }
1764     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1765
1766     if (my $mtu = $net->{mtu}) {
1767         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1768             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1769             if ($mtu == 1) {
1770                  $mtu = $bridge_mtu;
1771             } elsif ($mtu < 576) {
1772                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1773             } elsif ($mtu > $bridge_mtu) {
1774                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1775             }
1776             $tmpstr .= ",host_mtu=$mtu";
1777         } else {
1778             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1779         }
1780     }
1781
1782     if ($use_old_bios_files) {
1783         my $romfile;
1784         if ($device eq 'virtio-net-pci') {
1785             $romfile = 'pxe-virtio.rom';
1786         } elsif ($device eq 'e1000') {
1787             $romfile = 'pxe-e1000.rom';
1788         } elsif ($device eq 'e1000e') {
1789             $romfile = 'pxe-e1000e.rom';
1790         } elsif ($device eq 'ne2k') {
1791             $romfile = 'pxe-ne2k_pci.rom';
1792         } elsif ($device eq 'pcnet') {
1793             $romfile = 'pxe-pcnet.rom';
1794         } elsif ($device eq 'rtl8139') {
1795             $romfile = 'pxe-rtl8139.rom';
1796         }
1797         $tmpstr .= ",romfile=$romfile" if $romfile;
1798     }
1799
1800     return $tmpstr;
1801 }
1802
1803 sub print_netdev_full {
1804     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1805
1806     my $i = '';
1807     if ($netid =~ m/^net(\d+)$/) {
1808         $i = int($1);
1809     }
1810
1811     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1812
1813     my $ifname = "tap${vmid}i$i";
1814
1815     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1816     die "interface name '$ifname' is too long (max 15 character)\n"
1817         if length($ifname) >= 16;
1818
1819     my $vhostparam = '';
1820     if (is_native($arch)) {
1821         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1822     }
1823
1824     my $vmname = $conf->{name} || "vm$vmid";
1825
1826     my $netdev = "";
1827     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1828
1829     if ($net->{bridge}) {
1830         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1831             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1832     } else {
1833         $netdev = "type=user,id=$netid,hostname=$vmname";
1834     }
1835
1836     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1837
1838     return $netdev;
1839 }
1840
1841 my $vga_map = {
1842     'cirrus' => 'cirrus-vga',
1843     'std' => 'VGA',
1844     'vmware' => 'vmware-svga',
1845     'virtio' => 'virtio-vga',
1846     'virtio-gl' => 'virtio-vga-gl',
1847 };
1848
1849 sub print_vga_device {
1850     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1851
1852     my $type = $vga_map->{$vga->{type}};
1853     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1854         $type = 'virtio-gpu';
1855     }
1856     my $vgamem_mb = $vga->{memory};
1857
1858     my $max_outputs = '';
1859     if ($qxlnum) {
1860         $type = $id ? 'qxl' : 'qxl-vga';
1861
1862         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1863             # set max outputs so linux can have up to 4 qxl displays with one device
1864             if (min_version($machine_version, 4, 1)) {
1865                 $max_outputs = ",max_outputs=4";
1866             }
1867         }
1868     }
1869
1870     die "no devicetype for $vga->{type}\n" if !$type;
1871
1872     my $memory = "";
1873     if ($vgamem_mb) {
1874         if ($vga->{type} =~ /^virtio/) {
1875             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1876             $memory = ",max_hostmem=$bytes";
1877         } elsif ($qxlnum) {
1878             # from https://www.spice-space.org/multiple-monitors.html
1879             $memory = ",vgamem_mb=$vga->{memory}";
1880             my $ram = $vgamem_mb * 4;
1881             my $vram = $vgamem_mb * 2;
1882             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1883         } else {
1884             $memory = ",vgamem_mb=$vga->{memory}";
1885         }
1886     } elsif ($qxlnum && $id) {
1887         $memory = ",ram_size=67108864,vram_size=33554432";
1888     }
1889
1890     my $edidoff = "";
1891     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1892         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1893     }
1894
1895     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1896     my $vgaid = "vga" . ($id // '');
1897     my $pciaddr;
1898     if ($q35 && $vgaid eq 'vga') {
1899         # the first display uses pcie.0 bus on q35 machines
1900         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1901     } else {
1902         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1903     }
1904
1905     if ($vga->{type} eq 'virtio-gl') {
1906         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1907         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1908             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1909
1910         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1911             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1912     }
1913
1914     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1915 }
1916
1917 sub parse_number_sets {
1918     my ($set) = @_;
1919     my $res = [];
1920     foreach my $part (split(/;/, $set)) {
1921         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1922             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1923             push @$res, [ $1, $2 ];
1924         } else {
1925             die "invalid range: $part\n";
1926         }
1927     }
1928     return $res;
1929 }
1930
1931 sub parse_numa {
1932     my ($data) = @_;
1933
1934     my $res = parse_property_string($numa_fmt, $data);
1935     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1936     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1937     return $res;
1938 }
1939
1940 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1941 sub parse_net {
1942     my ($data) = @_;
1943
1944     my $res = eval { parse_property_string($net_fmt, $data) };
1945     if ($@) {
1946         warn $@;
1947         return;
1948     }
1949     if (!defined($res->{macaddr})) {
1950         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1951         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1952     }
1953     return $res;
1954 }
1955
1956 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1957 sub parse_ipconfig {
1958     my ($data) = @_;
1959
1960     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1961     if ($@) {
1962         warn $@;
1963         return;
1964     }
1965
1966     if ($res->{gw} && !$res->{ip}) {
1967         warn 'gateway specified without specifying an IP address';
1968         return;
1969     }
1970     if ($res->{gw6} && !$res->{ip6}) {
1971         warn 'IPv6 gateway specified without specifying an IPv6 address';
1972         return;
1973     }
1974     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1975         warn 'gateway specified together with DHCP';
1976         return;
1977     }
1978     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1979         # gw6 + auto/dhcp
1980         warn "IPv6 gateway specified together with $res->{ip6} address";
1981         return;
1982     }
1983
1984     if (!$res->{ip} && !$res->{ip6}) {
1985         return { ip => 'dhcp', ip6 => 'dhcp' };
1986     }
1987
1988     return $res;
1989 }
1990
1991 sub print_net {
1992     my $net = shift;
1993
1994     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1995 }
1996
1997 sub add_random_macs {
1998     my ($settings) = @_;
1999
2000     foreach my $opt (keys %$settings) {
2001         next if $opt !~ m/^net(\d+)$/;
2002         my $net = parse_net($settings->{$opt});
2003         next if !$net;
2004         $settings->{$opt} = print_net($net);
2005     }
2006 }
2007
2008 sub vm_is_volid_owner {
2009     my ($storecfg, $vmid, $volid) = @_;
2010
2011     if ($volid !~  m|^/|) {
2012         my ($path, $owner);
2013         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2014         if ($owner && ($owner == $vmid)) {
2015             return 1;
2016         }
2017     }
2018
2019     return;
2020 }
2021
2022 sub vmconfig_register_unused_drive {
2023     my ($storecfg, $vmid, $conf, $drive) = @_;
2024
2025     if (drive_is_cloudinit($drive)) {
2026         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2027         warn $@ if $@;
2028         delete $conf->{cloudinit};
2029     } elsif (!drive_is_cdrom($drive)) {
2030         my $volid = $drive->{file};
2031         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2032             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2033         }
2034     }
2035 }
2036
2037 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2038 my $smbios1_fmt = {
2039     uuid => {
2040         type => 'string',
2041         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2042         format_description => 'UUID',
2043         description => "Set SMBIOS1 UUID.",
2044         optional => 1,
2045     },
2046     version => {
2047         type => 'string',
2048         pattern => '[A-Za-z0-9+\/]+={0,2}',
2049         format_description => 'Base64 encoded string',
2050         description => "Set SMBIOS1 version.",
2051         optional => 1,
2052     },
2053     serial => {
2054         type => 'string',
2055         pattern => '[A-Za-z0-9+\/]+={0,2}',
2056         format_description => 'Base64 encoded string',
2057         description => "Set SMBIOS1 serial number.",
2058         optional => 1,
2059     },
2060     manufacturer => {
2061         type => 'string',
2062         pattern => '[A-Za-z0-9+\/]+={0,2}',
2063         format_description => 'Base64 encoded string',
2064         description => "Set SMBIOS1 manufacturer.",
2065         optional => 1,
2066     },
2067     product => {
2068         type => 'string',
2069         pattern => '[A-Za-z0-9+\/]+={0,2}',
2070         format_description => 'Base64 encoded string',
2071         description => "Set SMBIOS1 product ID.",
2072         optional => 1,
2073     },
2074     sku => {
2075         type => 'string',
2076         pattern => '[A-Za-z0-9+\/]+={0,2}',
2077         format_description => 'Base64 encoded string',
2078         description => "Set SMBIOS1 SKU string.",
2079         optional => 1,
2080     },
2081     family => {
2082         type => 'string',
2083         pattern => '[A-Za-z0-9+\/]+={0,2}',
2084         format_description => 'Base64 encoded string',
2085         description => "Set SMBIOS1 family string.",
2086         optional => 1,
2087     },
2088     base64 => {
2089         type => 'boolean',
2090         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2091         optional => 1,
2092     },
2093 };
2094
2095 sub parse_smbios1 {
2096     my ($data) = @_;
2097
2098     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2099     warn $@ if $@;
2100     return $res;
2101 }
2102
2103 sub print_smbios1 {
2104     my ($smbios1) = @_;
2105     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2106 }
2107
2108 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2109
2110 sub parse_watchdog {
2111     my ($value) = @_;
2112
2113     return if !$value;
2114
2115     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2116     warn $@ if $@;
2117     return $res;
2118 }
2119
2120 sub parse_guest_agent {
2121     my ($conf) = @_;
2122
2123     return {} if !defined($conf->{agent});
2124
2125     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2126     warn $@ if $@;
2127
2128     # if the agent is disabled ignore the other potentially set properties
2129     return {} if !$res->{enabled};
2130     return $res;
2131 }
2132
2133 sub get_qga_key {
2134     my ($conf, $key) = @_;
2135     return undef if !defined($conf->{agent});
2136
2137     my $agent = parse_guest_agent($conf);
2138     return $agent->{$key};
2139 }
2140
2141 sub parse_vga {
2142     my ($value) = @_;
2143
2144     return {} if !$value;
2145     my $res = eval { parse_property_string($vga_fmt, $value) };
2146     warn $@ if $@;
2147     return $res;
2148 }
2149
2150 sub parse_rng {
2151     my ($value) = @_;
2152
2153     return if !$value;
2154
2155     my $res = eval { parse_property_string($rng_fmt, $value) };
2156     warn $@ if $@;
2157     return $res;
2158 }
2159
2160 sub parse_meta_info {
2161     my ($value) = @_;
2162
2163     return if !$value;
2164
2165     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2166     warn $@ if $@;
2167     return $res;
2168 }
2169
2170 sub new_meta_info_string {
2171     my () = @_; # for now do not allow to override any value
2172
2173     return PVE::JSONSchema::print_property_string(
2174         {
2175             'creation-qemu' => kvm_user_version(),
2176             ctime => "". int(time()),
2177         },
2178         $meta_info_fmt
2179     );
2180 }
2181
2182 sub qemu_created_version_fixups {
2183     my ($conf, $forcemachine, $kvmver) = @_;
2184
2185     my $meta = parse_meta_info($conf->{meta}) // {};
2186     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2187
2188     # check if we need to apply some handling for VMs that always use the latest machine version but
2189     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2190     # would be required (we do not want to pin machine version for non-windows OS type)
2191     if (
2192         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2193         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2194         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2195         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2196     ) {
2197         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2198         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2199             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2200             # and thus with the predictable interface naming of systemd
2201             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2202         }
2203     }
2204     return;
2205 }
2206
2207 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2208 sub verify_usb_device {
2209     my ($value, $noerr) = @_;
2210
2211     return $value if parse_usb_device($value);
2212
2213     return if $noerr;
2214
2215     die "unable to parse usb device\n";
2216 }
2217
2218 # add JSON properties for create and set function
2219 sub json_config_properties {
2220     my ($prop, $with_disk_alloc) = @_;
2221
2222     my $skip_json_config_opts = {
2223         parent => 1,
2224         snaptime => 1,
2225         vmstate => 1,
2226         runningmachine => 1,
2227         runningcpu => 1,
2228         meta => 1,
2229     };
2230
2231     foreach my $opt (keys %$confdesc) {
2232         next if $skip_json_config_opts->{$opt};
2233
2234         if ($with_disk_alloc && is_valid_drivename($opt)) {
2235             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2236         } else {
2237             $prop->{$opt} = $confdesc->{$opt};
2238         }
2239     }
2240
2241     return $prop;
2242 }
2243
2244 # Properties that we can read from an OVF file
2245 sub json_ovf_properties {
2246     my $prop = {};
2247
2248     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2249         $prop->{$device} = {
2250             type => 'string',
2251             format => 'pve-volume-id-or-absolute-path',
2252             description => "Disk image that gets imported to $device",
2253             optional => 1,
2254         };
2255     }
2256
2257     $prop->{cores} = {
2258         type => 'integer',
2259         description => "The number of CPU cores.",
2260         optional => 1,
2261     };
2262     $prop->{memory} = {
2263         type => 'integer',
2264         description => "Amount of RAM for the VM in MB.",
2265         optional => 1,
2266     };
2267     $prop->{name} = {
2268         type => 'string',
2269         description => "Name of the VM.",
2270         optional => 1,
2271     };
2272
2273     return $prop;
2274 }
2275
2276 # return copy of $confdesc_cloudinit to generate documentation
2277 sub cloudinit_config_properties {
2278
2279     return dclone($confdesc_cloudinit);
2280 }
2281
2282 sub check_type {
2283     my ($key, $value) = @_;
2284
2285     die "unknown setting '$key'\n" if !$confdesc->{$key};
2286
2287     my $type = $confdesc->{$key}->{type};
2288
2289     if (!defined($value)) {
2290         die "got undefined value\n";
2291     }
2292
2293     if ($value =~ m/[\n\r]/) {
2294         die "property contains a line feed\n";
2295     }
2296
2297     if ($type eq 'boolean') {
2298         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2299         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2300         die "type check ('boolean') failed - got '$value'\n";
2301     } elsif ($type eq 'integer') {
2302         return int($1) if $value =~ m/^(\d+)$/;
2303         die "type check ('integer') failed - got '$value'\n";
2304     } elsif ($type eq 'number') {
2305         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2306         die "type check ('number') failed - got '$value'\n";
2307     } elsif ($type eq 'string') {
2308         if (my $fmt = $confdesc->{$key}->{format}) {
2309             PVE::JSONSchema::check_format($fmt, $value);
2310             return $value;
2311         }
2312         $value =~ s/^\"(.*)\"$/$1/;
2313         return $value;
2314     } else {
2315         die "internal error"
2316     }
2317 }
2318
2319 sub destroy_vm {
2320     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2321
2322     my $conf = PVE::QemuConfig->load_config($vmid);
2323
2324     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2325
2326     if ($conf->{template}) {
2327         # check if any base image is still used by a linked clone
2328         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2329                 my ($ds, $drive) = @_;
2330                 return if drive_is_cdrom($drive);
2331
2332                 my $volid = $drive->{file};
2333                 return if !$volid || $volid =~ m|^/|;
2334
2335                 die "base volume '$volid' is still in use by linked cloned\n"
2336                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2337
2338         });
2339     }
2340
2341     my $volids = {};
2342     my $remove_owned_drive = sub {
2343         my ($ds, $drive) = @_;
2344         return if drive_is_cdrom($drive, 1);
2345
2346         my $volid = $drive->{file};
2347         return if !$volid || $volid =~ m|^/|;
2348         return if $volids->{$volid};
2349
2350         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2351         return if !$path || !$owner || ($owner != $vmid);
2352
2353         $volids->{$volid} = 1;
2354         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2355         warn "Could not remove disk '$volid', check manually: $@" if $@;
2356     };
2357
2358     # only remove disks owned by this VM (referenced in the config)
2359     my $include_opts = {
2360         include_unused => 1,
2361         extra_keys => ['vmstate'],
2362     };
2363     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2364
2365     for my $snap (values %{$conf->{snapshots}}) {
2366         next if !defined($snap->{vmstate});
2367         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2368         next if !defined($drive);
2369         $remove_owned_drive->('vmstate', $drive);
2370     }
2371
2372     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2373
2374     if ($purge_unreferenced) { # also remove unreferenced disk
2375         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2376         PVE::Storage::foreach_volid($vmdisks, sub {
2377             my ($volid, $sid, $volname, $d) = @_;
2378             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2379             warn $@ if $@;
2380         });
2381     }
2382
2383     if (defined $replacement_conf) {
2384         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2385     } else {
2386         PVE::QemuConfig->destroy_config($vmid);
2387     }
2388 }
2389
2390 sub parse_vm_config {
2391     my ($filename, $raw, $strict) = @_;
2392
2393     return if !defined($raw);
2394
2395     my $res = {
2396         digest => Digest::SHA::sha1_hex($raw),
2397         snapshots => {},
2398         pending => {},
2399         cloudinit => {},
2400     };
2401
2402     my $handle_error = sub {
2403         my ($msg) = @_;
2404
2405         if ($strict) {
2406             die $msg;
2407         } else {
2408             warn $msg;
2409         }
2410     };
2411
2412     $filename =~ m|/qemu-server/(\d+)\.conf$|
2413         || die "got strange filename '$filename'";
2414
2415     my $vmid = $1;
2416
2417     my $conf = $res;
2418     my $descr;
2419     my $section = '';
2420
2421     my @lines = split(/\n/, $raw);
2422     foreach my $line (@lines) {
2423         next if $line =~ m/^\s*$/;
2424
2425         if ($line =~ m/^\[PENDING\]\s*$/i) {
2426             $section = 'pending';
2427             if (defined($descr)) {
2428                 $descr =~ s/\s+$//;
2429                 $conf->{description} = $descr;
2430             }
2431             $descr = undef;
2432             $conf = $res->{$section} = {};
2433             next;
2434         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2435             $section = 'cloudinit';
2436             $descr = undef;
2437             $conf = $res->{$section} = {};
2438             next;
2439
2440         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2441             $section = $1;
2442             if (defined($descr)) {
2443                 $descr =~ s/\s+$//;
2444                 $conf->{description} = $descr;
2445             }
2446             $descr = undef;
2447             $conf = $res->{snapshots}->{$section} = {};
2448             next;
2449         }
2450
2451         if ($line =~ m/^\#(.*)$/) {
2452             $descr = '' if !defined($descr);
2453             $descr .= PVE::Tools::decode_text($1) . "\n";
2454             next;
2455         }
2456
2457         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2458             $descr = '' if !defined($descr);
2459             $descr .= PVE::Tools::decode_text($2);
2460         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2461             $conf->{snapstate} = $1;
2462         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2463             my $key = $1;
2464             my $value = $2;
2465             $conf->{$key} = $value;
2466         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2467             my $value = $1;
2468             if ($section eq 'pending') {
2469                 $conf->{delete} = $value; # we parse this later
2470             } else {
2471                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2472             }
2473         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2474             my $key = $1;
2475             my $value = $2;
2476             eval { $value = check_type($key, $value); };
2477             if ($@) {
2478                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2479             } else {
2480                 $key = 'ide2' if $key eq 'cdrom';
2481                 my $fmt = $confdesc->{$key}->{format};
2482                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2483                     my $v = parse_drive($key, $value);
2484                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2485                         $v->{file} = $volid;
2486                         $value = print_drive($v);
2487                     } else {
2488                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2489                         next;
2490                     }
2491                 }
2492
2493                 $conf->{$key} = $value;
2494             }
2495         } else {
2496             $handle_error->("vm $vmid - unable to parse config: $line\n");
2497         }
2498     }
2499
2500     if (defined($descr)) {
2501         $descr =~ s/\s+$//;
2502         $conf->{description} = $descr;
2503     }
2504     delete $res->{snapstate}; # just to be sure
2505
2506     return $res;
2507 }
2508
2509 sub write_vm_config {
2510     my ($filename, $conf) = @_;
2511
2512     delete $conf->{snapstate}; # just to be sure
2513
2514     if ($conf->{cdrom}) {
2515         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2516         $conf->{ide2} = $conf->{cdrom};
2517         delete $conf->{cdrom};
2518     }
2519
2520     # we do not use 'smp' any longer
2521     if ($conf->{sockets}) {
2522         delete $conf->{smp};
2523     } elsif ($conf->{smp}) {
2524         $conf->{sockets} = $conf->{smp};
2525         delete $conf->{cores};
2526         delete $conf->{smp};
2527     }
2528
2529     my $used_volids = {};
2530
2531     my $cleanup_config = sub {
2532         my ($cref, $pending, $snapname) = @_;
2533
2534         foreach my $key (keys %$cref) {
2535             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2536                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2537             my $value = $cref->{$key};
2538             if ($key eq 'delete') {
2539                 die "propertry 'delete' is only allowed in [PENDING]\n"
2540                     if !$pending;
2541                 # fixme: check syntax?
2542                 next;
2543             }
2544             eval { $value = check_type($key, $value); };
2545             die "unable to parse value of '$key' - $@" if $@;
2546
2547             $cref->{$key} = $value;
2548
2549             if (!$snapname && is_valid_drivename($key)) {
2550                 my $drive = parse_drive($key, $value);
2551                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2552             }
2553         }
2554     };
2555
2556     &$cleanup_config($conf);
2557
2558     &$cleanup_config($conf->{pending}, 1);
2559
2560     &$cleanup_config($conf->{cloudinit});
2561
2562     foreach my $snapname (keys %{$conf->{snapshots}}) {
2563         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2564         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2565     }
2566
2567     # remove 'unusedX' settings if we re-add a volume
2568     foreach my $key (keys %$conf) {
2569         my $value = $conf->{$key};
2570         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2571             delete $conf->{$key};
2572         }
2573     }
2574
2575     my $generate_raw_config = sub {
2576         my ($conf, $pending) = @_;
2577
2578         my $raw = '';
2579
2580         # add description as comment to top of file
2581         if (defined(my $descr = $conf->{description})) {
2582             if ($descr) {
2583                 foreach my $cl (split(/\n/, $descr)) {
2584                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2585                 }
2586             } else {
2587                 $raw .= "#\n" if $pending;
2588             }
2589         }
2590
2591         foreach my $key (sort keys %$conf) {
2592             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2593             $raw .= "$key: $conf->{$key}\n";
2594         }
2595         return $raw;
2596     };
2597
2598     my $raw = &$generate_raw_config($conf);
2599
2600     if (scalar(keys %{$conf->{pending}})){
2601         $raw .= "\n[PENDING]\n";
2602         $raw .= &$generate_raw_config($conf->{pending}, 1);
2603     }
2604
2605     if (scalar(keys %{$conf->{cloudinit}})){
2606         $raw .= "\n[special:cloudinit]\n";
2607         $raw .= &$generate_raw_config($conf->{cloudinit});
2608     }
2609
2610     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2611         $raw .= "\n[$snapname]\n";
2612         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2613     }
2614
2615     return $raw;
2616 }
2617
2618 sub load_defaults {
2619
2620     my $res = {};
2621
2622     # we use static defaults from our JSON schema configuration
2623     foreach my $key (keys %$confdesc) {
2624         if (defined(my $default = $confdesc->{$key}->{default})) {
2625             $res->{$key} = $default;
2626         }
2627     }
2628
2629     return $res;
2630 }
2631
2632 sub config_list {
2633     my $vmlist = PVE::Cluster::get_vmlist();
2634     my $res = {};
2635     return $res if !$vmlist || !$vmlist->{ids};
2636     my $ids = $vmlist->{ids};
2637     my $nodename = nodename();
2638
2639     foreach my $vmid (keys %$ids) {
2640         my $d = $ids->{$vmid};
2641         next if !$d->{node} || $d->{node} ne $nodename;
2642         next if !$d->{type} || $d->{type} ne 'qemu';
2643         $res->{$vmid}->{exists} = 1;
2644     }
2645     return $res;
2646 }
2647
2648 # test if VM uses local resources (to prevent migration)
2649 sub check_local_resources {
2650     my ($conf, $noerr) = @_;
2651
2652     my @loc_res = ();
2653
2654     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2655     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2656
2657     push @loc_res, "ivshmem" if $conf->{ivshmem};
2658
2659     foreach my $k (keys %$conf) {
2660         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2661         # sockets are safe: they will recreated be on the target side post-migrate
2662         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2663         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2664     }
2665
2666     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2667
2668     return \@loc_res;
2669 }
2670
2671 # check if used storages are available on all nodes (use by migrate)
2672 sub check_storage_availability {
2673     my ($storecfg, $conf, $node) = @_;
2674
2675     PVE::QemuConfig->foreach_volume($conf, sub {
2676         my ($ds, $drive) = @_;
2677
2678         my $volid = $drive->{file};
2679         return if !$volid;
2680
2681         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2682         return if !$sid;
2683
2684         # check if storage is available on both nodes
2685         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2686         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2687
2688         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2689
2690         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2691             if !$scfg->{content}->{$vtype};
2692    });
2693 }
2694
2695 # list nodes where all VM images are available (used by has_feature API)
2696 sub shared_nodes {
2697     my ($conf, $storecfg) = @_;
2698
2699     my $nodelist = PVE::Cluster::get_nodelist();
2700     my $nodehash = { map { $_ => 1 } @$nodelist };
2701     my $nodename = nodename();
2702
2703     PVE::QemuConfig->foreach_volume($conf, sub {
2704         my ($ds, $drive) = @_;
2705
2706         my $volid = $drive->{file};
2707         return if !$volid;
2708
2709         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2710         if ($storeid) {
2711             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2712             if ($scfg->{disable}) {
2713                 $nodehash = {};
2714             } elsif (my $avail = $scfg->{nodes}) {
2715                 foreach my $node (keys %$nodehash) {
2716                     delete $nodehash->{$node} if !$avail->{$node};
2717                 }
2718             } elsif (!$scfg->{shared}) {
2719                 foreach my $node (keys %$nodehash) {
2720                     delete $nodehash->{$node} if $node ne $nodename
2721                 }
2722             }
2723         }
2724     });
2725
2726     return $nodehash
2727 }
2728
2729 sub check_local_storage_availability {
2730     my ($conf, $storecfg) = @_;
2731
2732     my $nodelist = PVE::Cluster::get_nodelist();
2733     my $nodehash = { map { $_ => {} } @$nodelist };
2734
2735     PVE::QemuConfig->foreach_volume($conf, sub {
2736         my ($ds, $drive) = @_;
2737
2738         my $volid = $drive->{file};
2739         return if !$volid;
2740
2741         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2742         if ($storeid) {
2743             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2744
2745             if ($scfg->{disable}) {
2746                 foreach my $node (keys %$nodehash) {
2747                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2748                 }
2749             } elsif (my $avail = $scfg->{nodes}) {
2750                 foreach my $node (keys %$nodehash) {
2751                     if (!$avail->{$node}) {
2752                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2753                     }
2754                 }
2755             }
2756         }
2757     });
2758
2759     foreach my $node (values %$nodehash) {
2760         if (my $unavail = $node->{unavailable_storages}) {
2761             $node->{unavailable_storages} = [ sort keys %$unavail ];
2762         }
2763     }
2764
2765     return $nodehash
2766 }
2767
2768 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2769 sub check_running {
2770     my ($vmid, $nocheck, $node) = @_;
2771
2772     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2773     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2774 }
2775
2776 sub vzlist {
2777
2778     my $vzlist = config_list();
2779
2780     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2781
2782     while (defined(my $de = $fd->read)) {
2783         next if $de !~ m/^(\d+)\.pid$/;
2784         my $vmid = $1;
2785         next if !defined($vzlist->{$vmid});
2786         if (my $pid = check_running($vmid)) {
2787             $vzlist->{$vmid}->{pid} = $pid;
2788         }
2789     }
2790
2791     return $vzlist;
2792 }
2793
2794 our $vmstatus_return_properties = {
2795     vmid => get_standard_option('pve-vmid'),
2796     status => {
2797         description => "Qemu process status.",
2798         type => 'string',
2799         enum => ['stopped', 'running'],
2800     },
2801     maxmem => {
2802         description => "Maximum memory in bytes.",
2803         type => 'integer',
2804         optional => 1,
2805         renderer => 'bytes',
2806     },
2807     maxdisk => {
2808         description => "Root disk size in bytes.",
2809         type => 'integer',
2810         optional => 1,
2811         renderer => 'bytes',
2812     },
2813     name => {
2814         description => "VM name.",
2815         type => 'string',
2816         optional => 1,
2817     },
2818     qmpstatus => {
2819         description => "Qemu QMP agent status.",
2820         type => 'string',
2821         optional => 1,
2822     },
2823     pid => {
2824         description => "PID of running qemu process.",
2825         type => 'integer',
2826         optional => 1,
2827     },
2828     uptime => {
2829         description => "Uptime.",
2830         type => 'integer',
2831         optional => 1,
2832         renderer => 'duration',
2833     },
2834     cpus => {
2835         description => "Maximum usable CPUs.",
2836         type => 'number',
2837         optional => 1,
2838     },
2839     lock => {
2840         description => "The current config lock, if any.",
2841         type => 'string',
2842         optional => 1,
2843     },
2844     tags => {
2845         description  => "The current configured tags, if any",
2846         type => 'string',
2847         optional => 1,
2848     },
2849     'running-machine' => {
2850         description => "The currently running machine type (if running).",
2851         type => 'string',
2852         optional => 1,
2853     },
2854     'running-qemu' => {
2855         description => "The currently running QEMU version (if running).",
2856         type => 'string',
2857         optional => 1,
2858     },
2859 };
2860
2861 my $last_proc_pid_stat;
2862
2863 # get VM status information
2864 # This must be fast and should not block ($full == false)
2865 # We only query KVM using QMP if $full == true (this can be slow)
2866 sub vmstatus {
2867     my ($opt_vmid, $full) = @_;
2868
2869     my $res = {};
2870
2871     my $storecfg = PVE::Storage::config();
2872
2873     my $list = vzlist();
2874     my $defaults = load_defaults();
2875
2876     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2877
2878     my $cpucount = $cpuinfo->{cpus} || 1;
2879
2880     foreach my $vmid (keys %$list) {
2881         next if $opt_vmid && ($vmid ne $opt_vmid);
2882
2883         my $conf = PVE::QemuConfig->load_config($vmid);
2884
2885         my $d = { vmid => int($vmid) };
2886         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2887
2888         # fixme: better status?
2889         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2890
2891         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2892         if (defined($size)) {
2893             $d->{disk} = 0; # no info available
2894             $d->{maxdisk} = $size;
2895         } else {
2896             $d->{disk} = 0;
2897             $d->{maxdisk} = 0;
2898         }
2899
2900         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2901             * ($conf->{cores} || $defaults->{cores});
2902         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2903         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2904
2905         $d->{name} = $conf->{name} || "VM $vmid";
2906         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2907             : $defaults->{memory}*(1024*1024);
2908
2909         if ($conf->{balloon}) {
2910             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2911             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2912                 : $defaults->{shares};
2913         }
2914
2915         $d->{uptime} = 0;
2916         $d->{cpu} = 0;
2917         $d->{mem} = 0;
2918
2919         $d->{netout} = 0;
2920         $d->{netin} = 0;
2921
2922         $d->{diskread} = 0;
2923         $d->{diskwrite} = 0;
2924
2925         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2926
2927         $d->{serial} = 1 if conf_has_serial($conf);
2928         $d->{lock} = $conf->{lock} if $conf->{lock};
2929         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2930
2931         $res->{$vmid} = $d;
2932     }
2933
2934     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2935     foreach my $dev (keys %$netdev) {
2936         next if $dev !~ m/^tap([1-9]\d*)i/;
2937         my $vmid = $1;
2938         my $d = $res->{$vmid};
2939         next if !$d;
2940
2941         $d->{netout} += $netdev->{$dev}->{receive};
2942         $d->{netin} += $netdev->{$dev}->{transmit};
2943
2944         if ($full) {
2945             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2946             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2947         }
2948
2949     }
2950
2951     my $ctime = gettimeofday;
2952
2953     foreach my $vmid (keys %$list) {
2954
2955         my $d = $res->{$vmid};
2956         my $pid = $d->{pid};
2957         next if !$pid;
2958
2959         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2960         next if !$pstat; # not running
2961
2962         my $used = $pstat->{utime} + $pstat->{stime};
2963
2964         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2965
2966         if ($pstat->{vsize}) {
2967             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2968         }
2969
2970         my $old = $last_proc_pid_stat->{$pid};
2971         if (!$old) {
2972             $last_proc_pid_stat->{$pid} = {
2973                 time => $ctime,
2974                 used => $used,
2975                 cpu => 0,
2976             };
2977             next;
2978         }
2979
2980         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2981
2982         if ($dtime > 1000) {
2983             my $dutime = $used -  $old->{used};
2984
2985             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2986             $last_proc_pid_stat->{$pid} = {
2987                 time => $ctime,
2988                 used => $used,
2989                 cpu => $d->{cpu},
2990             };
2991         } else {
2992             $d->{cpu} = $old->{cpu};
2993         }
2994     }
2995
2996     return $res if !$full;
2997
2998     my $qmpclient = PVE::QMPClient->new();
2999
3000     my $ballooncb = sub {
3001         my ($vmid, $resp) = @_;
3002
3003         my $info = $resp->{'return'};
3004         return if !$info->{max_mem};
3005
3006         my $d = $res->{$vmid};
3007
3008         # use memory assigned to VM
3009         $d->{maxmem} = $info->{max_mem};
3010         $d->{balloon} = $info->{actual};
3011
3012         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3013             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3014             $d->{freemem} = $info->{free_mem};
3015         }
3016
3017         $d->{ballooninfo} = $info;
3018     };
3019
3020     my $blockstatscb = sub {
3021         my ($vmid, $resp) = @_;
3022         my $data = $resp->{'return'} || [];
3023         my $totalrdbytes = 0;
3024         my $totalwrbytes = 0;
3025
3026         for my $blockstat (@$data) {
3027             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3028             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3029
3030             $blockstat->{device} =~ s/drive-//;
3031             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3032         }
3033         $res->{$vmid}->{diskread} = $totalrdbytes;
3034         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3035     };
3036
3037     my $machinecb = sub {
3038         my ($vmid, $resp) = @_;
3039         my $data = $resp->{'return'} || [];
3040
3041         $res->{$vmid}->{'running-machine'} =
3042             PVE::QemuServer::Machine::current_from_query_machines($data);
3043     };
3044
3045     my $versioncb = sub {
3046         my ($vmid, $resp) = @_;
3047         my $data = $resp->{'return'} // {};
3048         my $version = 'unknown';
3049
3050         if (my $v = $data->{qemu}) {
3051             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3052         }
3053
3054         $res->{$vmid}->{'running-qemu'} = $version;
3055     };
3056
3057     my $statuscb = sub {
3058         my ($vmid, $resp) = @_;
3059
3060         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3061         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3062         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3063         # this fails if ballon driver is not loaded, so this must be
3064         # the last commnand (following command are aborted if this fails).
3065         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3066
3067         my $status = 'unknown';
3068         if (!defined($status = $resp->{'return'}->{status})) {
3069             warn "unable to get VM status\n";
3070             return;
3071         }
3072
3073         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3074     };
3075
3076     foreach my $vmid (keys %$list) {
3077         next if $opt_vmid && ($vmid ne $opt_vmid);
3078         next if !$res->{$vmid}->{pid}; # not running
3079         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3080     }
3081
3082     $qmpclient->queue_execute(undef, 2);
3083
3084     foreach my $vmid (keys %$list) {
3085         next if $opt_vmid && ($vmid ne $opt_vmid);
3086         next if !$res->{$vmid}->{pid}; #not running
3087
3088         # we can't use the $qmpclient since it might have already aborted on
3089         # 'query-balloon', but this might also fail for older versions...
3090         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3091         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3092     }
3093
3094     foreach my $vmid (keys %$list) {
3095         next if $opt_vmid && ($vmid ne $opt_vmid);
3096         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3097     }
3098
3099     return $res;
3100 }
3101
3102 sub conf_has_serial {
3103     my ($conf) = @_;
3104
3105     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3106         if ($conf->{"serial$i"}) {
3107             return 1;
3108         }
3109     }
3110
3111     return 0;
3112 }
3113
3114 sub conf_has_audio {
3115     my ($conf, $id) = @_;
3116
3117     $id //= 0;
3118     my $audio = $conf->{"audio$id"};
3119     return if !defined($audio);
3120
3121     my $audioproperties = parse_property_string($audio_fmt, $audio);
3122     my $audiodriver = $audioproperties->{driver} // 'spice';
3123
3124     return {
3125         dev => $audioproperties->{device},
3126         dev_id => "audiodev$id",
3127         backend => $audiodriver,
3128         backend_id => "$audiodriver-backend${id}",
3129     };
3130 }
3131
3132 sub audio_devs {
3133     my ($audio, $audiopciaddr, $machine_version) = @_;
3134
3135     my $devs = [];
3136
3137     my $id = $audio->{dev_id};
3138     my $audiodev = "";
3139     if (min_version($machine_version, 4, 2)) {
3140         $audiodev = ",audiodev=$audio->{backend_id}";
3141     }
3142
3143     if ($audio->{dev} eq 'AC97') {
3144         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3145     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3146         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3147         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3148         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3149     } else {
3150         die "unkown audio device '$audio->{dev}', implement me!";
3151     }
3152
3153     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3154
3155     return $devs;
3156 }
3157
3158 sub get_tpm_paths {
3159     my ($vmid) = @_;
3160     return {
3161         socket => "/var/run/qemu-server/$vmid.swtpm",
3162         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3163     };
3164 }
3165
3166 sub add_tpm_device {
3167     my ($vmid, $devices, $conf) = @_;
3168
3169     return if !$conf->{tpmstate0};
3170
3171     my $paths = get_tpm_paths($vmid);
3172
3173     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3174     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3175     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3176 }
3177
3178 sub start_swtpm {
3179     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3180
3181     return if !$tpmdrive;
3182
3183     my $state;
3184     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3185     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3186     if ($storeid) {
3187         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3188     } else {
3189         $state = $tpm->{file};
3190     }
3191
3192     my $paths = get_tpm_paths($vmid);
3193
3194     # during migration, we will get state from remote
3195     #
3196     if (!$migration) {
3197         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3198         my $setup_cmd = [
3199             "swtpm_setup",
3200             "--tpmstate",
3201             "file://$state",
3202             "--createek",
3203             "--create-ek-cert",
3204             "--create-platform-cert",
3205             "--lock-nvram",
3206             "--config",
3207             "/etc/swtpm_setup.conf", # do not use XDG configs
3208             "--runas",
3209             "0", # force creation as root, error if not possible
3210             "--not-overwrite", # ignore existing state, do not modify
3211         ];
3212
3213         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3214         # TPM 2.0 supports ECC crypto, use if possible
3215         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3216
3217         run_command($setup_cmd, outfunc => sub {
3218             print "swtpm_setup: $1\n";
3219         });
3220     }
3221
3222     my $emulator_cmd = [
3223         "swtpm",
3224         "socket",
3225         "--tpmstate",
3226         "backend-uri=file://$state,mode=0600",
3227         "--ctrl",
3228         "type=unixio,path=$paths->{socket},mode=0600",
3229         "--pid",
3230         "file=$paths->{pid}",
3231         "--terminate", # terminate on QEMU disconnect
3232         "--daemon",
3233     ];
3234     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3235     run_command($emulator_cmd, outfunc => sub { print $1; });
3236
3237     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3238     while (! -e $paths->{pid}) {
3239         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3240         usleep(50_000);
3241     }
3242
3243     # return untainted PID of swtpm daemon so it can be killed on error
3244     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3245     return $1;
3246 }
3247
3248 sub vga_conf_has_spice {
3249     my ($vga) = @_;
3250
3251     my $vgaconf = parse_vga($vga);
3252     my $vgatype = $vgaconf->{type};
3253     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3254
3255     return $1 || 1;
3256 }
3257
3258 sub is_native($) {
3259     my ($arch) = @_;
3260     return get_host_arch() eq $arch;
3261 }
3262
3263 sub get_vm_arch {
3264     my ($conf) = @_;
3265     return $conf->{arch} // get_host_arch();
3266 }
3267
3268 my $default_machines = {
3269     x86_64 => 'pc',
3270     aarch64 => 'virt',
3271 };
3272
3273 sub get_installed_machine_version {
3274     my ($kvmversion) = @_;
3275     $kvmversion = kvm_user_version() if !defined($kvmversion);
3276     $kvmversion =~ m/^(\d+\.\d+)/;
3277     return $1;
3278 }
3279
3280 sub windows_get_pinned_machine_version {
3281     my ($machine, $base_version, $kvmversion) = @_;
3282
3283     my $pin_version = $base_version;
3284     if (!defined($base_version) ||
3285         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3286     ) {
3287         $pin_version = get_installed_machine_version($kvmversion);
3288     }
3289     if (!$machine || $machine eq 'pc') {
3290         $machine = "pc-i440fx-$pin_version";
3291     } elsif ($machine eq 'q35') {
3292         $machine = "pc-q35-$pin_version";
3293     } elsif ($machine eq 'virt') {
3294         $machine = "virt-$pin_version";
3295     } else {
3296         warn "unknown machine type '$machine', not touching that!\n";
3297     }
3298
3299     return $machine;
3300 }
3301
3302 sub get_vm_machine {
3303     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3304
3305     my $machine = $forcemachine || $conf->{machine};
3306
3307     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3308         $kvmversion //= kvm_user_version();
3309         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3310         # layout which confuses windows quite a bit and may result in various regressions..
3311         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3312         if (windows_version($conf->{ostype})) {
3313             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3314         }
3315         $arch //= 'x86_64';
3316         $machine ||= $default_machines->{$arch};
3317         if ($add_pve_version) {
3318             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3319             $machine .= "+pve$pvever";
3320         }
3321     }
3322
3323     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3324         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3325         $machine = $1 if $is_pxe;
3326
3327         # for version-pinned machines that do not include a pve-version (e.g.
3328         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3329         $machine .= '+pve0';
3330
3331         $machine .= '.pxe' if $is_pxe;
3332     }
3333
3334     return $machine;
3335 }
3336
3337 sub get_ovmf_files($$$) {
3338     my ($arch, $efidisk, $smm) = @_;
3339
3340     my $types = $OVMF->{$arch}
3341         or die "no OVMF images known for architecture '$arch'\n";
3342
3343     my $type = 'default';
3344     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3345         $type = $smm ? "4m" : "4m-no-smm";
3346         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3347     }
3348
3349     return $types->{$type}->@*;
3350 }
3351
3352 my $Arch2Qemu = {
3353     aarch64 => '/usr/bin/qemu-system-aarch64',
3354     x86_64 => '/usr/bin/qemu-system-x86_64',
3355 };
3356 sub get_command_for_arch($) {
3357     my ($arch) = @_;
3358     return '/usr/bin/kvm' if is_native($arch);
3359
3360     my $cmd = $Arch2Qemu->{$arch}
3361         or die "don't know how to emulate architecture '$arch'\n";
3362     return $cmd;
3363 }
3364
3365 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3366 # to use in a QEMU command line (-cpu element), first array_intersect the result
3367 # of query_supported_ with query_understood_. This is necessary because:
3368 #
3369 # a) query_understood_ returns flags the host cannot use and
3370 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3371 #    flags, but CPU settings - with most of them being flags. Those settings
3372 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3373 #
3374 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3375 # expensive. If you need the value returned from this, you can get it much
3376 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3377 # $accel being 'kvm' or 'tcg'.
3378 #
3379 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3380 # changes, automatically populating pmxcfs.
3381 #
3382 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3383 # since kvm and tcg machines support different flags
3384 #
3385 sub query_supported_cpu_flags {
3386     my ($arch) = @_;
3387
3388     $arch //= get_host_arch();
3389     my $default_machine = $default_machines->{$arch};
3390
3391     my $flags = {};
3392
3393     # FIXME: Once this is merged, the code below should work for ARM as well:
3394     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3395     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3396         $arch eq "aarch64";
3397
3398     my $kvm_supported = defined(kvm_version());
3399     my $qemu_cmd = get_command_for_arch($arch);
3400     my $fakevmid = -1;
3401     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3402
3403     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3404     my $query_supported_run_qemu = sub {
3405         my ($kvm) = @_;
3406
3407         my $flags = {};
3408         my $cmd = [
3409             $qemu_cmd,
3410             '-machine', $default_machine,
3411             '-display', 'none',
3412             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3413             '-mon', 'chardev=qmp,mode=control',
3414             '-pidfile', $pidfile,
3415             '-S', '-daemonize'
3416         ];
3417
3418         if (!$kvm) {
3419             push @$cmd, '-accel', 'tcg';
3420         }
3421
3422         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3423         die "QEMU flag querying VM exited with code " . $rc if $rc;
3424
3425         eval {
3426             my $cmd_result = mon_cmd(
3427                 $fakevmid,
3428                 'query-cpu-model-expansion',
3429                 type => 'full',
3430                 model => { name => 'host' }
3431             );
3432
3433             my $props = $cmd_result->{model}->{props};
3434             foreach my $prop (keys %$props) {
3435                 next if $props->{$prop} ne '1';
3436                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3437                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3438                 # We only keep those with underscores, to match /proc/cpuinfo
3439                 $prop =~ s/\.|-/_/g;
3440                 $flags->{$prop} = 1;
3441             }
3442         };
3443         my $err = $@;
3444
3445         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3446         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3447
3448         die $err if $err;
3449
3450         return [ sort keys %$flags ];
3451     };
3452
3453     # We need to query QEMU twice, since KVM and TCG have different supported flags
3454     PVE::QemuConfig->lock_config($fakevmid, sub {
3455         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3456         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3457
3458         if ($kvm_supported) {
3459             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3460             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3461         }
3462     });
3463
3464     return $flags;
3465 }
3466
3467 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3468 my $understood_cpu_flag_dir = "/usr/share/kvm";
3469 sub query_understood_cpu_flags {
3470     my $arch = get_host_arch();
3471     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3472
3473     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3474         if ! -e $filepath;
3475
3476     my $raw = file_get_contents($filepath);
3477     $raw =~ s/^\s+|\s+$//g;
3478     my @flags = split(/\s+/, $raw);
3479
3480     return \@flags;
3481 }
3482
3483 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3484 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3485 my sub should_disable_smm {
3486     my ($conf, $vga) = @_;
3487
3488     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3489         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3490 }
3491
3492 sub config_to_command {
3493     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3494         $pbs_backing) = @_;
3495
3496     my $cmd = [];
3497     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3498     my $devices = [];
3499     my $bridges = {};
3500     my $ostype = $conf->{ostype};
3501     my $winversion = windows_version($ostype);
3502     my $kvm = $conf->{kvm};
3503     my $nodename = nodename();
3504
3505     my $arch = get_vm_arch($conf);
3506     my $kvm_binary = get_command_for_arch($arch);
3507     my $kvmver = kvm_user_version($kvm_binary);
3508
3509     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3510         $kvmver //= "undefined";
3511         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3512     }
3513
3514     my $add_pve_version = min_version($kvmver, 4, 1);
3515
3516     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3517     my $machine_version = extract_version($machine_type, $kvmver);
3518     $kvm //= 1 if is_native($arch);
3519
3520     $machine_version =~ m/(\d+)\.(\d+)/;
3521     my ($machine_major, $machine_minor) = ($1, $2);
3522
3523     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3524         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3525     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3526         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3527             ." please upgrade node '$nodename'\n"
3528     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3529         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3530         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3531             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3532             ." node '$nodename'\n";
3533     }
3534
3535     # if a specific +pve version is required for a feature, use $version_guard
3536     # instead of min_version to allow machines to be run with the minimum
3537     # required version
3538     my $required_pve_version = 0;
3539     my $version_guard = sub {
3540         my ($major, $minor, $pve) = @_;
3541         return 0 if !min_version($machine_version, $major, $minor, $pve);
3542         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3543         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3544         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3545         return 1;
3546     };
3547
3548     if ($kvm && !defined kvm_version()) {
3549         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3550             ." or enable in BIOS.\n";
3551     }
3552
3553     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3554     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3555     my $use_old_bios_files = undef;
3556     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3557
3558     if ($conf->{affinity}) {
3559         push @$cmd, "/usr/bin/taskset";
3560         push @$cmd, "--cpu-list";
3561         push @$cmd, "--all-tasks";
3562         push @$cmd, $conf->{affinity};
3563     }
3564
3565     push @$cmd, $kvm_binary;
3566
3567     push @$cmd, '-id', $vmid;
3568
3569     my $vmname = $conf->{name} || "vm$vmid";
3570
3571     push @$cmd, '-name', "$vmname,debug-threads=on";
3572
3573     push @$cmd, '-no-shutdown';
3574
3575     my $use_virtio = 0;
3576
3577     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3578     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3579     push @$cmd, '-mon', "chardev=qmp,mode=control";
3580
3581     if (min_version($machine_version, 2, 12)) {
3582         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3583         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3584     }
3585
3586     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3587
3588     push @$cmd, '-daemonize';
3589
3590     if ($conf->{smbios1}) {
3591         my $smbios_conf = parse_smbios1($conf->{smbios1});
3592         if ($smbios_conf->{base64}) {
3593             # Do not pass base64 flag to qemu
3594             delete $smbios_conf->{base64};
3595             my $smbios_string = "";
3596             foreach my $key (keys %$smbios_conf) {
3597                 my $value;
3598                 if ($key eq "uuid") {
3599                     $value = $smbios_conf->{uuid}
3600                 } else {
3601                     $value = decode_base64($smbios_conf->{$key});
3602                 }
3603                 # qemu accepts any binary data, only commas need escaping by double comma
3604                 $value =~ s/,/,,/g;
3605                 $smbios_string .= "," . $key . "=" . $value if $value;
3606             }
3607             push @$cmd, '-smbios', "type=1" . $smbios_string;
3608         } else {
3609             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3610         }
3611     }
3612
3613     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3614         my $d;
3615         if (my $efidisk = $conf->{efidisk0}) {
3616             $d = parse_drive('efidisk0', $efidisk);
3617         }
3618
3619         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3620         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3621
3622         my ($path, $format);
3623         my $read_only_str = '';
3624         if ($d) {
3625             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3626             $format = $d->{format};
3627             if ($storeid) {
3628                 $path = PVE::Storage::path($storecfg, $d->{file});
3629                 if (!defined($format)) {
3630                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3631                     $format = qemu_img_format($scfg, $volname);
3632                 }
3633             } else {
3634                 $path = $d->{file};
3635                 die "efidisk format must be specified\n"
3636                     if !defined($format);
3637             }
3638
3639             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3640         } else {
3641             log_warn("no efidisk configured! Using temporary efivars disk.");
3642             $path = "/tmp/$vmid-ovmf.fd";
3643             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3644             $format = 'raw';
3645         }
3646
3647         my $size_str = "";
3648
3649         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3650             $size_str = ",size=" . (-s $ovmf_vars);
3651         }
3652
3653         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3654         my $cache = "";
3655         if ($path =~ m/^rbd:/) {
3656                 $cache = ',cache=writeback';
3657                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3658         }
3659
3660         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3661         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3662     }
3663
3664     if ($q35) { # tell QEMU to load q35 config early
3665         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3666         if (min_version($machine_version, 4, 0)) {
3667             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3668         } else {
3669             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3670         }
3671     }
3672
3673     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3674         push @$cmd, $fixups->@*;
3675     }
3676
3677     if ($conf->{vmgenid}) {
3678         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3679     }
3680
3681     # add usb controllers
3682     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3683         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3684     push @$devices, @usbcontrollers if @usbcontrollers;
3685     my $vga = parse_vga($conf->{vga});
3686
3687     my $qxlnum = vga_conf_has_spice($conf->{vga});
3688     $vga->{type} = 'qxl' if $qxlnum;
3689
3690     if (!$vga->{type}) {
3691         if ($arch eq 'aarch64') {
3692             $vga->{type} = 'virtio';
3693         } elsif (min_version($machine_version, 2, 9)) {
3694             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3695         } else {
3696             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3697         }
3698     }
3699
3700     # enable absolute mouse coordinates (needed by vnc)
3701     my $tablet = $conf->{tablet};
3702     if (!defined($tablet)) {
3703         $tablet = $defaults->{tablet};
3704         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3705         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3706     }
3707
3708     if ($tablet) {
3709         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3710         my $kbd = print_keyboarddevice_full($conf, $arch);
3711         push @$devices, '-device', $kbd if defined($kbd);
3712     }
3713
3714     my $bootorder = device_bootorder($conf);
3715
3716     # host pci device passthrough
3717     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3718         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3719
3720     # usb devices
3721     my $usb_dev_features = {};
3722     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3723
3724     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3725         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3726     push @$devices, @usbdevices if @usbdevices;
3727
3728     # serial devices
3729     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3730         my $path = $conf->{"serial$i"} or next;
3731         if ($path eq 'socket') {
3732             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3733             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3734             # On aarch64, serial0 is the UART device. Qemu only allows
3735             # connecting UART devices via the '-serial' command line, as
3736             # the device has a fixed slot on the hardware...
3737             if ($arch eq 'aarch64' && $i == 0) {
3738                 push @$devices, '-serial', "chardev:serial$i";
3739             } else {
3740                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3741             }
3742         } else {
3743             die "no such serial device\n" if ! -c $path;
3744             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3745             push @$devices, '-device', "isa-serial,chardev=serial$i";
3746         }
3747     }
3748
3749     # parallel devices
3750     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3751         if (my $path = $conf->{"parallel$i"}) {
3752             die "no such parallel device\n" if ! -c $path;
3753             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3754             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3755             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3756         }
3757     }
3758
3759     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3760         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3761         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3762         push @$devices, @$audio_devs;
3763     }
3764
3765     add_tpm_device($vmid, $devices, $conf);
3766
3767     my $sockets = 1;
3768     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3769     $sockets = $conf->{sockets} if  $conf->{sockets};
3770
3771     my $cores = $conf->{cores} || 1;
3772
3773     my $maxcpus = $sockets * $cores;
3774
3775     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3776
3777     my $allowed_vcpus = $cpuinfo->{cpus};
3778
3779     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3780
3781     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3782         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3783         for (my $i = 2; $i <= $vcpus; $i++)  {
3784             my $cpustr = print_cpu_device($conf,$i);
3785             push @$cmd, '-device', $cpustr;
3786         }
3787
3788     } else {
3789
3790         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3791     }
3792     push @$cmd, '-nodefaults';
3793
3794     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3795
3796     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3797
3798     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3799
3800     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3801         push @$devices, '-device', print_vga_device(
3802             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3803
3804         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3805
3806         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3807         push @$cmd,  '-vnc', "unix:$socket,password=on";
3808     } else {
3809         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3810         push @$cmd, '-nographic';
3811     }
3812
3813     # time drift fix
3814     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3815     my $useLocaltime = $conf->{localtime};
3816
3817     if ($winversion >= 5) { # windows
3818         $useLocaltime = 1 if !defined($conf->{localtime});
3819
3820         # use time drift fix when acpi is enabled
3821         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3822             $tdf = 1 if !defined($conf->{tdf});
3823         }
3824     }
3825
3826     if ($winversion >= 6) {
3827         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3828         push @$cmd, '-no-hpet';
3829     }
3830
3831     push @$rtcFlags, 'driftfix=slew' if $tdf;
3832
3833     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3834         push @$rtcFlags, "base=$conf->{startdate}";
3835     } elsif ($useLocaltime) {
3836         push @$rtcFlags, 'base=localtime';
3837     }
3838
3839     if ($forcecpu) {
3840         push @$cmd, '-cpu', $forcecpu;
3841     } else {
3842         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3843     }
3844
3845     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3846
3847     push @$cmd, '-S' if $conf->{freeze};
3848
3849     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3850
3851     my $guest_agent = parse_guest_agent($conf);
3852
3853     if ($guest_agent->{enabled}) {
3854         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3855         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3856
3857         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3858             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3859             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3860             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3861         } elsif ($guest_agent->{type} eq 'isa') {
3862             push @$devices, '-device', "isa-serial,chardev=qga0";
3863         }
3864     }
3865
3866     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3867     if ($rng && $version_guard->(4, 1, 2)) {
3868         check_rng_source($rng->{source});
3869
3870         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3871         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3872         my $limiter_str = "";
3873         if ($max_bytes) {
3874             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3875         }
3876
3877         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3878         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3879         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3880     }
3881
3882     my $spice_port;
3883
3884     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3885         if ($qxlnum > 1) {
3886             if ($winversion){
3887                 for (my $i = 1; $i < $qxlnum; $i++){
3888                     push @$devices, '-device', print_vga_device(
3889                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3890                 }
3891             } else {
3892                 # assume other OS works like Linux
3893                 my ($ram, $vram) = ("134217728", "67108864");
3894                 if ($vga->{memory}) {
3895                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3896                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3897                 }
3898                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3899                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3900             }
3901         }
3902
3903         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3904
3905         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3906         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3907         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3908
3909         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3910         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3911         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3912
3913         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3914         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3915
3916         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3917         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3918         if ($spice_enhancement->{foldersharing}) {
3919             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3920             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3921         }
3922
3923         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3924         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3925             if $spice_enhancement->{videostreaming};
3926
3927         push @$devices, '-spice', "$spice_opts";
3928     }
3929
3930     # enable balloon by default, unless explicitly disabled
3931     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3932         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3933         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3934         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3935         push @$devices, '-device', $ballooncmd;
3936     }
3937
3938     if ($conf->{watchdog}) {
3939         my $wdopts = parse_watchdog($conf->{watchdog});
3940         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3941         my $watchdog = $wdopts->{model} || 'i6300esb';
3942         push @$devices, '-device', "$watchdog$pciaddr";
3943         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3944     }
3945
3946     my $vollist = [];
3947     my $scsicontroller = {};
3948     my $ahcicontroller = {};
3949     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3950
3951     # Add iscsi initiator name if available
3952     if (my $initiator = get_initiator_name()) {
3953         push @$devices, '-iscsi', "initiator-name=$initiator";
3954     }
3955
3956     PVE::QemuConfig->foreach_volume($conf, sub {
3957         my ($ds, $drive) = @_;
3958
3959         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3960             check_volume_storage_type($storecfg, $drive->{file});
3961             push @$vollist, $drive->{file};
3962         }
3963
3964         # ignore efidisk here, already added in bios/fw handling code above
3965         return if $drive->{interface} eq 'efidisk';
3966         # similar for TPM
3967         return if $drive->{interface} eq 'tpmstate';
3968
3969         $use_virtio = 1 if $ds =~ m/^virtio/;
3970
3971         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3972
3973         if ($drive->{interface} eq 'virtio'){
3974            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3975         }
3976
3977         if ($drive->{interface} eq 'scsi') {
3978
3979             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3980
3981             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3982                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3983
3984             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3985             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3986
3987             my $iothread = '';
3988             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3989                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3990                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3991             } elsif ($drive->{iothread}) {
3992                 log_warn(
3993                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3994                 );
3995             }
3996
3997             my $queues = '';
3998             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3999                 $queues = ",num_queues=$drive->{queues}";
4000             }
4001
4002             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4003                 if !$scsicontroller->{$controller};
4004             $scsicontroller->{$controller}=1;
4005         }
4006
4007         if ($drive->{interface} eq 'sata') {
4008             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4009             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4010             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4011                 if !$ahcicontroller->{$controller};
4012             $ahcicontroller->{$controller}=1;
4013         }
4014
4015         my $pbs_conf = $pbs_backing->{$ds};
4016         my $pbs_name = undef;
4017         if ($pbs_conf) {
4018             $pbs_name = "drive-$ds-pbs";
4019             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4020         }
4021
4022         my $drive_cmd = print_drive_commandline_full(
4023             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4024
4025         # extra protection for templates, but SATA and IDE don't support it..
4026         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4027
4028         push @$devices, '-drive',$drive_cmd;
4029         push @$devices, '-device', print_drivedevice_full(
4030             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4031     });
4032
4033     for (my $i = 0; $i < $MAX_NETS; $i++) {
4034         my $netname = "net$i";
4035
4036         next if !$conf->{$netname};
4037         my $d = parse_net($conf->{$netname});
4038         next if !$d;
4039
4040         $use_virtio = 1 if $d->{model} eq 'virtio';
4041
4042         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4043
4044         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4045         push @$devices, '-netdev', $netdevfull;
4046
4047         my $netdevicefull = print_netdevice_full(
4048             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4049
4050         push @$devices, '-device', $netdevicefull;
4051     }
4052
4053     if ($conf->{ivshmem}) {
4054         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4055
4056         my $bus;
4057         if ($q35) {
4058             $bus = print_pcie_addr("ivshmem");
4059         } else {
4060             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4061         }
4062
4063         my $ivshmem_name = $ivshmem->{name} // $vmid;
4064         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4065
4066         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4067         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4068             .",size=$ivshmem->{size}M";
4069     }
4070
4071     # pci.4 is nested in pci.1
4072     $bridges->{1} = 1 if $bridges->{4};
4073
4074     if (!$q35) { # add pci bridges
4075         if (min_version($machine_version, 2, 3)) {
4076            $bridges->{1} = 1;
4077            $bridges->{2} = 1;
4078         }
4079         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4080     }
4081
4082     for my $k (sort {$b cmp $a} keys %$bridges) {
4083         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4084
4085         my $k_name = $k;
4086         if ($k == 2 && $legacy_igd) {
4087             $k_name = "$k-igd";
4088         }
4089         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4090         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4091
4092         if ($q35) { # add after -readconfig pve-q35.cfg
4093             splice @$devices, 2, 0, '-device', $devstr;
4094         } else {
4095             unshift @$devices, '-device', $devstr if $k > 0;
4096         }
4097     }
4098
4099     if (!$kvm) {
4100         push @$machineFlags, 'accel=tcg';
4101     }
4102
4103     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4104
4105     my $machine_type_min = $machine_type;
4106     if ($add_pve_version) {
4107         $machine_type_min =~ s/\+pve\d+$//;
4108         $machine_type_min .= "+pve$required_pve_version";
4109     }
4110     push @$machineFlags, "type=${machine_type_min}";
4111
4112     push @$cmd, @$devices;
4113     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4114     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4115     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4116
4117     if (my $vmstate = $conf->{vmstate}) {
4118         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4119         push @$vollist, $vmstate;
4120         push @$cmd, '-loadstate', $statepath;
4121         print "activating and using '$vmstate' as vmstate\n";
4122     }
4123
4124     if (PVE::QemuConfig->is_template($conf)) {
4125         # needed to workaround base volumes being read-only
4126         push @$cmd, '-snapshot';
4127     }
4128
4129     # add custom args
4130     if ($conf->{args}) {
4131         my $aa = PVE::Tools::split_args($conf->{args});
4132         push @$cmd, @$aa;
4133     }
4134
4135     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4136 }
4137
4138 sub check_rng_source {
4139     my ($source) = @_;
4140
4141     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4142     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4143         if ! -e $source;
4144
4145     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4146     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4147         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4148         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4149         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4150             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4151             ." to the host.\n";
4152     }
4153 }
4154
4155 sub spice_port {
4156     my ($vmid) = @_;
4157
4158     my $res = mon_cmd($vmid, 'query-spice');
4159
4160     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4161 }
4162
4163 sub vm_devices_list {
4164     my ($vmid) = @_;
4165
4166     my $res = mon_cmd($vmid, 'query-pci');
4167     my $devices_to_check = [];
4168     my $devices = {};
4169     foreach my $pcibus (@$res) {
4170         push @$devices_to_check, @{$pcibus->{devices}},
4171     }
4172
4173     while (@$devices_to_check) {
4174         my $to_check = [];
4175         for my $d (@$devices_to_check) {
4176             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4177             next if !$d->{'pci_bridge'};
4178
4179             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4180             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4181         }
4182         $devices_to_check = $to_check;
4183     }
4184
4185     my $resblock = mon_cmd($vmid, 'query-block');
4186     foreach my $block (@$resblock) {
4187         if($block->{device} =~ m/^drive-(\S+)/){
4188                 $devices->{$1} = 1;
4189         }
4190     }
4191
4192     my $resmice = mon_cmd($vmid, 'query-mice');
4193     foreach my $mice (@$resmice) {
4194         if ($mice->{name} eq 'QEMU HID Tablet') {
4195             $devices->{tablet} = 1;
4196             last;
4197         }
4198     }
4199
4200     # for usb devices there is no query-usb
4201     # but we can iterate over the entries in
4202     # qom-list path=/machine/peripheral
4203     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4204     foreach my $per (@$resperipheral) {
4205         if ($per->{name} =~ m/^usb\d+$/) {
4206             $devices->{$per->{name}} = 1;
4207         }
4208     }
4209
4210     return $devices;
4211 }
4212
4213 sub vm_deviceplug {
4214     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4215
4216     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4217
4218     my $devices_list = vm_devices_list($vmid);
4219     return 1 if defined($devices_list->{$deviceid});
4220
4221     # add PCI bridge if we need it for the device
4222     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4223
4224     if ($deviceid eq 'tablet') {
4225         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4226     } elsif ($deviceid eq 'keyboard') {
4227         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4228     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4229         die "usb hotplug currently not reliable\n";
4230         # since we can't reliably hot unplug all added usb devices and usb
4231         # passthrough breaks live migration we disable usb hotplugging for now
4232         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4233     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4234         qemu_iothread_add($vmid, $deviceid, $device);
4235
4236         qemu_driveadd($storecfg, $vmid, $device);
4237         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4238
4239         qemu_deviceadd($vmid, $devicefull);
4240         eval { qemu_deviceaddverify($vmid, $deviceid); };
4241         if (my $err = $@) {
4242             eval { qemu_drivedel($vmid, $deviceid); };
4243             warn $@ if $@;
4244             die $err;
4245         }
4246     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4247         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4248         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4249         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4250
4251         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4252
4253         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4254             qemu_iothread_add($vmid, $deviceid, $device);
4255             $devicefull .= ",iothread=iothread-$deviceid";
4256         }
4257
4258         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4259             $devicefull .= ",num_queues=$device->{queues}";
4260         }
4261
4262         qemu_deviceadd($vmid, $devicefull);
4263         qemu_deviceaddverify($vmid, $deviceid);
4264     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4265         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4266         qemu_driveadd($storecfg, $vmid, $device);
4267
4268         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4269         eval { qemu_deviceadd($vmid, $devicefull); };
4270         if (my $err = $@) {
4271             eval { qemu_drivedel($vmid, $deviceid); };
4272             warn $@ if $@;
4273             die $err;
4274         }
4275     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4276         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4277
4278         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4279         my $use_old_bios_files = undef;
4280         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4281
4282         my $netdevicefull = print_netdevice_full(
4283             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4284         qemu_deviceadd($vmid, $netdevicefull);
4285         eval {
4286             qemu_deviceaddverify($vmid, $deviceid);
4287             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4288         };
4289         if (my $err = $@) {
4290             eval { qemu_netdevdel($vmid, $deviceid); };
4291             warn $@ if $@;
4292             die $err;
4293         }
4294     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4295         my $bridgeid = $2;
4296         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4297         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4298
4299         qemu_deviceadd($vmid, $devicefull);
4300         qemu_deviceaddverify($vmid, $deviceid);
4301     } else {
4302         die "can't hotplug device '$deviceid'\n";
4303     }
4304
4305     return 1;
4306 }
4307
4308 # fixme: this should raise exceptions on error!
4309 sub vm_deviceunplug {
4310     my ($vmid, $conf, $deviceid) = @_;
4311
4312     my $devices_list = vm_devices_list($vmid);
4313     return 1 if !defined($devices_list->{$deviceid});
4314
4315     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4316     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4317
4318     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4319         qemu_devicedel($vmid, $deviceid);
4320     } elsif ($deviceid =~ m/^usb\d+$/) {
4321         die "usb hotplug currently not reliable\n";
4322         # when unplugging usb devices this way, there may be remaining usb
4323         # controllers/hubs so we disable it for now
4324         #qemu_devicedel($vmid, $deviceid);
4325         #qemu_devicedelverify($vmid, $deviceid);
4326     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4327         my $device = parse_drive($deviceid, $conf->{$deviceid});
4328
4329         qemu_devicedel($vmid, $deviceid);
4330         qemu_devicedelverify($vmid, $deviceid);
4331         qemu_drivedel($vmid, $deviceid);
4332         qemu_iothread_del($vmid, $deviceid, $device);
4333     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4334         qemu_devicedel($vmid, $deviceid);
4335         qemu_devicedelverify($vmid, $deviceid);
4336     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4337         my $device = parse_drive($deviceid, $conf->{$deviceid});
4338
4339         qemu_devicedel($vmid, $deviceid);
4340         qemu_devicedelverify($vmid, $deviceid);
4341         qemu_drivedel($vmid, $deviceid);
4342         qemu_deletescsihw($conf, $vmid, $deviceid);
4343
4344         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4345             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4346     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4347         qemu_devicedel($vmid, $deviceid);
4348         qemu_devicedelverify($vmid, $deviceid);
4349         qemu_netdevdel($vmid, $deviceid);
4350     } else {
4351         die "can't unplug device '$deviceid'\n";
4352     }
4353
4354     return 1;
4355 }
4356
4357 sub qemu_deviceadd {
4358     my ($vmid, $devicefull) = @_;
4359
4360     $devicefull = "driver=".$devicefull;
4361     my %options =  split(/[=,]/, $devicefull);
4362
4363     mon_cmd($vmid, "device_add" , %options);
4364 }
4365
4366 sub qemu_devicedel {
4367     my ($vmid, $deviceid) = @_;
4368
4369     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4370 }
4371
4372 sub qemu_iothread_add {
4373     my ($vmid, $deviceid, $device) = @_;
4374
4375     if ($device->{iothread}) {
4376         my $iothreads = vm_iothreads_list($vmid);
4377         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4378     }
4379 }
4380
4381 sub qemu_iothread_del {
4382     my ($vmid, $deviceid, $device) = @_;
4383
4384     if ($device->{iothread}) {
4385         my $iothreads = vm_iothreads_list($vmid);
4386         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4387     }
4388 }
4389
4390 sub qemu_objectadd {
4391     my ($vmid, $objectid, $qomtype) = @_;
4392
4393     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4394
4395     return 1;
4396 }
4397
4398 sub qemu_objectdel {
4399     my ($vmid, $objectid) = @_;
4400
4401     mon_cmd($vmid, "object-del", id => $objectid);
4402
4403     return 1;
4404 }
4405
4406 sub qemu_driveadd {
4407     my ($storecfg, $vmid, $device) = @_;
4408
4409     my $kvmver = get_running_qemu_version($vmid);
4410     my $io_uring = min_version($kvmver, 6, 0);
4411     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4412     $drive =~ s/\\/\\\\/g;
4413     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4414
4415     # If the command succeeds qemu prints: "OK"
4416     return 1 if $ret =~ m/OK/s;
4417
4418     die "adding drive failed: $ret\n";
4419 }
4420
4421 sub qemu_drivedel {
4422     my ($vmid, $deviceid) = @_;
4423
4424     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4425     $ret =~ s/^\s+//;
4426
4427     return 1 if $ret eq "";
4428
4429     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4430     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4431
4432     die "deleting drive $deviceid failed : $ret\n";
4433 }
4434
4435 sub qemu_deviceaddverify {
4436     my ($vmid, $deviceid) = @_;
4437
4438     for (my $i = 0; $i <= 5; $i++) {
4439          my $devices_list = vm_devices_list($vmid);
4440          return 1 if defined($devices_list->{$deviceid});
4441          sleep 1;
4442     }
4443
4444     die "error on hotplug device '$deviceid'\n";
4445 }
4446
4447
4448 sub qemu_devicedelverify {
4449     my ($vmid, $deviceid) = @_;
4450
4451     # need to verify that the device is correctly removed as device_del
4452     # is async and empty return is not reliable
4453
4454     for (my $i = 0; $i <= 5; $i++) {
4455          my $devices_list = vm_devices_list($vmid);
4456          return 1 if !defined($devices_list->{$deviceid});
4457          sleep 1;
4458     }
4459
4460     die "error on hot-unplugging device '$deviceid'\n";
4461 }
4462
4463 sub qemu_findorcreatescsihw {
4464     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4465
4466     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4467
4468     my $scsihwid="$controller_prefix$controller";
4469     my $devices_list = vm_devices_list($vmid);
4470
4471     if (!defined($devices_list->{$scsihwid})) {
4472         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4473     }
4474
4475     return 1;
4476 }
4477
4478 sub qemu_deletescsihw {
4479     my ($conf, $vmid, $opt) = @_;
4480
4481     my $device = parse_drive($opt, $conf->{$opt});
4482
4483     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4484         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4485         return 1;
4486     }
4487
4488     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4489
4490     my $devices_list = vm_devices_list($vmid);
4491     foreach my $opt (keys %{$devices_list}) {
4492         if (is_valid_drivename($opt)) {
4493             my $drive = parse_drive($opt, $conf->{$opt});
4494             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4495                 return 1;
4496             }
4497         }
4498     }
4499
4500     my $scsihwid="scsihw$controller";
4501
4502     vm_deviceunplug($vmid, $conf, $scsihwid);
4503
4504     return 1;
4505 }
4506
4507 sub qemu_add_pci_bridge {
4508     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4509
4510     my $bridges = {};
4511
4512     my $bridgeid;
4513
4514     print_pci_addr($device, $bridges, $arch, $machine_type);
4515
4516     while (my ($k, $v) = each %$bridges) {
4517         $bridgeid = $k;
4518     }
4519     return 1 if !defined($bridgeid) || $bridgeid < 1;
4520
4521     my $bridge = "pci.$bridgeid";
4522     my $devices_list = vm_devices_list($vmid);
4523
4524     if (!defined($devices_list->{$bridge})) {
4525         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4526     }
4527
4528     return 1;
4529 }
4530
4531 sub qemu_set_link_status {
4532     my ($vmid, $device, $up) = @_;
4533
4534     mon_cmd($vmid, "set_link", name => $device,
4535                up => $up ? JSON::true : JSON::false);
4536 }
4537
4538 sub qemu_netdevadd {
4539     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4540
4541     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4542     my %options =  split(/[=,]/, $netdev);
4543
4544     if (defined(my $vhost = $options{vhost})) {
4545         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4546     }
4547
4548     if (defined(my $queues = $options{queues})) {
4549         $options{queues} = $queues + 0;
4550     }
4551
4552     mon_cmd($vmid, "netdev_add",  %options);
4553     return 1;
4554 }
4555
4556 sub qemu_netdevdel {
4557     my ($vmid, $deviceid) = @_;
4558
4559     mon_cmd($vmid, "netdev_del", id => $deviceid);
4560 }
4561
4562 sub qemu_usb_hotplug {
4563     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4564
4565     return if !$device;
4566
4567     # remove the old one first
4568     vm_deviceunplug($vmid, $conf, $deviceid);
4569
4570     # check if xhci controller is necessary and available
4571     if ($device->{usb3}) {
4572
4573         my $devicelist = vm_devices_list($vmid);
4574
4575         if (!$devicelist->{xhci}) {
4576             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4577             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4578         }
4579     }
4580     my $d = parse_usb_device($device->{host});
4581     $d->{usb3} = $device->{usb3};
4582
4583     # add the new one
4584     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4585 }
4586
4587 sub qemu_cpu_hotplug {
4588     my ($vmid, $conf, $vcpus) = @_;
4589
4590     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4591
4592     my $sockets = 1;
4593     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4594     $sockets = $conf->{sockets} if  $conf->{sockets};
4595     my $cores = $conf->{cores} || 1;
4596     my $maxcpus = $sockets * $cores;
4597
4598     $vcpus = $maxcpus if !$vcpus;
4599
4600     die "you can't add more vcpus than maxcpus\n"
4601         if $vcpus > $maxcpus;
4602
4603     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4604
4605     if ($vcpus < $currentvcpus) {
4606
4607         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4608
4609             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4610                 qemu_devicedel($vmid, "cpu$i");
4611                 my $retry = 0;
4612                 my $currentrunningvcpus = undef;
4613                 while (1) {
4614                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4615                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4616                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4617                     $retry++;
4618                     sleep 1;
4619                 }
4620                 #update conf after each succesfull cpu unplug
4621                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4622                 PVE::QemuConfig->write_config($vmid, $conf);
4623             }
4624         } else {
4625             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4626         }
4627
4628         return;
4629     }
4630
4631     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4632     die "vcpus in running vm does not match its configuration\n"
4633         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4634
4635     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4636
4637         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4638             my $cpustr = print_cpu_device($conf, $i);
4639             qemu_deviceadd($vmid, $cpustr);
4640
4641             my $retry = 0;
4642             my $currentrunningvcpus = undef;
4643             while (1) {
4644                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4645                 last if scalar(@{$currentrunningvcpus}) == $i;
4646                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4647                 sleep 1;
4648                 $retry++;
4649             }
4650             #update conf after each succesfull cpu hotplug
4651             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4652             PVE::QemuConfig->write_config($vmid, $conf);
4653         }
4654     } else {
4655
4656         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4657             mon_cmd($vmid, "cpu-add", id => int($i));
4658         }
4659     }
4660 }
4661
4662 sub qemu_block_set_io_throttle {
4663     my ($vmid, $deviceid,
4664         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4665         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4666         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4667         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4668
4669     return if !check_running($vmid) ;
4670
4671     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4672         bps => int($bps),
4673         bps_rd => int($bps_rd),
4674         bps_wr => int($bps_wr),
4675         iops => int($iops),
4676         iops_rd => int($iops_rd),
4677         iops_wr => int($iops_wr),
4678         bps_max => int($bps_max),
4679         bps_rd_max => int($bps_rd_max),
4680         bps_wr_max => int($bps_wr_max),
4681         iops_max => int($iops_max),
4682         iops_rd_max => int($iops_rd_max),
4683         iops_wr_max => int($iops_wr_max),
4684         bps_max_length => int($bps_max_length),
4685         bps_rd_max_length => int($bps_rd_max_length),
4686         bps_wr_max_length => int($bps_wr_max_length),
4687         iops_max_length => int($iops_max_length),
4688         iops_rd_max_length => int($iops_rd_max_length),
4689         iops_wr_max_length => int($iops_wr_max_length),
4690     );
4691
4692 }
4693
4694 sub qemu_block_resize {
4695     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4696
4697     my $running = check_running($vmid);
4698
4699     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4700
4701     return if !$running;
4702
4703     my $padding = (1024 - $size % 1024) % 1024;
4704     $size = $size + $padding;
4705
4706     mon_cmd(
4707         $vmid,
4708         "block_resize",
4709         device => $deviceid,
4710         size => int($size),
4711         timeout => 60,
4712     );
4713 }
4714
4715 sub qemu_volume_snapshot {
4716     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4717
4718     my $running = check_running($vmid);
4719
4720     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4721         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4722     } else {
4723         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4724     }
4725 }
4726
4727 sub qemu_volume_snapshot_delete {
4728     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4729
4730     my $running = check_running($vmid);
4731
4732     if($running) {
4733
4734         $running = undef;
4735         my $conf = PVE::QemuConfig->load_config($vmid);
4736         PVE::QemuConfig->foreach_volume($conf, sub {
4737             my ($ds, $drive) = @_;
4738             $running = 1 if $drive->{file} eq $volid;
4739         });
4740     }
4741
4742     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4743         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4744     } else {
4745         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4746     }
4747 }
4748
4749 sub set_migration_caps {
4750     my ($vmid, $savevm) = @_;
4751
4752     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4753
4754     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4755     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4756
4757     my $cap_ref = [];
4758
4759     my $enabled_cap = {
4760         "auto-converge" => 1,
4761         "xbzrle" => 1,
4762         "x-rdma-pin-all" => 0,
4763         "zero-blocks" => 0,
4764         "compress" => 0,
4765         "dirty-bitmaps" => $dirty_bitmaps,
4766     };
4767
4768     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4769
4770     for my $supported_capability (@$supported_capabilities) {
4771         push @$cap_ref, {
4772             capability => $supported_capability->{capability},
4773             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4774         };
4775     }
4776
4777     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4778 }
4779
4780 sub foreach_volid {
4781     my ($conf, $func, @param) = @_;
4782
4783     my $volhash = {};
4784
4785     my $test_volid = sub {
4786         my ($key, $drive, $snapname) = @_;
4787
4788         my $volid = $drive->{file};
4789         return if !$volid;
4790
4791         $volhash->{$volid}->{cdrom} //= 1;
4792         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4793
4794         my $replicate = $drive->{replicate} // 1;
4795         $volhash->{$volid}->{replicate} //= 0;
4796         $volhash->{$volid}->{replicate} = 1 if $replicate;
4797
4798         $volhash->{$volid}->{shared} //= 0;
4799         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4800
4801         $volhash->{$volid}->{referenced_in_config} //= 0;
4802         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4803
4804         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4805             if defined($snapname);
4806
4807         my $size = $drive->{size};
4808         $volhash->{$volid}->{size} //= $size if $size;
4809
4810         $volhash->{$volid}->{is_vmstate} //= 0;
4811         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4812
4813         $volhash->{$volid}->{is_tpmstate} //= 0;
4814         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4815
4816         $volhash->{$volid}->{is_unused} //= 0;
4817         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4818
4819         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4820     };
4821
4822     my $include_opts = {
4823         extra_keys => ['vmstate'],
4824         include_unused => 1,
4825     };
4826
4827     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4828     foreach my $snapname (keys %{$conf->{snapshots}}) {
4829         my $snap = $conf->{snapshots}->{$snapname};
4830         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4831     }
4832
4833     foreach my $volid (keys %$volhash) {
4834         &$func($volid, $volhash->{$volid}, @param);
4835     }
4836 }
4837
4838 my $fast_plug_option = {
4839     'lock' => 1,
4840     'name' => 1,
4841     'onboot' => 1,
4842     'shares' => 1,
4843     'startup' => 1,
4844     'description' => 1,
4845     'protection' => 1,
4846     'vmstatestorage' => 1,
4847     'hookscript' => 1,
4848     'tags' => 1,
4849 };
4850
4851 for my $opt (keys %$confdesc_cloudinit) {
4852     $fast_plug_option->{$opt} = 1;
4853 };
4854
4855 # hotplug changes in [PENDING]
4856 # $selection hash can be used to only apply specified options, for
4857 # example: { cores => 1 } (only apply changed 'cores')
4858 # $errors ref is used to return error messages
4859 sub vmconfig_hotplug_pending {
4860     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4861
4862     my $defaults = load_defaults();
4863     my $arch = get_vm_arch($conf);
4864     my $machine_type = get_vm_machine($conf, undef, $arch);
4865
4866     # commit values which do not have any impact on running VM first
4867     # Note: those option cannot raise errors, we we do not care about
4868     # $selection and always apply them.
4869
4870     my $add_error = sub {
4871         my ($opt, $msg) = @_;
4872         $errors->{$opt} = "hotplug problem - $msg";
4873     };
4874
4875     my $changes = 0;
4876     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4877         if ($fast_plug_option->{$opt}) {
4878             $conf->{$opt} = $conf->{pending}->{$opt};
4879             delete $conf->{pending}->{$opt};
4880             $changes = 1;
4881         }
4882     }
4883
4884     if ($changes) {
4885         PVE::QemuConfig->write_config($vmid, $conf);
4886     }
4887
4888     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4889
4890     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4891     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4892     foreach my $opt (sort keys %$pending_delete_hash) {
4893         next if $selection && !$selection->{$opt};
4894         my $force = $pending_delete_hash->{$opt}->{force};
4895         eval {
4896             if ($opt eq 'hotplug') {
4897                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4898             } elsif ($opt eq 'tablet') {
4899                 die "skip\n" if !$hotplug_features->{usb};
4900                 if ($defaults->{tablet}) {
4901                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4902                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4903                         if $arch eq 'aarch64';
4904                 } else {
4905                     vm_deviceunplug($vmid, $conf, 'tablet');
4906                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4907                 }
4908             } elsif ($opt =~ m/^usb\d+/) {
4909                 die "skip\n";
4910                 # since we cannot reliably hot unplug usb devices we are disabling it
4911                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4912                 #vm_deviceunplug($vmid, $conf, $opt);
4913             } elsif ($opt eq 'vcpus') {
4914                 die "skip\n" if !$hotplug_features->{cpu};
4915                 qemu_cpu_hotplug($vmid, $conf, undef);
4916             } elsif ($opt eq 'balloon') {
4917                 # enable balloon device is not hotpluggable
4918                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4919                 # here we reset the ballooning value to memory
4920                 my $balloon = $conf->{memory} || $defaults->{memory};
4921                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4922             } elsif ($fast_plug_option->{$opt}) {
4923                 # do nothing
4924             } elsif ($opt =~ m/^net(\d+)$/) {
4925                 die "skip\n" if !$hotplug_features->{network};
4926                 vm_deviceunplug($vmid, $conf, $opt);
4927             } elsif (is_valid_drivename($opt)) {
4928                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4929                 vm_deviceunplug($vmid, $conf, $opt);
4930                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4931             } elsif ($opt =~ m/^memory$/) {
4932                 die "skip\n" if !$hotplug_features->{memory};
4933                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4934             } elsif ($opt eq 'cpuunits') {
4935                 $cgroup->change_cpu_shares(undef);
4936             } elsif ($opt eq 'cpulimit') {
4937                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4938             } else {
4939                 die "skip\n";
4940             }
4941         };
4942         if (my $err = $@) {
4943             &$add_error($opt, $err) if $err ne "skip\n";
4944         } else {
4945             delete $conf->{$opt};
4946             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4947         }
4948     }
4949
4950     foreach my $opt (keys %{$conf->{pending}}) {
4951         next if $selection && !$selection->{$opt};
4952         my $value = $conf->{pending}->{$opt};
4953         eval {
4954             if ($opt eq 'hotplug') {
4955                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4956             } elsif ($opt eq 'tablet') {
4957                 die "skip\n" if !$hotplug_features->{usb};
4958                 if ($value == 1) {
4959                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4960                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4961                         if $arch eq 'aarch64';
4962                 } elsif ($value == 0) {
4963                     vm_deviceunplug($vmid, $conf, 'tablet');
4964                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4965                 }
4966             } elsif ($opt =~ m/^usb\d+$/) {
4967                 die "skip\n";
4968                 # since we cannot reliably hot unplug usb devices we disable it for now
4969                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4970                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4971                 #die "skip\n" if !$d;
4972                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4973             } elsif ($opt eq 'vcpus') {
4974                 die "skip\n" if !$hotplug_features->{cpu};
4975                 qemu_cpu_hotplug($vmid, $conf, $value);
4976             } elsif ($opt eq 'balloon') {
4977                 # enable/disable balloning device is not hotpluggable
4978                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4979                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4980                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4981
4982                 # allow manual ballooning if shares is set to zero
4983                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4984                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4985                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4986                 }
4987             } elsif ($opt =~ m/^net(\d+)$/) {
4988                 # some changes can be done without hotplug
4989                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4990                                     $vmid, $opt, $value, $arch, $machine_type);
4991             } elsif (is_valid_drivename($opt)) {
4992                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4993                 # some changes can be done without hotplug
4994                 my $drive = parse_drive($opt, $value);
4995                 if (drive_is_cloudinit($drive)) {
4996                     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4997                 }
4998                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4999                                      $vmid, $opt, $value, $arch, $machine_type);
5000             } elsif ($opt =~ m/^memory$/) { #dimms
5001                 die "skip\n" if !$hotplug_features->{memory};
5002                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5003             } elsif ($opt eq 'cpuunits') {
5004                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5005                 $cgroup->change_cpu_shares($new_cpuunits);
5006             } elsif ($opt eq 'cpulimit') {
5007                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5008                 $cgroup->change_cpu_quota($cpulimit, 100000);
5009             } elsif ($opt eq 'agent') {
5010                 vmconfig_update_agent($conf, $opt, $value);
5011             } else {
5012                 die "skip\n";  # skip non-hot-pluggable options
5013             }
5014         };
5015         if (my $err = $@) {
5016             &$add_error($opt, $err) if $err ne "skip\n";
5017         } else {
5018             $conf->{$opt} = $value;
5019             delete $conf->{pending}->{$opt};
5020         }
5021     }
5022     PVE::QemuConfig->write_config($vmid, $conf);
5023
5024     if($hotplug_features->{cloudinit}) {
5025         my $pending = PVE::QemuServer::Cloudinit::get_pending_config($conf, $vmid);
5026         my $regenerate = undef;
5027         for my $item (@$pending) {
5028             $regenerate = 1 if defined($item->{delete}) or defined($item->{pending});
5029         }
5030         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid) if $regenerate;
5031     }
5032 }
5033
5034 sub try_deallocate_drive {
5035     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5036
5037     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5038         my $volid = $drive->{file};
5039         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5040             my $sid = PVE::Storage::parse_volume_id($volid);
5041             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5042
5043             # check if the disk is really unused
5044             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5045                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5046             PVE::Storage::vdisk_free($storecfg, $volid);
5047             return 1;
5048         } else {
5049             # If vm is not owner of this disk remove from config
5050             return 1;
5051         }
5052     }
5053
5054     return;
5055 }
5056
5057 sub vmconfig_delete_or_detach_drive {
5058     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5059
5060     my $drive = parse_drive($opt, $conf->{$opt});
5061
5062     my $rpcenv = PVE::RPCEnvironment::get();
5063     my $authuser = $rpcenv->get_user();
5064
5065     if ($force) {
5066         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5067         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5068     } else {
5069         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5070     }
5071 }
5072
5073
5074
5075 sub vmconfig_apply_pending {
5076     my ($vmid, $conf, $storecfg, $errors) = @_;
5077
5078     return if !scalar(keys %{$conf->{pending}});
5079
5080     my $add_apply_error = sub {
5081         my ($opt, $msg) = @_;
5082         my $err_msg = "unable to apply pending change $opt : $msg";
5083         $errors->{$opt} = $err_msg;
5084         warn $err_msg;
5085     };
5086
5087     # cold plug
5088
5089     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5090     foreach my $opt (sort keys %$pending_delete_hash) {
5091         my $force = $pending_delete_hash->{$opt}->{force};
5092         eval {
5093             if ($opt =~ m/^unused/) {
5094                 die "internal error";
5095             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5096                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5097             }
5098         };
5099         if (my $err = $@) {
5100             $add_apply_error->($opt, $err);
5101         } else {
5102             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5103             delete $conf->{$opt};
5104         }
5105     }
5106
5107     PVE::QemuConfig->cleanup_pending($conf);
5108
5109     my $generate_cloudnit = undef;
5110
5111     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5112         next if $opt eq 'delete'; # just to be sure
5113         eval {
5114             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5115                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5116             }
5117         };
5118         if (my $err = $@) {
5119             $add_apply_error->($opt, $err);
5120         } else {
5121
5122             if (is_valid_drivename($opt)) {
5123                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5124                 $generate_cloudnit = 1 if drive_is_cloudinit($drive);
5125             }
5126
5127             $conf->{$opt} = delete $conf->{pending}->{$opt};
5128         }
5129     }
5130
5131     # write all changes at once to avoid unnecessary i/o
5132     PVE::QemuConfig->write_config($vmid, $conf);
5133     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if $generate_cloudnit;
5134 }
5135
5136 sub vmconfig_update_net {
5137     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5138
5139     my $newnet = parse_net($value);
5140
5141     if ($conf->{$opt}) {
5142         my $oldnet = parse_net($conf->{$opt});
5143
5144         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5145             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5146             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5147             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5148
5149             # for non online change, we try to hot-unplug
5150             die "skip\n" if !$hotplug;
5151             vm_deviceunplug($vmid, $conf, $opt);
5152         } else {
5153
5154             die "internal error" if $opt !~ m/net(\d+)/;
5155             my $iface = "tap${vmid}i$1";
5156
5157             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5158                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5159                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5160                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5161                 PVE::Network::tap_unplug($iface);
5162
5163                 if ($have_sdn) {
5164                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5165                 } else {
5166                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5167                 }
5168             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5169                 # Rate can be applied on its own but any change above needs to
5170                 # include the rate in tap_plug since OVS resets everything.
5171                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5172             }
5173
5174             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5175                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5176             }
5177
5178             return 1;
5179         }
5180     }
5181
5182     if ($hotplug) {
5183         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5184     } else {
5185         die "skip\n";
5186     }
5187 }
5188
5189 sub vmconfig_update_agent {
5190     my ($conf, $opt, $value) = @_;
5191
5192     die "skip\n" if !$conf->{$opt};
5193
5194     my $hotplug_options = { fstrim_cloned_disks => 1 };
5195
5196     my $old_agent = parse_guest_agent($conf);
5197     my $agent = parse_guest_agent({$opt => $value});
5198
5199     for my $option (keys %$agent) { # added/changed options
5200         next if defined($hotplug_options->{$option});
5201         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5202     }
5203
5204     for my $option (keys %$old_agent) { # removed options
5205         next if defined($hotplug_options->{$option});
5206         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5207     }
5208
5209     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5210 }
5211
5212 sub vmconfig_update_disk {
5213     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5214
5215     my $drive = parse_drive($opt, $value);
5216
5217     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5218         my $media = $drive->{media} || 'disk';
5219         my $oldmedia = $old_drive->{media} || 'disk';
5220         die "unable to change media type\n" if $media ne $oldmedia;
5221
5222         if (!drive_is_cdrom($old_drive)) {
5223
5224             if ($drive->{file} ne $old_drive->{file}) {
5225
5226                 die "skip\n" if !$hotplug;
5227
5228                 # unplug and register as unused
5229                 vm_deviceunplug($vmid, $conf, $opt);
5230                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5231
5232             } else {
5233                 # update existing disk
5234
5235                 # skip non hotpluggable value
5236                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5237                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5238                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5239                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5240                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5241                     die "skip\n";
5242                 }
5243
5244                 # apply throttle
5245                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5246                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5247                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5248                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5249                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5250                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5251                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5252                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5253                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5254                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5255                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5256                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5257                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5258                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5259                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5260                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5261                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5262                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5263
5264                     qemu_block_set_io_throttle(
5265                         $vmid,"drive-$opt",
5266                         ($drive->{mbps} || 0)*1024*1024,
5267                         ($drive->{mbps_rd} || 0)*1024*1024,
5268                         ($drive->{mbps_wr} || 0)*1024*1024,
5269                          $drive->{iops} || 0,
5270                          $drive->{iops_rd} || 0,
5271                          $drive->{iops_wr} || 0,
5272                         ($drive->{mbps_max} || 0)*1024*1024,
5273                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5274                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5275                          $drive->{iops_max} || 0,
5276                          $drive->{iops_rd_max} || 0,
5277                          $drive->{iops_wr_max} || 0,
5278                          $drive->{bps_max_length} || 1,
5279                          $drive->{bps_rd_max_length} || 1,
5280                          $drive->{bps_wr_max_length} || 1,
5281                          $drive->{iops_max_length} || 1,
5282                          $drive->{iops_rd_max_length} || 1,
5283                          $drive->{iops_wr_max_length} || 1,
5284                     );
5285
5286                 }
5287
5288                 return 1;
5289             }
5290
5291         } else { # cdrom
5292
5293             if ($drive->{file} eq 'none') {
5294                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5295                 if (drive_is_cloudinit($old_drive)) {
5296                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5297                 }
5298             } else {
5299                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5300
5301                 # force eject if locked
5302                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5303
5304                 if ($path) {
5305                     mon_cmd($vmid, "blockdev-change-medium",
5306                         id => "$opt", filename => "$path");
5307                 }
5308             }
5309
5310             return 1;
5311         }
5312     }
5313
5314     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5315     # hotplug new disks
5316     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5317     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5318 }
5319
5320 sub vmconfig_update_cloudinit_drive {
5321     my ($storecfg, $conf, $vmid) = @_;
5322
5323     my $cloudinit_ds = undef;
5324     my $cloudinit_drive = undef;
5325
5326     PVE::QemuConfig->foreach_volume($conf, sub {
5327         my ($ds, $drive) = @_;
5328         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5329             $cloudinit_ds = $ds;
5330             $cloudinit_drive = $drive;
5331         }
5332     });
5333
5334     return if !$cloudinit_drive;
5335
5336     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5337     my $running = PVE::QemuServer::check_running($vmid);
5338
5339     if ($running) {
5340         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5341         if ($path) {
5342             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5343             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5344         }
5345     }
5346 }
5347
5348 # called in locked context by incoming migration
5349 sub vm_migrate_get_nbd_disks {
5350     my ($storecfg, $conf, $replicated_volumes) = @_;
5351
5352     my $local_volumes = {};
5353     PVE::QemuConfig->foreach_volume($conf, sub {
5354         my ($ds, $drive) = @_;
5355
5356         return if drive_is_cdrom($drive);
5357         return if $ds eq 'tpmstate0';
5358
5359         my $volid = $drive->{file};
5360
5361         return if !$volid;
5362
5363         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5364
5365         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5366         return if $scfg->{shared};
5367
5368         # replicated disks re-use existing state via bitmap
5369         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5370         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5371     });
5372     return $local_volumes;
5373 }
5374
5375 # called in locked context by incoming migration
5376 sub vm_migrate_alloc_nbd_disks {
5377     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5378
5379     my $nbd = {};
5380     foreach my $opt (sort keys %$source_volumes) {
5381         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5382
5383         if ($use_existing) {
5384             $nbd->{$opt}->{drivestr} = print_drive($drive);
5385             $nbd->{$opt}->{volid} = $volid;
5386             $nbd->{$opt}->{replicated} = 1;
5387             next;
5388         }
5389
5390         # storage mapping + volname = regular migration
5391         # storage mapping + format = remote migration
5392         # order of precedence, filtered by whether storage supports it:
5393         # 1. explicit requested format
5394         # 2. format of current volume
5395         # 3. default format of storage
5396         if (!$storagemap->{identity}) {
5397             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5398             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5399             if (!$format || !grep { $format eq $_ } @$validFormats) {
5400                 if ($volname) {
5401                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5402                     my $fileFormat = qemu_img_format($scfg, $volname);
5403                     $format = $fileFormat
5404                         if grep { $fileFormat eq $_ } @$validFormats;
5405                 }
5406                 $format //= $defFormat;
5407             }
5408         } else {
5409             # can't happen for remote migration, so $volname is always defined
5410             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5411             $format = qemu_img_format($scfg, $volname);
5412         }
5413
5414         my $size = $drive->{size} / 1024;
5415         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5416         my $newdrive = $drive;
5417         $newdrive->{format} = $format;
5418         $newdrive->{file} = $newvolid;
5419         my $drivestr = print_drive($newdrive);
5420         $nbd->{$opt}->{drivestr} = $drivestr;
5421         $nbd->{$opt}->{volid} = $newvolid;
5422     }
5423
5424     return $nbd;
5425 }
5426
5427 # see vm_start_nolock for parameters, additionally:
5428 # migrate_opts:
5429 #   storagemap = parsed storage map for allocating NBD disks
5430 sub vm_start {
5431     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5432
5433     return PVE::QemuConfig->lock_config($vmid, sub {
5434         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5435
5436         die "you can't start a vm if it's a template\n"
5437             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5438
5439         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5440         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5441
5442         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5443
5444         if ($has_backup_lock && $running) {
5445             # a backup is currently running, attempt to start the guest in the
5446             # existing QEMU instance
5447             return vm_resume($vmid);
5448         }
5449
5450         PVE::QemuConfig->check_lock($conf)
5451             if !($params->{skiplock} || $has_suspended_lock);
5452
5453         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5454
5455         die "VM $vmid already running\n" if $running;
5456
5457         if (my $storagemap = $migrate_opts->{storagemap}) {
5458             my $replicated = $migrate_opts->{replicated_volumes};
5459             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5460             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5461
5462             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5463                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5464             }
5465         }
5466
5467         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5468     });
5469 }
5470
5471
5472 # params:
5473 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5474 #   skiplock => 0/1, skip checking for config lock
5475 #   skiptemplate => 0/1, skip checking whether VM is template
5476 #   forcemachine => to force Qemu machine (rollback/migration)
5477 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5478 #   timeout => in seconds
5479 #   paused => start VM in paused state (backup)
5480 #   resume => resume from hibernation
5481 #   pbs-backing => {
5482 #      sata0 => {
5483 #         repository
5484 #         snapshot
5485 #         keyfile
5486 #         archive
5487 #      },
5488 #      virtio2 => ...
5489 #   }
5490 # migrate_opts:
5491 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5492 #   migratedfrom => source node
5493 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5494 #   network => CIDR of migration network
5495 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5496 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5497 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5498 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5499 #       contained in config
5500 sub vm_start_nolock {
5501     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5502
5503     my $statefile = $params->{statefile};
5504     my $resume = $params->{resume};
5505
5506     my $migratedfrom = $migrate_opts->{migratedfrom};
5507     my $migration_type = $migrate_opts->{type};
5508
5509     my $res = {};
5510
5511     # clean up leftover reboot request files
5512     eval { clear_reboot_request($vmid); };
5513     warn $@ if $@;
5514
5515     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5516         vmconfig_apply_pending($vmid, $conf, $storecfg);
5517         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5518     }
5519
5520     # don't regenerate the ISO if the VM is started as part of a live migration
5521     # this way we can reuse the old ISO with the correct config
5522     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5523
5524     # override offline migrated volumes, conf is out of date still
5525     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5526         for my $key (sort keys $offline_volumes->%*) {
5527             my $parsed = parse_drive($key, $conf->{$key});
5528             $parsed->{file} = $offline_volumes->{$key};
5529             $conf->{$key} = print_drive($parsed);
5530         }
5531     }
5532
5533     my $defaults = load_defaults();
5534
5535     # set environment variable useful inside network script
5536     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5537
5538     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5539
5540     my $forcemachine = $params->{forcemachine};
5541     my $forcecpu = $params->{forcecpu};
5542     if ($resume) {
5543         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5544         $forcemachine = $conf->{runningmachine};
5545         $forcecpu = $conf->{runningcpu};
5546         print "Resuming suspended VM\n";
5547     }
5548
5549     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5550         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5551
5552     my $migration_ip;
5553     my $get_migration_ip = sub {
5554         my ($nodename) = @_;
5555
5556         return $migration_ip if defined($migration_ip);
5557
5558         my $cidr = $migrate_opts->{network};
5559
5560         if (!defined($cidr)) {
5561             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5562             $cidr = $dc_conf->{migration}->{network};
5563         }
5564
5565         if (defined($cidr)) {
5566             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5567
5568             die "could not get IP: no address configured on local " .
5569                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5570
5571             die "could not get IP: multiple addresses configured on local " .
5572                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5573
5574             $migration_ip = @$ips[0];
5575         }
5576
5577         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5578             if !defined($migration_ip);
5579
5580         return $migration_ip;
5581     };
5582
5583     my $migrate_uri;
5584     if ($statefile) {
5585         if ($statefile eq 'tcp') {
5586             my $localip = "localhost";
5587             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5588             my $nodename = nodename();
5589
5590             if (!defined($migration_type)) {
5591                 if (defined($datacenterconf->{migration}->{type})) {
5592                     $migration_type = $datacenterconf->{migration}->{type};
5593                 } else {
5594                     $migration_type = 'secure';
5595                 }
5596             }
5597
5598             if ($migration_type eq 'insecure') {
5599                 $localip = $get_migration_ip->($nodename);
5600                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5601             }
5602
5603             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5604             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5605             $migrate_uri = "tcp:${localip}:${migrate_port}";
5606             push @$cmd, '-incoming', $migrate_uri;
5607             push @$cmd, '-S';
5608
5609         } elsif ($statefile eq 'unix') {
5610             # should be default for secure migrations as a ssh TCP forward
5611             # tunnel is not deterministic reliable ready and fails regurarly
5612             # to set up in time, so use UNIX socket forwards
5613             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5614             unlink $socket_addr;
5615
5616             $migrate_uri = "unix:$socket_addr";
5617
5618             push @$cmd, '-incoming', $migrate_uri;
5619             push @$cmd, '-S';
5620
5621         } elsif (-e $statefile) {
5622             push @$cmd, '-loadstate', $statefile;
5623         } else {
5624             my $statepath = PVE::Storage::path($storecfg, $statefile);
5625             push @$vollist, $statefile;
5626             push @$cmd, '-loadstate', $statepath;
5627         }
5628     } elsif ($params->{paused}) {
5629         push @$cmd, '-S';
5630     }
5631
5632     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5633
5634     my $pci_devices = {}; # host pci devices
5635     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5636         my $dev = $conf->{"hostpci$i"} or next;
5637         $pci_devices->{$i} = parse_hostpci($dev);
5638     }
5639
5640     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5641     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5642
5643     # map to a flat list of pci ids
5644     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5645
5646     # reserve all PCI IDs before actually doing anything with them
5647     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5648
5649     eval {
5650         my $uuid;
5651         for my $id (sort keys %$pci_devices) {
5652             my $d = $pci_devices->{$id};
5653             for my $dev ($d->{pciid}->@*) {
5654                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5655
5656                 # nvidia grid needs the uuid of the mdev as qemu parameter
5657                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5658                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5659                 }
5660             }
5661         }
5662         push @$cmd, '-uuid', $uuid if defined($uuid);
5663     };
5664     if (my $err = $@) {
5665         eval { cleanup_pci_devices($vmid, $conf) };
5666         warn $@ if $@;
5667         die $err;
5668     }
5669
5670     PVE::Storage::activate_volumes($storecfg, $vollist);
5671
5672     eval {
5673         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5674     };
5675     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5676     # timeout should be more than enough here...
5677     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5678
5679     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5680
5681     my %run_params = (
5682         timeout => $statefile ? undef : $start_timeout,
5683         umask => 0077,
5684         noerr => 1,
5685     );
5686
5687     # when migrating, prefix QEMU output so other side can pick up any
5688     # errors that might occur and show the user
5689     if ($migratedfrom) {
5690         $run_params{quiet} = 1;
5691         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5692     }
5693
5694     my %systemd_properties = (
5695         Slice => 'qemu.slice',
5696         KillMode => 'process',
5697         SendSIGKILL => 0,
5698         TimeoutStopUSec => ULONG_MAX, # infinity
5699     );
5700
5701     if (PVE::CGroup::cgroup_mode() == 2) {
5702         $systemd_properties{CPUWeight} = $cpuunits;
5703     } else {
5704         $systemd_properties{CPUShares} = $cpuunits;
5705     }
5706
5707     if (my $cpulimit = $conf->{cpulimit}) {
5708         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5709     }
5710     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5711
5712     my $run_qemu = sub {
5713         PVE::Tools::run_fork sub {
5714             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5715
5716             my $tpmpid;
5717             if (my $tpm = $conf->{tpmstate0}) {
5718                 # start the TPM emulator so QEMU can connect on start
5719                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5720             }
5721
5722             my $exitcode = run_command($cmd, %run_params);
5723             if ($exitcode) {
5724                 if ($tpmpid) {
5725                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5726                     kill 'TERM', $tpmpid;
5727                 }
5728                 die "QEMU exited with code $exitcode\n";
5729             }
5730         };
5731     };
5732
5733     if ($conf->{hugepages}) {
5734
5735         my $code = sub {
5736             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5737             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5738
5739             PVE::QemuServer::Memory::hugepages_mount();
5740             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5741
5742             eval { $run_qemu->() };
5743             if (my $err = $@) {
5744                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5745                     if !$conf->{keephugepages};
5746                 die $err;
5747             }
5748
5749             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5750                 if !$conf->{keephugepages};
5751         };
5752         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5753
5754     } else {
5755         eval { $run_qemu->() };
5756     }
5757
5758     if (my $err = $@) {
5759         # deactivate volumes if start fails
5760         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5761         warn $@ if $@;
5762         eval { cleanup_pci_devices($vmid, $conf) };
5763         warn $@ if $@;
5764
5765         die "start failed: $err";
5766     }
5767
5768     # re-reserve all PCI IDs now that we can know the actual VM PID
5769     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5770     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5771     warn $@ if $@;
5772
5773     print "migration listens on $migrate_uri\n" if $migrate_uri;
5774     $res->{migrate_uri} = $migrate_uri;
5775
5776     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5777         eval { mon_cmd($vmid, "cont"); };
5778         warn $@ if $@;
5779     }
5780
5781     #start nbd server for storage migration
5782     if (my $nbd = $migrate_opts->{nbd}) {
5783         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5784
5785         my $migrate_storage_uri;
5786         # nbd_protocol_version > 0 for unix socket support
5787         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5788             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5789             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5790             $migrate_storage_uri = "nbd:unix:$socket_path";
5791         } else {
5792             my $nodename = nodename();
5793             my $localip = $get_migration_ip->($nodename);
5794             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5795             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5796
5797             mon_cmd($vmid, "nbd-server-start", addr => {
5798                 type => 'inet',
5799                 data => {
5800                     host => "${localip}",
5801                     port => "${storage_migrate_port}",
5802                 },
5803             });
5804             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5805             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5806         }
5807
5808         $res->{migrate_storage_uri} = $migrate_storage_uri;
5809
5810         foreach my $opt (sort keys %$nbd) {
5811             my $drivestr = $nbd->{$opt}->{drivestr};
5812             my $volid = $nbd->{$opt}->{volid};
5813             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5814             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5815             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5816             print "re-using replicated volume: $opt - $volid\n"
5817                 if $nbd->{$opt}->{replicated};
5818
5819             $res->{drives}->{$opt} = $nbd->{$opt};
5820             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5821         }
5822     }
5823
5824     if ($migratedfrom) {
5825         eval {
5826             set_migration_caps($vmid);
5827         };
5828         warn $@ if $@;
5829
5830         if ($spice_port) {
5831             print "spice listens on port $spice_port\n";
5832             $res->{spice_port} = $spice_port;
5833             if ($migrate_opts->{spice_ticket}) {
5834                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5835                     $migrate_opts->{spice_ticket});
5836                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5837             }
5838         }
5839
5840     } else {
5841         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5842             if !$statefile && $conf->{balloon};
5843
5844         foreach my $opt (keys %$conf) {
5845             next if $opt !~  m/^net\d+$/;
5846             my $nicconf = parse_net($conf->{$opt});
5847             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5848         }
5849     }
5850
5851     mon_cmd($vmid, 'qom-set',
5852                 path => "machine/peripheral/balloon0",
5853                 property => "guest-stats-polling-interval",
5854                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5855
5856     if ($resume) {
5857         print "Resumed VM, removing state\n";
5858         if (my $vmstate = $conf->{vmstate}) {
5859             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5860             PVE::Storage::vdisk_free($storecfg, $vmstate);
5861         }
5862         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5863         PVE::QemuConfig->write_config($vmid, $conf);
5864     }
5865
5866     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5867
5868     return $res;
5869 }
5870
5871 sub vm_commandline {
5872     my ($storecfg, $vmid, $snapname) = @_;
5873
5874     my $conf = PVE::QemuConfig->load_config($vmid);
5875
5876     my ($forcemachine, $forcecpu);
5877     if ($snapname) {
5878         my $snapshot = $conf->{snapshots}->{$snapname};
5879         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5880
5881         # check for machine or CPU overrides in snapshot
5882         $forcemachine = $snapshot->{runningmachine};
5883         $forcecpu = $snapshot->{runningcpu};
5884
5885         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5886
5887         $conf = $snapshot;
5888     }
5889
5890     my $defaults = load_defaults();
5891
5892     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5893
5894     return PVE::Tools::cmd2string($cmd);
5895 }
5896
5897 sub vm_reset {
5898     my ($vmid, $skiplock) = @_;
5899
5900     PVE::QemuConfig->lock_config($vmid, sub {
5901
5902         my $conf = PVE::QemuConfig->load_config($vmid);
5903
5904         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5905
5906         mon_cmd($vmid, "system_reset");
5907     });
5908 }
5909
5910 sub get_vm_volumes {
5911     my ($conf) = @_;
5912
5913     my $vollist = [];
5914     foreach_volid($conf, sub {
5915         my ($volid, $attr) = @_;
5916
5917         return if $volid =~ m|^/|;
5918
5919         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5920         return if !$sid;
5921
5922         push @$vollist, $volid;
5923     });
5924
5925     return $vollist;
5926 }
5927
5928 sub cleanup_pci_devices {
5929     my ($vmid, $conf) = @_;
5930
5931     foreach my $key (keys %$conf) {
5932         next if $key !~ m/^hostpci(\d+)$/;
5933         my $hostpciindex = $1;
5934         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5935         my $d = parse_hostpci($conf->{$key});
5936         if ($d->{mdev}) {
5937             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
5938             # don't want to break ABI just for this two liner
5939             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
5940             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
5941         }
5942     }
5943     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
5944 }
5945
5946 sub vm_stop_cleanup {
5947     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5948
5949     eval {
5950
5951         if (!$keepActive) {
5952             my $vollist = get_vm_volumes($conf);
5953             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5954
5955             if (my $tpmdrive = $conf->{tpmstate0}) {
5956                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5957                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5958                 if ($storeid) {
5959                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5960                 }
5961             }
5962         }
5963
5964         foreach my $ext (qw(mon qmp pid vnc qga)) {
5965             unlink "/var/run/qemu-server/${vmid}.$ext";
5966         }
5967
5968         if ($conf->{ivshmem}) {
5969             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5970             # just delete it for now, VMs which have this already open do not
5971             # are affected, but new VMs will get a separated one. If this
5972             # becomes an issue we either add some sort of ref-counting or just
5973             # add a "don't delete on stop" flag to the ivshmem format.
5974             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5975         }
5976
5977         cleanup_pci_devices($vmid, $conf);
5978
5979         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5980     };
5981     warn $@ if $@; # avoid errors - just warn
5982 }
5983
5984 # call only in locked context
5985 sub _do_vm_stop {
5986     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5987
5988     my $pid = check_running($vmid, $nocheck);
5989     return if !$pid;
5990
5991     my $conf;
5992     if (!$nocheck) {
5993         $conf = PVE::QemuConfig->load_config($vmid);
5994         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5995         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5996             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5997             $timeout = $opts->{down} if $opts->{down};
5998         }
5999         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6000     }
6001
6002     eval {
6003         if ($shutdown) {
6004             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6005                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6006             } else {
6007                 mon_cmd($vmid, "system_powerdown");
6008             }
6009         } else {
6010             mon_cmd($vmid, "quit");
6011         }
6012     };
6013     my $err = $@;
6014
6015     if (!$err) {
6016         $timeout = 60 if !defined($timeout);
6017
6018         my $count = 0;
6019         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6020             $count++;
6021             sleep 1;
6022         }
6023
6024         if ($count >= $timeout) {
6025             if ($force) {
6026                 warn "VM still running - terminating now with SIGTERM\n";
6027                 kill 15, $pid;
6028             } else {
6029                 die "VM quit/powerdown failed - got timeout\n";
6030             }
6031         } else {
6032             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6033             return;
6034         }
6035     } else {
6036        if (!check_running($vmid, $nocheck)) {
6037             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6038             return;
6039        }
6040        if ($force) {
6041             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6042             kill 15, $pid;
6043         } else {
6044             die "VM quit/powerdown failed\n";
6045         }
6046     }
6047
6048     # wait again
6049     $timeout = 10;
6050
6051     my $count = 0;
6052     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6053         $count++;
6054         sleep 1;
6055     }
6056
6057     if ($count >= $timeout) {
6058         warn "VM still running - terminating now with SIGKILL\n";
6059         kill 9, $pid;
6060         sleep 1;
6061     }
6062
6063     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6064 }
6065
6066 # Note: use $nocheck to skip tests if VM configuration file exists.
6067 # We need that when migration VMs to other nodes (files already moved)
6068 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6069 sub vm_stop {
6070     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6071
6072     $force = 1 if !defined($force) && !$shutdown;
6073
6074     if ($migratedfrom){
6075         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6076         kill 15, $pid if $pid;
6077         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6078         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6079         return;
6080     }
6081
6082     PVE::QemuConfig->lock_config($vmid, sub {
6083         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6084    });
6085 }
6086
6087 sub vm_reboot {
6088     my ($vmid, $timeout) = @_;
6089
6090     PVE::QemuConfig->lock_config($vmid, sub {
6091         eval {
6092
6093             # only reboot if running, as qmeventd starts it again on a stop event
6094             return if !check_running($vmid);
6095
6096             create_reboot_request($vmid);
6097
6098             my $storecfg = PVE::Storage::config();
6099             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6100
6101         };
6102         if (my $err = $@) {
6103             # avoid that the next normal shutdown will be confused for a reboot
6104             clear_reboot_request($vmid);
6105             die $err;
6106         }
6107    });
6108 }
6109
6110 # note: if using the statestorage parameter, the caller has to check privileges
6111 sub vm_suspend {
6112     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6113
6114     my $conf;
6115     my $path;
6116     my $storecfg;
6117     my $vmstate;
6118
6119     PVE::QemuConfig->lock_config($vmid, sub {
6120
6121         $conf = PVE::QemuConfig->load_config($vmid);
6122
6123         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6124         PVE::QemuConfig->check_lock($conf)
6125             if !($skiplock || $is_backing_up);
6126
6127         die "cannot suspend to disk during backup\n"
6128             if $is_backing_up && $includestate;
6129
6130         if ($includestate) {
6131             $conf->{lock} = 'suspending';
6132             my $date = strftime("%Y-%m-%d", localtime(time()));
6133             $storecfg = PVE::Storage::config();
6134             if (!$statestorage) {
6135                 $statestorage = find_vmstate_storage($conf, $storecfg);
6136                 # check permissions for the storage
6137                 my $rpcenv = PVE::RPCEnvironment::get();
6138                 if ($rpcenv->{type} ne 'cli') {
6139                     my $authuser = $rpcenv->get_user();
6140                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6141                 }
6142             }
6143
6144
6145             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6146                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6147             $path = PVE::Storage::path($storecfg, $vmstate);
6148             PVE::QemuConfig->write_config($vmid, $conf);
6149         } else {
6150             mon_cmd($vmid, "stop");
6151         }
6152     });
6153
6154     if ($includestate) {
6155         # save vm state
6156         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6157
6158         eval {
6159             set_migration_caps($vmid, 1);
6160             mon_cmd($vmid, "savevm-start", statefile => $path);
6161             for(;;) {
6162                 my $state = mon_cmd($vmid, "query-savevm");
6163                 if (!$state->{status}) {
6164                     die "savevm not active\n";
6165                 } elsif ($state->{status} eq 'active') {
6166                     sleep(1);
6167                     next;
6168                 } elsif ($state->{status} eq 'completed') {
6169                     print "State saved, quitting\n";
6170                     last;
6171                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6172                     die "query-savevm failed with error '$state->{error}'\n"
6173                 } else {
6174                     die "query-savevm returned status '$state->{status}'\n";
6175                 }
6176             }
6177         };
6178         my $err = $@;
6179
6180         PVE::QemuConfig->lock_config($vmid, sub {
6181             $conf = PVE::QemuConfig->load_config($vmid);
6182             if ($err) {
6183                 # cleanup, but leave suspending lock, to indicate something went wrong
6184                 eval {
6185                     mon_cmd($vmid, "savevm-end");
6186                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6187                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6188                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6189                     PVE::QemuConfig->write_config($vmid, $conf);
6190                 };
6191                 warn $@ if $@;
6192                 die $err;
6193             }
6194
6195             die "lock changed unexpectedly\n"
6196                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6197
6198             mon_cmd($vmid, "quit");
6199             $conf->{lock} = 'suspended';
6200             PVE::QemuConfig->write_config($vmid, $conf);
6201         });
6202     }
6203 }
6204
6205 sub vm_resume {
6206     my ($vmid, $skiplock, $nocheck) = @_;
6207
6208     PVE::QemuConfig->lock_config($vmid, sub {
6209         my $res = mon_cmd($vmid, 'query-status');
6210         my $resume_cmd = 'cont';
6211         my $reset = 0;
6212
6213         if ($res->{status}) {
6214             return if $res->{status} eq 'running'; # job done, go home
6215             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6216             $reset = 1 if $res->{status} eq 'shutdown';
6217         }
6218
6219         if (!$nocheck) {
6220
6221             my $conf = PVE::QemuConfig->load_config($vmid);
6222
6223             PVE::QemuConfig->check_lock($conf)
6224                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6225         }
6226
6227         if ($reset) {
6228             # required if a VM shuts down during a backup and we get a resume
6229             # request before the backup finishes for example
6230             mon_cmd($vmid, "system_reset");
6231         }
6232         mon_cmd($vmid, $resume_cmd);
6233     });
6234 }
6235
6236 sub vm_sendkey {
6237     my ($vmid, $skiplock, $key) = @_;
6238
6239     PVE::QemuConfig->lock_config($vmid, sub {
6240
6241         my $conf = PVE::QemuConfig->load_config($vmid);
6242
6243         # there is no qmp command, so we use the human monitor command
6244         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6245         die $res if $res ne '';
6246     });
6247 }
6248
6249 # vzdump restore implementaion
6250
6251 sub tar_archive_read_firstfile {
6252     my $archive = shift;
6253
6254     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6255
6256     # try to detect archive type first
6257     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6258         die "unable to open file '$archive'\n";
6259     my $firstfile = <$fh>;
6260     kill 15, $pid;
6261     close $fh;
6262
6263     die "ERROR: archive contaions no data\n" if !$firstfile;
6264     chomp $firstfile;
6265
6266     return $firstfile;
6267 }
6268
6269 sub tar_restore_cleanup {
6270     my ($storecfg, $statfile) = @_;
6271
6272     print STDERR "starting cleanup\n";
6273
6274     if (my $fd = IO::File->new($statfile, "r")) {
6275         while (defined(my $line = <$fd>)) {
6276             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6277                 my $volid = $2;
6278                 eval {
6279                     if ($volid =~ m|^/|) {
6280                         unlink $volid || die 'unlink failed\n';
6281                     } else {
6282                         PVE::Storage::vdisk_free($storecfg, $volid);
6283                     }
6284                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6285                 };
6286                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6287             } else {
6288                 print STDERR "unable to parse line in statfile - $line";
6289             }
6290         }
6291         $fd->close();
6292     }
6293 }
6294
6295 sub restore_file_archive {
6296     my ($archive, $vmid, $user, $opts) = @_;
6297
6298     return restore_vma_archive($archive, $vmid, $user, $opts)
6299         if $archive eq '-';
6300
6301     my $info = PVE::Storage::archive_info($archive);
6302     my $format = $opts->{format} // $info->{format};
6303     my $comp = $info->{compression};
6304
6305     # try to detect archive format
6306     if ($format eq 'tar') {
6307         return restore_tar_archive($archive, $vmid, $user, $opts);
6308     } else {
6309         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6310     }
6311 }
6312
6313 # hepler to remove disks that will not be used after restore
6314 my $restore_cleanup_oldconf = sub {
6315     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6316
6317     my $kept_disks = {};
6318
6319     PVE::QemuConfig->foreach_volume($oldconf, sub {
6320         my ($ds, $drive) = @_;
6321
6322         return if drive_is_cdrom($drive, 1);
6323
6324         my $volid = $drive->{file};
6325         return if !$volid || $volid =~ m|^/|;
6326
6327         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6328         return if !$path || !$owner || ($owner != $vmid);
6329
6330         # Note: only delete disk we want to restore
6331         # other volumes will become unused
6332         if ($virtdev_hash->{$ds}) {
6333             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6334             if (my $err = $@) {
6335                 warn $err;
6336             }
6337         } else {
6338             $kept_disks->{$volid} = 1;
6339         }
6340     });
6341
6342     # after the restore we have no snapshots anymore
6343     for my $snapname (keys $oldconf->{snapshots}->%*) {
6344         my $snap = $oldconf->{snapshots}->{$snapname};
6345         if ($snap->{vmstate}) {
6346             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6347             if (my $err = $@) {
6348                 warn $err;
6349             }
6350         }
6351
6352         for my $volid (keys $kept_disks->%*) {
6353             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6354             warn $@ if $@;
6355         }
6356     }
6357 };
6358
6359 # Helper to parse vzdump backup device hints
6360 #
6361 # $rpcenv: Environment, used to ckeck storage permissions
6362 # $user: User ID, to check storage permissions
6363 # $storecfg: Storage configuration
6364 # $fh: the file handle for reading the configuration
6365 # $devinfo: should contain device sizes for all backu-up'ed devices
6366 # $options: backup options (pool, default storage)
6367 #
6368 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6369 my $parse_backup_hints = sub {
6370     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6371
6372     my $check_storage = sub { # assert if an image can be allocate
6373         my ($storeid, $scfg) = @_;
6374         die "Content type 'images' is not available on storage '$storeid'\n"
6375             if !$scfg->{content}->{images};
6376         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6377             if $user ne 'root@pam';
6378     };
6379
6380     my $virtdev_hash = {};
6381     while (defined(my $line = <$fh>)) {
6382         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6383             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6384             die "archive does not contain data for drive '$virtdev'\n"
6385                 if !$devinfo->{$devname};
6386
6387             if (defined($options->{storage})) {
6388                 $storeid = $options->{storage} || 'local';
6389             } elsif (!$storeid) {
6390                 $storeid = 'local';
6391             }
6392             $format = 'raw' if !$format;
6393             $devinfo->{$devname}->{devname} = $devname;
6394             $devinfo->{$devname}->{virtdev} = $virtdev;
6395             $devinfo->{$devname}->{format} = $format;
6396             $devinfo->{$devname}->{storeid} = $storeid;
6397
6398             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6399             $check_storage->($storeid, $scfg); # permission and content type check
6400
6401             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6402         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6403             my $virtdev = $1;
6404             my $drive = parse_drive($virtdev, $2);
6405
6406             if (drive_is_cloudinit($drive)) {
6407                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6408                 $storeid = $options->{storage} if defined ($options->{storage});
6409                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6410                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6411
6412                 $check_storage->($storeid, $scfg); # permission and content type check
6413
6414                 $virtdev_hash->{$virtdev} = {
6415                     format => $format,
6416                     storeid => $storeid,
6417                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6418                     is_cloudinit => 1,
6419                 };
6420             }
6421         }
6422     }
6423
6424     return $virtdev_hash;
6425 };
6426
6427 # Helper to allocate and activate all volumes required for a restore
6428 #
6429 # $storecfg: Storage configuration
6430 # $virtdev_hash: as returned by parse_backup_hints()
6431 #
6432 # Returns: { $virtdev => $volid }
6433 my $restore_allocate_devices = sub {
6434     my ($storecfg, $virtdev_hash, $vmid) = @_;
6435
6436     my $map = {};
6437     foreach my $virtdev (sort keys %$virtdev_hash) {
6438         my $d = $virtdev_hash->{$virtdev};
6439         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6440         my $storeid = $d->{storeid};
6441         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6442
6443         # test if requested format is supported
6444         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6445         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6446         $d->{format} = $defFormat if !$supported;
6447
6448         my $name;
6449         if ($d->{is_cloudinit}) {
6450             $name = "vm-$vmid-cloudinit";
6451             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6452             if ($scfg->{path}) {
6453                 $name .= ".$d->{format}";
6454             }
6455         }
6456
6457         my $volid = PVE::Storage::vdisk_alloc(
6458             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6459
6460         print STDERR "new volume ID is '$volid'\n";
6461         $d->{volid} = $volid;
6462
6463         PVE::Storage::activate_volumes($storecfg, [$volid]);
6464
6465         $map->{$virtdev} = $volid;
6466     }
6467
6468     return $map;
6469 };
6470
6471 sub restore_update_config_line {
6472     my ($cookie, $map, $line, $unique) = @_;
6473
6474     return '' if $line =~ m/^\#qmdump\#/;
6475     return '' if $line =~ m/^\#vzdump\#/;
6476     return '' if $line =~ m/^lock:/;
6477     return '' if $line =~ m/^unused\d+:/;
6478     return '' if $line =~ m/^parent:/;
6479
6480     my $res = '';
6481
6482     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6483     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6484         # try to convert old 1.X settings
6485         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6486         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6487             my ($model, $macaddr) = split(/\=/, $devconfig);
6488             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6489             my $net = {
6490                 model => $model,
6491                 bridge => "vmbr$ind",
6492                 macaddr => $macaddr,
6493             };
6494             my $netstr = print_net($net);
6495
6496             $res .= "net$cookie->{netcount}: $netstr\n";
6497             $cookie->{netcount}++;
6498         }
6499     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6500         my ($id, $netstr) = ($1, $2);
6501         my $net = parse_net($netstr);
6502         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6503         $netstr = print_net($net);
6504         $res .= "$id: $netstr\n";
6505     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6506         my $virtdev = $1;
6507         my $value = $3;
6508         my $di = parse_drive($virtdev, $value);
6509         if (defined($di->{backup}) && !$di->{backup}) {
6510             $res .= "#$line";
6511         } elsif ($map->{$virtdev}) {
6512             delete $di->{format}; # format can change on restore
6513             $di->{file} = $map->{$virtdev};
6514             $value = print_drive($di);
6515             $res .= "$virtdev: $value\n";
6516         } else {
6517             $res .= $line;
6518         }
6519     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6520         my $vmgenid = $1;
6521         if ($vmgenid ne '0') {
6522             # always generate a new vmgenid if there was a valid one setup
6523             $vmgenid = generate_uuid();
6524         }
6525         $res .= "vmgenid: $vmgenid\n";
6526     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6527         my ($uuid, $uuid_str);
6528         UUID::generate($uuid);
6529         UUID::unparse($uuid, $uuid_str);
6530         my $smbios1 = parse_smbios1($2);
6531         $smbios1->{uuid} = $uuid_str;
6532         $res .= $1.print_smbios1($smbios1)."\n";
6533     } else {
6534         $res .= $line;
6535     }
6536
6537     return $res;
6538 }
6539
6540 my $restore_deactivate_volumes = sub {
6541     my ($storecfg, $virtdev_hash) = @_;
6542
6543     my $vollist = [];
6544     for my $dev (values $virtdev_hash->%*) {
6545         push $vollist->@*, $dev->{volid} if $dev->{volid};
6546     }
6547
6548     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6549     print STDERR $@ if $@;
6550 };
6551
6552 my $restore_destroy_volumes = sub {
6553     my ($storecfg, $virtdev_hash) = @_;
6554
6555     for my $dev (values $virtdev_hash->%*) {
6556         my $volid = $dev->{volid} or next;
6557         eval {
6558             PVE::Storage::vdisk_free($storecfg, $volid);
6559             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6560         };
6561         print STDERR "unable to cleanup '$volid' - $@" if $@;
6562     }
6563 };
6564
6565 my $restore_merge_config = sub {
6566     my ($filename, $backup_conf_raw, $override_conf) = @_;
6567
6568     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6569     for my $key (keys $override_conf->%*) {
6570         $backup_conf->{$key} = $override_conf->{$key};
6571     }
6572
6573     return $backup_conf;
6574 };
6575
6576 sub scan_volids {
6577     my ($cfg, $vmid) = @_;
6578
6579     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6580
6581     my $volid_hash = {};
6582     foreach my $storeid (keys %$info) {
6583         foreach my $item (@{$info->{$storeid}}) {
6584             next if !($item->{volid} && $item->{size});
6585             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6586             $volid_hash->{$item->{volid}} = $item;
6587         }
6588     }
6589
6590     return $volid_hash;
6591 }
6592
6593 sub update_disk_config {
6594     my ($vmid, $conf, $volid_hash) = @_;
6595
6596     my $changes;
6597     my $prefix = "VM $vmid";
6598
6599     # used and unused disks
6600     my $referenced = {};
6601
6602     # Note: it is allowed to define multiple storages with same path (alias), so
6603     # we need to check both 'volid' and real 'path' (two different volid can point
6604     # to the same path).
6605
6606     my $referencedpath = {};
6607
6608     # update size info
6609     PVE::QemuConfig->foreach_volume($conf, sub {
6610         my ($opt, $drive) = @_;
6611
6612         my $volid = $drive->{file};
6613         return if !$volid;
6614         my $volume = $volid_hash->{$volid};
6615
6616         # mark volid as "in-use" for next step
6617         $referenced->{$volid} = 1;
6618         if ($volume && (my $path = $volume->{path})) {
6619             $referencedpath->{$path} = 1;
6620         }
6621
6622         return if drive_is_cdrom($drive);
6623         return if !$volume;
6624
6625         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6626         if (defined($updated)) {
6627             $changes = 1;
6628             $conf->{$opt} = print_drive($updated);
6629             print "$prefix ($opt): $msg\n";
6630         }
6631     });
6632
6633     # remove 'unusedX' entry if volume is used
6634     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6635         my ($opt, $drive) = @_;
6636
6637         my $volid = $drive->{file};
6638         return if !$volid;
6639
6640         my $path;
6641         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6642         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6643             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6644             $changes = 1;
6645             delete $conf->{$opt};
6646         }
6647
6648         $referenced->{$volid} = 1;
6649         $referencedpath->{$path} = 1 if $path;
6650     });
6651
6652     foreach my $volid (sort keys %$volid_hash) {
6653         next if $volid =~ m/vm-$vmid-state-/;
6654         next if $referenced->{$volid};
6655         my $path = $volid_hash->{$volid}->{path};
6656         next if !$path; # just to be sure
6657         next if $referencedpath->{$path};
6658         $changes = 1;
6659         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6660         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6661         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6662     }
6663
6664     return $changes;
6665 }
6666
6667 sub rescan {
6668     my ($vmid, $nolock, $dryrun) = @_;
6669
6670     my $cfg = PVE::Storage::config();
6671
6672     print "rescan volumes...\n";
6673     my $volid_hash = scan_volids($cfg, $vmid);
6674
6675     my $updatefn =  sub {
6676         my ($vmid) = @_;
6677
6678         my $conf = PVE::QemuConfig->load_config($vmid);
6679
6680         PVE::QemuConfig->check_lock($conf);
6681
6682         my $vm_volids = {};
6683         foreach my $volid (keys %$volid_hash) {
6684             my $info = $volid_hash->{$volid};
6685             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6686         }
6687
6688         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6689
6690         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6691     };
6692
6693     if (defined($vmid)) {
6694         if ($nolock) {
6695             &$updatefn($vmid);
6696         } else {
6697             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6698         }
6699     } else {
6700         my $vmlist = config_list();
6701         foreach my $vmid (keys %$vmlist) {
6702             if ($nolock) {
6703                 &$updatefn($vmid);
6704             } else {
6705                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6706             }
6707         }
6708     }
6709 }
6710
6711 sub restore_proxmox_backup_archive {
6712     my ($archive, $vmid, $user, $options) = @_;
6713
6714     my $storecfg = PVE::Storage::config();
6715
6716     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6717     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6718
6719     my $fingerprint = $scfg->{fingerprint};
6720     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6721
6722     my $repo = PVE::PBSClient::get_repository($scfg);
6723     my $namespace = $scfg->{namespace};
6724
6725     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6726     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6727     local $ENV{PBS_PASSWORD} = $password;
6728     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6729
6730     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6731         PVE::Storage::parse_volname($storecfg, $archive);
6732
6733     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6734
6735     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6736
6737     my $tmpdir = "/var/tmp/vzdumptmp$$";
6738     rmtree $tmpdir;
6739     mkpath $tmpdir;
6740
6741     my $conffile = PVE::QemuConfig->config_file($vmid);
6742      # disable interrupts (always do cleanups)
6743     local $SIG{INT} =
6744         local $SIG{TERM} =
6745         local $SIG{QUIT} =
6746         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6747
6748     # Note: $oldconf is undef if VM does not exists
6749     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6750     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6751     my $new_conf_raw = '';
6752
6753     my $rpcenv = PVE::RPCEnvironment::get();
6754     my $devinfo = {}; # info about drives included in backup
6755     my $virtdev_hash = {}; # info about allocated drives
6756
6757     eval {
6758         # enable interrupts
6759         local $SIG{INT} =
6760             local $SIG{TERM} =
6761             local $SIG{QUIT} =
6762             local $SIG{HUP} =
6763             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6764
6765         my $cfgfn = "$tmpdir/qemu-server.conf";
6766         my $firewall_config_fn = "$tmpdir/fw.conf";
6767         my $index_fn = "$tmpdir/index.json";
6768
6769         my $cmd = "restore";
6770
6771         my $param = [$pbs_backup_name, "index.json", $index_fn];
6772         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6773         my $index = PVE::Tools::file_get_contents($index_fn);
6774         $index = decode_json($index);
6775
6776         foreach my $info (@{$index->{files}}) {
6777             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6778                 my $devname = $1;
6779                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6780                     $devinfo->{$devname}->{size} = $1;
6781                 } else {
6782                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6783                 }
6784             }
6785         }
6786
6787         my $is_qemu_server_backup = scalar(
6788             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6789         );
6790         if (!$is_qemu_server_backup) {
6791             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6792         }
6793         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6794
6795         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6796         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6797
6798         if ($has_firewall_config) {
6799             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6800             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6801
6802             my $pve_firewall_dir = '/etc/pve/firewall';
6803             mkdir $pve_firewall_dir; # make sure the dir exists
6804             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6805         }
6806
6807         my $fh = IO::File->new($cfgfn, "r") ||
6808             die "unable to read qemu-server.conf - $!\n";
6809
6810         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6811
6812         # fixme: rate limit?
6813
6814         # create empty/temp config
6815         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6816
6817         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6818
6819         # allocate volumes
6820         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6821
6822         foreach my $virtdev (sort keys %$virtdev_hash) {
6823             my $d = $virtdev_hash->{$virtdev};
6824             next if $d->{is_cloudinit}; # no need to restore cloudinit
6825
6826             # this fails if storage is unavailable
6827             my $volid = $d->{volid};
6828             my $path = PVE::Storage::path($storecfg, $volid);
6829
6830             # for live-restore we only want to preload the efidisk and TPM state
6831             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6832
6833             my @ns_arg;
6834             if (defined(my $ns = $scfg->{namespace})) {
6835                 @ns_arg = ('--ns', $ns);
6836             }
6837
6838             my $pbs_restore_cmd = [
6839                 '/usr/bin/pbs-restore',
6840                 '--repository', $repo,
6841                 @ns_arg,
6842                 $pbs_backup_name,
6843                 "$d->{devname}.img.fidx",
6844                 $path,
6845                 '--verbose',
6846                 ];
6847
6848             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6849             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6850
6851             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6852                 push @$pbs_restore_cmd, '--skip-zero';
6853             }
6854
6855             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6856             print "restore proxmox backup image: $dbg_cmdstring\n";
6857             run_command($pbs_restore_cmd);
6858         }
6859
6860         $fh->seek(0, 0) || die "seek failed - $!\n";
6861
6862         my $cookie = { netcount => 0 };
6863         while (defined(my $line = <$fh>)) {
6864             $new_conf_raw .= restore_update_config_line(
6865                 $cookie,
6866                 $map,
6867                 $line,
6868                 $options->{unique},
6869             );
6870         }
6871
6872         $fh->close();
6873     };
6874     my $err = $@;
6875
6876     if ($err || !$options->{live}) {
6877         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6878     }
6879
6880     rmtree $tmpdir;
6881
6882     if ($err) {
6883         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6884         die $err;
6885     }
6886
6887     if ($options->{live}) {
6888         # keep lock during live-restore
6889         $new_conf_raw .= "\nlock: create";
6890     }
6891
6892     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6893     PVE::QemuConfig->write_config($vmid, $new_conf);
6894
6895     eval { rescan($vmid, 1); };
6896     warn $@ if $@;
6897
6898     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6899
6900     if ($options->{live}) {
6901         # enable interrupts
6902         local $SIG{INT} =
6903             local $SIG{TERM} =
6904             local $SIG{QUIT} =
6905             local $SIG{HUP} =
6906             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6907
6908         my $conf = PVE::QemuConfig->load_config($vmid);
6909         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6910
6911         # these special drives are already restored before start
6912         delete $devinfo->{'drive-efidisk0'};
6913         delete $devinfo->{'drive-tpmstate0-backup'};
6914
6915         my $pbs_opts = {
6916             repo => $repo,
6917             keyfile => $keyfile,
6918             snapshot => $pbs_backup_name,
6919             namespace => $namespace,
6920         };
6921         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6922
6923         PVE::QemuConfig->remove_lock($vmid, "create");
6924     }
6925 }
6926
6927 sub pbs_live_restore {
6928     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6929
6930     print "starting VM for live-restore\n";
6931     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6932
6933     my $pbs_backing = {};
6934     for my $ds (keys %$restored_disks) {
6935         $ds =~ m/^drive-(.*)$/;
6936         my $confname = $1;
6937         $pbs_backing->{$confname} = {
6938             repository => $opts->{repo},
6939             snapshot => $opts->{snapshot},
6940             archive => "$ds.img.fidx",
6941         };
6942         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6943         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6944
6945         my $drive = parse_drive($confname, $conf->{$confname});
6946         print "restoring '$ds' to '$drive->{file}'\n";
6947     }
6948
6949     my $drives_streamed = 0;
6950     eval {
6951         # make sure HA doesn't interrupt our restore by stopping the VM
6952         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6953             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6954         }
6955
6956         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6957         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6958         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6959
6960         my $qmeventd_fd = register_qmeventd_handle($vmid);
6961
6962         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6963         # this will effectively collapse the backing image chain consisting of
6964         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6965         # removes itself once all backing images vanish with 'auto-remove=on')
6966         my $jobs = {};
6967         for my $ds (sort keys %$restored_disks) {
6968             my $job_id = "restore-$ds";
6969             mon_cmd($vmid, 'block-stream',
6970                 'job-id' => $job_id,
6971                 device => "$ds",
6972             );
6973             $jobs->{$job_id} = {};
6974         }
6975
6976         mon_cmd($vmid, 'cont');
6977         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6978
6979         print "restore-drive jobs finished successfully, removing all tracking block devices"
6980             ." to disconnect from Proxmox Backup Server\n";
6981
6982         for my $ds (sort keys %$restored_disks) {
6983             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6984         }
6985
6986         close($qmeventd_fd);
6987     };
6988
6989     my $err = $@;
6990
6991     if ($err) {
6992         warn "An error occurred during live-restore: $err\n";
6993         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6994         die "live-restore failed\n";
6995     }
6996 }
6997
6998 sub restore_vma_archive {
6999     my ($archive, $vmid, $user, $opts, $comp) = @_;
7000
7001     my $readfrom = $archive;
7002
7003     my $cfg = PVE::Storage::config();
7004     my $commands = [];
7005     my $bwlimit = $opts->{bwlimit};
7006
7007     my $dbg_cmdstring = '';
7008     my $add_pipe = sub {
7009         my ($cmd) = @_;
7010         push @$commands, $cmd;
7011         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7012         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7013         $readfrom = '-';
7014     };
7015
7016     my $input = undef;
7017     if ($archive eq '-') {
7018         $input = '<&STDIN';
7019     } else {
7020         # If we use a backup from a PVE defined storage we also consider that
7021         # storage's rate limit:
7022         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7023         if (defined($volid)) {
7024             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7025             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7026             if ($readlimit) {
7027                 print STDERR "applying read rate limit: $readlimit\n";
7028                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7029                 $add_pipe->($cstream);
7030             }
7031         }
7032     }
7033
7034     if ($comp) {
7035         my $info = PVE::Storage::decompressor_info('vma', $comp);
7036         my $cmd = $info->{decompressor};
7037         push @$cmd, $readfrom;
7038         $add_pipe->($cmd);
7039     }
7040
7041     my $tmpdir = "/var/tmp/vzdumptmp$$";
7042     rmtree $tmpdir;
7043
7044     # disable interrupts (always do cleanups)
7045     local $SIG{INT} =
7046         local $SIG{TERM} =
7047         local $SIG{QUIT} =
7048         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7049
7050     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7051     POSIX::mkfifo($mapfifo, 0600);
7052     my $fifofh;
7053     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7054
7055     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7056
7057     my $oldtimeout;
7058     my $timeout = 5;
7059
7060     my $devinfo = {}; # info about drives included in backup
7061     my $virtdev_hash = {}; # info about allocated drives
7062
7063     my $rpcenv = PVE::RPCEnvironment::get();
7064
7065     my $conffile = PVE::QemuConfig->config_file($vmid);
7066
7067     # Note: $oldconf is undef if VM does not exist
7068     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7069     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7070     my $new_conf_raw = '';
7071
7072     my %storage_limits;
7073
7074     my $print_devmap = sub {
7075         my $cfgfn = "$tmpdir/qemu-server.conf";
7076
7077         # we can read the config - that is already extracted
7078         my $fh = IO::File->new($cfgfn, "r") ||
7079             die "unable to read qemu-server.conf - $!\n";
7080
7081         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7082         if (-f $fwcfgfn) {
7083             my $pve_firewall_dir = '/etc/pve/firewall';
7084             mkdir $pve_firewall_dir; # make sure the dir exists
7085             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7086         }
7087
7088         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7089
7090         foreach my $info (values %{$virtdev_hash}) {
7091             my $storeid = $info->{storeid};
7092             next if defined($storage_limits{$storeid});
7093
7094             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7095             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7096             $storage_limits{$storeid} = $limit * 1024;
7097         }
7098
7099         foreach my $devname (keys %$devinfo) {
7100             die "found no device mapping information for device '$devname'\n"
7101                 if !$devinfo->{$devname}->{virtdev};
7102         }
7103
7104         # create empty/temp config
7105         if ($oldconf) {
7106             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7107             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7108         }
7109
7110         # allocate volumes
7111         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7112
7113         # print restore information to $fifofh
7114         foreach my $virtdev (sort keys %$virtdev_hash) {
7115             my $d = $virtdev_hash->{$virtdev};
7116             next if $d->{is_cloudinit}; # no need to restore cloudinit
7117
7118             my $storeid = $d->{storeid};
7119             my $volid = $d->{volid};
7120
7121             my $map_opts = '';
7122             if (my $limit = $storage_limits{$storeid}) {
7123                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7124             }
7125
7126             my $write_zeros = 1;
7127             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7128                 $write_zeros = 0;
7129             }
7130
7131             my $path = PVE::Storage::path($cfg, $volid);
7132
7133             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7134
7135             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7136         }
7137
7138         $fh->seek(0, 0) || die "seek failed - $!\n";
7139
7140         my $cookie = { netcount => 0 };
7141         while (defined(my $line = <$fh>)) {
7142             $new_conf_raw .= restore_update_config_line(
7143                 $cookie,
7144                 $map,
7145                 $line,
7146                 $opts->{unique},
7147             );
7148         }
7149
7150         $fh->close();
7151     };
7152
7153     eval {
7154         # enable interrupts
7155         local $SIG{INT} =
7156             local $SIG{TERM} =
7157             local $SIG{QUIT} =
7158             local $SIG{HUP} =
7159             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7160         local $SIG{ALRM} = sub { die "got timeout\n"; };
7161
7162         $oldtimeout = alarm($timeout);
7163
7164         my $parser = sub {
7165             my $line = shift;
7166
7167             print "$line\n";
7168
7169             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7170                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7171                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7172             } elsif ($line =~ m/^CTIME: /) {
7173                 # we correctly received the vma config, so we can disable
7174                 # the timeout now for disk allocation (set to 10 minutes, so
7175                 # that we always timeout if something goes wrong)
7176                 alarm(600);
7177                 &$print_devmap();
7178                 print $fifofh "done\n";
7179                 my $tmp = $oldtimeout || 0;
7180                 $oldtimeout = undef;
7181                 alarm($tmp);
7182                 close($fifofh);
7183                 $fifofh = undef;
7184             }
7185         };
7186
7187         print "restore vma archive: $dbg_cmdstring\n";
7188         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7189     };
7190     my $err = $@;
7191
7192     alarm($oldtimeout) if $oldtimeout;
7193
7194     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7195
7196     close($fifofh) if $fifofh;
7197     unlink $mapfifo;
7198     rmtree $tmpdir;
7199
7200     if ($err) {
7201         $restore_destroy_volumes->($cfg, $virtdev_hash);
7202         die $err;
7203     }
7204
7205     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7206     PVE::QemuConfig->write_config($vmid, $new_conf);
7207
7208     eval { rescan($vmid, 1); };
7209     warn $@ if $@;
7210
7211     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7212 }
7213
7214 sub restore_tar_archive {
7215     my ($archive, $vmid, $user, $opts) = @_;
7216
7217     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7218         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7219         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7220     }
7221
7222     if ($archive ne '-') {
7223         my $firstfile = tar_archive_read_firstfile($archive);
7224         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7225             if $firstfile ne 'qemu-server.conf';
7226     }
7227
7228     my $storecfg = PVE::Storage::config();
7229
7230     # avoid zombie disks when restoring over an existing VM -> cleanup first
7231     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7232     # skiplock=1 because qmrestore has set the 'create' lock itself already
7233     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7234     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7235
7236     my $tocmd = "/usr/lib/qemu-server/qmextract";
7237
7238     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7239     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7240     $tocmd .= ' --prealloc' if $opts->{prealloc};
7241     $tocmd .= ' --info' if $opts->{info};
7242
7243     # tar option "xf" does not autodetect compression when read from STDIN,
7244     # so we pipe to zcat
7245     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7246         PVE::Tools::shellquote("--to-command=$tocmd");
7247
7248     my $tmpdir = "/var/tmp/vzdumptmp$$";
7249     mkpath $tmpdir;
7250
7251     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7252     local $ENV{VZDUMP_VMID} = $vmid;
7253     local $ENV{VZDUMP_USER} = $user;
7254
7255     my $conffile = PVE::QemuConfig->config_file($vmid);
7256     my $new_conf_raw = '';
7257
7258     # disable interrupts (always do cleanups)
7259     local $SIG{INT} =
7260         local $SIG{TERM} =
7261         local $SIG{QUIT} =
7262         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7263
7264     eval {
7265         # enable interrupts
7266         local $SIG{INT} =
7267             local $SIG{TERM} =
7268             local $SIG{QUIT} =
7269             local $SIG{HUP} =
7270             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7271
7272         if ($archive eq '-') {
7273             print "extracting archive from STDIN\n";
7274             run_command($cmd, input => "<&STDIN");
7275         } else {
7276             print "extracting archive '$archive'\n";
7277             run_command($cmd);
7278         }
7279
7280         return if $opts->{info};
7281
7282         # read new mapping
7283         my $map = {};
7284         my $statfile = "$tmpdir/qmrestore.stat";
7285         if (my $fd = IO::File->new($statfile, "r")) {
7286             while (defined (my $line = <$fd>)) {
7287                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7288                     $map->{$1} = $2 if $1;
7289                 } else {
7290                     print STDERR "unable to parse line in statfile - $line\n";
7291                 }
7292             }
7293             $fd->close();
7294         }
7295
7296         my $confsrc = "$tmpdir/qemu-server.conf";
7297
7298         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7299
7300         my $cookie = { netcount => 0 };
7301         while (defined (my $line = <$srcfd>)) {
7302             $new_conf_raw .= restore_update_config_line(
7303                 $cookie,
7304                 $map,
7305                 $line,
7306                 $opts->{unique},
7307             );
7308         }
7309
7310         $srcfd->close();
7311     };
7312     if (my $err = $@) {
7313         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7314         die $err;
7315     }
7316
7317     rmtree $tmpdir;
7318
7319     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7320
7321     PVE::Cluster::cfs_update(); # make sure we read new file
7322
7323     eval { rescan($vmid, 1); };
7324     warn $@ if $@;
7325 };
7326
7327 sub foreach_storage_used_by_vm {
7328     my ($conf, $func) = @_;
7329
7330     my $sidhash = {};
7331
7332     PVE::QemuConfig->foreach_volume($conf, sub {
7333         my ($ds, $drive) = @_;
7334         return if drive_is_cdrom($drive);
7335
7336         my $volid = $drive->{file};
7337
7338         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7339         $sidhash->{$sid} = $sid if $sid;
7340     });
7341
7342     foreach my $sid (sort keys %$sidhash) {
7343         &$func($sid);
7344     }
7345 }
7346
7347 my $qemu_snap_storage = {
7348     rbd => 1,
7349 };
7350 sub do_snapshots_with_qemu {
7351     my ($storecfg, $volid, $deviceid) = @_;
7352
7353     return if $deviceid =~ m/tpmstate0/;
7354
7355     my $storage_name = PVE::Storage::parse_volume_id($volid);
7356     my $scfg = $storecfg->{ids}->{$storage_name};
7357     die "could not find storage '$storage_name'\n" if !defined($scfg);
7358
7359     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7360         return 1;
7361     }
7362
7363     if ($volid =~ m/\.(qcow2|qed)$/){
7364         return 1;
7365     }
7366
7367     return;
7368 }
7369
7370 sub qga_check_running {
7371     my ($vmid, $nowarn) = @_;
7372
7373     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7374     if ($@) {
7375         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7376         return 0;
7377     }
7378     return 1;
7379 }
7380
7381 sub template_create {
7382     my ($vmid, $conf, $disk) = @_;
7383
7384     my $storecfg = PVE::Storage::config();
7385
7386     PVE::QemuConfig->foreach_volume($conf, sub {
7387         my ($ds, $drive) = @_;
7388
7389         return if drive_is_cdrom($drive);
7390         return if $disk && $ds ne $disk;
7391
7392         my $volid = $drive->{file};
7393         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7394
7395         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7396         $drive->{file} = $voliddst;
7397         $conf->{$ds} = print_drive($drive);
7398         PVE::QemuConfig->write_config($vmid, $conf);
7399     });
7400 }
7401
7402 sub convert_iscsi_path {
7403     my ($path) = @_;
7404
7405     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7406         my $portal = $1;
7407         my $target = $2;
7408         my $lun = $3;
7409
7410         my $initiator_name = get_initiator_name();
7411
7412         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7413                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7414     }
7415
7416     die "cannot convert iscsi path '$path', unkown format\n";
7417 }
7418
7419 sub qemu_img_convert {
7420     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7421
7422     my $storecfg = PVE::Storage::config();
7423     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7424     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7425
7426     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7427
7428     my $cachemode;
7429     my $src_path;
7430     my $src_is_iscsi = 0;
7431     my $src_format;
7432
7433     if ($src_storeid) {
7434         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7435         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7436         $src_format = qemu_img_format($src_scfg, $src_volname);
7437         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7438         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7439         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7440     } elsif (-f $src_volid || -b $src_volid) {
7441         $src_path = $src_volid;
7442         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7443             $src_format = $1;
7444         }
7445     }
7446
7447     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7448
7449     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7450     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7451     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7452     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7453
7454     my $cmd = [];
7455     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7456     push @$cmd, '-l', "snapshot.name=$snapname"
7457         if $snapname && $src_format && $src_format eq "qcow2";
7458     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7459     push @$cmd, '-T', $cachemode if defined($cachemode);
7460
7461     if ($src_is_iscsi) {
7462         push @$cmd, '--image-opts';
7463         $src_path = convert_iscsi_path($src_path);
7464     } elsif ($src_format) {
7465         push @$cmd, '-f', $src_format;
7466     }
7467
7468     if ($dst_is_iscsi) {
7469         push @$cmd, '--target-image-opts';
7470         $dst_path = convert_iscsi_path($dst_path);
7471     } else {
7472         push @$cmd, '-O', $dst_format;
7473     }
7474
7475     push @$cmd, $src_path;
7476
7477     if (!$dst_is_iscsi && $is_zero_initialized) {
7478         push @$cmd, "zeroinit:$dst_path";
7479     } else {
7480         push @$cmd, $dst_path;
7481     }
7482
7483     my $parser = sub {
7484         my $line = shift;
7485         if($line =~ m/\((\S+)\/100\%\)/){
7486             my $percent = $1;
7487             my $transferred = int($size * $percent / 100);
7488             my $total_h = render_bytes($size, 1);
7489             my $transferred_h = render_bytes($transferred, 1);
7490
7491             print "transferred $transferred_h of $total_h ($percent%)\n";
7492         }
7493
7494     };
7495
7496     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7497     my $err = $@;
7498     die "copy failed: $err" if $err;
7499 }
7500
7501 sub qemu_img_format {
7502     my ($scfg, $volname) = @_;
7503
7504     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7505         return $1;
7506     } else {
7507         return "raw";
7508     }
7509 }
7510
7511 sub qemu_drive_mirror {
7512     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7513
7514     $jobs = {} if !$jobs;
7515
7516     my $qemu_target;
7517     my $format;
7518     $jobs->{"drive-$drive"} = {};
7519
7520     if ($dst_volid =~ /^nbd:/) {
7521         $qemu_target = $dst_volid;
7522         $format = "nbd";
7523     } else {
7524         my $storecfg = PVE::Storage::config();
7525         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7526
7527         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7528
7529         $format = qemu_img_format($dst_scfg, $dst_volname);
7530
7531         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7532
7533         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7534     }
7535
7536     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7537     $opts->{format} = $format if $format;
7538
7539     if (defined($src_bitmap)) {
7540         $opts->{sync} = 'incremental';
7541         $opts->{bitmap} = $src_bitmap;
7542         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7543     }
7544
7545     if (defined($bwlimit)) {
7546         $opts->{speed} = $bwlimit * 1024;
7547         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7548     } else {
7549         print "drive mirror is starting for drive-$drive\n";
7550     }
7551
7552     # if a job already runs for this device we get an error, catch it for cleanup
7553     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7554     if (my $err = $@) {
7555         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7556         warn "$@\n" if $@;
7557         die "mirroring error: $err\n";
7558     }
7559
7560     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7561 }
7562
7563 # $completion can be either
7564 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7565 # 'cancel': wait until all jobs are ready, block-job-cancel them
7566 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7567 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7568 sub qemu_drive_mirror_monitor {
7569     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7570
7571     $completion //= 'complete';
7572     $op //= "mirror";
7573
7574     eval {
7575         my $err_complete = 0;
7576
7577         my $starttime = time ();
7578         while (1) {
7579             die "block job ('$op') timed out\n" if $err_complete > 300;
7580
7581             my $stats = mon_cmd($vmid, "query-block-jobs");
7582             my $ctime = time();
7583
7584             my $running_jobs = {};
7585             for my $stat (@$stats) {
7586                 next if $stat->{type} ne $op;
7587                 $running_jobs->{$stat->{device}} = $stat;
7588             }
7589
7590             my $readycounter = 0;
7591
7592             for my $job_id (sort keys %$jobs) {
7593                 my $job = $running_jobs->{$job_id};
7594
7595                 my $vanished = !defined($job);
7596                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7597                 if($complete || ($vanished && $completion eq 'auto')) {
7598                     print "$job_id: $op-job finished\n";
7599                     delete $jobs->{$job_id};
7600                     next;
7601                 }
7602
7603                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7604
7605                 my $busy = $job->{busy};
7606                 my $ready = $job->{ready};
7607                 if (my $total = $job->{len}) {
7608                     my $transferred = $job->{offset} || 0;
7609                     my $remaining = $total - $transferred;
7610                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7611
7612                     my $duration = $ctime - $starttime;
7613                     my $total_h = render_bytes($total, 1);
7614                     my $transferred_h = render_bytes($transferred, 1);
7615
7616                     my $status = sprintf(
7617                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7618                         render_duration($duration),
7619                     );
7620
7621                     if ($ready) {
7622                         if ($busy) {
7623                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7624                         } else {
7625                             $status .= ", ready";
7626                         }
7627                     }
7628                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7629                     $jobs->{$job_id}->{ready} = $ready;
7630                 }
7631
7632                 $readycounter++ if $job->{ready};
7633             }
7634
7635             last if scalar(keys %$jobs) == 0;
7636
7637             if ($readycounter == scalar(keys %$jobs)) {
7638                 print "all '$op' jobs are ready\n";
7639
7640                 # do the complete later (or has already been done)
7641                 last if $completion eq 'skip' || $completion eq 'auto';
7642
7643                 if ($vmiddst && $vmiddst != $vmid) {
7644                     my $agent_running = $qga && qga_check_running($vmid);
7645                     if ($agent_running) {
7646                         print "freeze filesystem\n";
7647                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7648                         warn $@ if $@;
7649                     } else {
7650                         print "suspend vm\n";
7651                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7652                         warn $@ if $@;
7653                     }
7654
7655                     # if we clone a disk for a new target vm, we don't switch the disk
7656                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7657
7658                     if ($agent_running) {
7659                         print "unfreeze filesystem\n";
7660                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7661                         warn $@ if $@;
7662                     } else {
7663                         print "resume vm\n";
7664                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7665                         warn $@ if $@;
7666                     }
7667
7668                     last;
7669                 } else {
7670
7671                     for my $job_id (sort keys %$jobs) {
7672                         # try to switch the disk if source and destination are on the same guest
7673                         print "$job_id: Completing block job_id...\n";
7674
7675                         my $op;
7676                         if ($completion eq 'complete') {
7677                             $op = 'block-job-complete';
7678                         } elsif ($completion eq 'cancel') {
7679                             $op = 'block-job-cancel';
7680                         } else {
7681                             die "invalid completion value: $completion\n";
7682                         }
7683                         eval { mon_cmd($vmid, $op, device => $job_id) };
7684                         if ($@ =~ m/cannot be completed/) {
7685                             print "$job_id: block job cannot be completed, trying again.\n";
7686                             $err_complete++;
7687                         }else {
7688                             print "$job_id: Completed successfully.\n";
7689                             $jobs->{$job_id}->{complete} = 1;
7690                         }
7691                     }
7692                 }
7693             }
7694             sleep 1;
7695         }
7696     };
7697     my $err = $@;
7698
7699     if ($err) {
7700         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7701         die "block job ($op) error: $err";
7702     }
7703 }
7704
7705 sub qemu_blockjobs_cancel {
7706     my ($vmid, $jobs) = @_;
7707
7708     foreach my $job (keys %$jobs) {
7709         print "$job: Cancelling block job\n";
7710         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7711         $jobs->{$job}->{cancel} = 1;
7712     }
7713
7714     while (1) {
7715         my $stats = mon_cmd($vmid, "query-block-jobs");
7716
7717         my $running_jobs = {};
7718         foreach my $stat (@$stats) {
7719             $running_jobs->{$stat->{device}} = $stat;
7720         }
7721
7722         foreach my $job (keys %$jobs) {
7723
7724             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7725                 print "$job: Done.\n";
7726                 delete $jobs->{$job};
7727             }
7728         }
7729
7730         last if scalar(keys %$jobs) == 0;
7731
7732         sleep 1;
7733     }
7734 }
7735
7736 sub clone_disk {
7737     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7738
7739     my ($vmid, $running) = $source->@{qw(vmid running)};
7740     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7741
7742     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7743     my ($storage, $format) = $dest->@{qw(storage format)};
7744
7745     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7746
7747     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7748         die "cloning from/to EFI disk requires EFI disk\n"
7749             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7750         die "cloning from/to TPM state requires TPM state\n"
7751             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7752
7753         # This would lead to two device nodes in QEMU pointing to the same backing image!
7754         die "cannot change drive name when cloning disk from/to the same VM\n"
7755             if $use_drive_mirror && $vmid == $newvmid;
7756     }
7757
7758     die "cannot move TPM state while VM is running\n"
7759         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7760
7761     my $newvolid;
7762
7763     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7764     print "$src_drivename " if $src_drivename;
7765     print "($drive->{file})\n";
7766
7767     if (!$full) {
7768         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7769         push @$newvollist, $newvolid;
7770     } else {
7771
7772         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7773         $storeid = $storage if $storage;
7774
7775         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7776
7777         my $name = undef;
7778         my $size = undef;
7779         if (drive_is_cloudinit($drive)) {
7780             $name = "vm-$newvmid-cloudinit";
7781             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7782             if ($scfg->{path}) {
7783                 $name .= ".$dst_format";
7784             }
7785             $snapname = undef;
7786             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7787         } elsif ($dst_drivename eq 'efidisk0') {
7788             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7789         } elsif ($dst_drivename eq 'tpmstate0') {
7790             $dst_format = 'raw';
7791             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7792         } else {
7793             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7794         }
7795         $newvolid = PVE::Storage::vdisk_alloc(
7796             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7797         );
7798         push @$newvollist, $newvolid;
7799
7800         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7801
7802         if (drive_is_cloudinit($drive)) {
7803             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7804             # if this is the case, we have to complete any block-jobs still there from
7805             # previous drive-mirrors
7806             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7807                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7808             }
7809             goto no_data_clone;
7810         }
7811
7812         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7813         if ($use_drive_mirror) {
7814             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7815                 $completion, $qga, $bwlimit);
7816         } else {
7817             # TODO: handle bwlimits
7818             if ($dst_drivename eq 'efidisk0') {
7819                 # the relevant data on the efidisk may be smaller than the source
7820                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7821                 # that is given by the OVMF_VARS.fd
7822                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7823                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7824
7825                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7826
7827                 # better for Ceph if block size is not too small, see bug #3324
7828                 my $bs = 1024*1024;
7829
7830                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7831
7832                 if ($src_format eq 'qcow2' && $snapname) {
7833                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7834                         if !min_version(kvm_user_version(), 6, 2);
7835                     push $cmd->@*, '-l', $snapname;
7836                 }
7837                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7838                 run_command($cmd);
7839             } else {
7840                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7841             }
7842         }
7843     }
7844
7845 no_data_clone:
7846     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7847
7848     my $disk = dclone($drive);
7849     delete $disk->{format};
7850     $disk->{file} = $newvolid;
7851     $disk->{size} = $size if defined($size);
7852
7853     return $disk;
7854 }
7855
7856 sub get_running_qemu_version {
7857     my ($vmid) = @_;
7858     my $res = mon_cmd($vmid, "query-version");
7859     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7860 }
7861
7862 sub qemu_use_old_bios_files {
7863     my ($machine_type) = @_;
7864
7865     return if !$machine_type;
7866
7867     my $use_old_bios_files = undef;
7868
7869     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7870         $machine_type = $1;
7871         $use_old_bios_files = 1;
7872     } else {
7873         my $version = extract_version($machine_type, kvm_user_version());
7874         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7875         # load new efi bios files on migration. So this hack is required to allow
7876         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7877         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7878         $use_old_bios_files = !min_version($version, 2, 4);
7879     }
7880
7881     return ($use_old_bios_files, $machine_type);
7882 }
7883
7884 sub get_efivars_size {
7885     my ($conf, $efidisk) = @_;
7886
7887     my $arch = get_vm_arch($conf);
7888     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7889     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7890     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7891     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7892     return -s $ovmf_vars;
7893 }
7894
7895 sub update_efidisk_size {
7896     my ($conf) = @_;
7897
7898     return if !defined($conf->{efidisk0});
7899
7900     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7901     $disk->{size} = get_efivars_size($conf);
7902     $conf->{efidisk0} = print_drive($disk);
7903
7904     return;
7905 }
7906
7907 sub update_tpmstate_size {
7908     my ($conf) = @_;
7909
7910     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7911     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7912     $conf->{tpmstate0} = print_drive($disk);
7913 }
7914
7915 sub create_efidisk($$$$$$$) {
7916     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7917
7918     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7919     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7920
7921     my $vars_size_b = -s $ovmf_vars;
7922     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7923     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7924     PVE::Storage::activate_volumes($storecfg, [$volid]);
7925
7926     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7927     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7928
7929     return ($volid, $size/1024);
7930 }
7931
7932 sub vm_iothreads_list {
7933     my ($vmid) = @_;
7934
7935     my $res = mon_cmd($vmid, 'query-iothreads');
7936
7937     my $iothreads = {};
7938     foreach my $iothread (@$res) {
7939         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7940     }
7941
7942     return $iothreads;
7943 }
7944
7945 sub scsihw_infos {
7946     my ($conf, $drive) = @_;
7947
7948     my $maxdev = 0;
7949
7950     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7951         $maxdev = 7;
7952     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7953         $maxdev = 1;
7954     } else {
7955         $maxdev = 256;
7956     }
7957
7958     my $controller = int($drive->{index} / $maxdev);
7959     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7960         ? "virtioscsi"
7961         : "scsihw";
7962
7963     return ($maxdev, $controller, $controller_prefix);
7964 }
7965
7966 sub resolve_dst_disk_format {
7967         my ($storecfg, $storeid, $src_volname, $format) = @_;
7968         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7969
7970         if (!$format) {
7971             # if no target format is specified, use the source disk format as hint
7972             if ($src_volname) {
7973                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7974                 $format = qemu_img_format($scfg, $src_volname);
7975             } else {
7976                 return $defFormat;
7977             }
7978         }
7979
7980         # test if requested format is supported - else use default
7981         my $supported = grep { $_ eq $format } @$validFormats;
7982         $format = $defFormat if !$supported;
7983         return $format;
7984 }
7985
7986 # NOTE: if this logic changes, please update docs & possibly gui logic
7987 sub find_vmstate_storage {
7988     my ($conf, $storecfg) = @_;
7989
7990     # first, return storage from conf if set
7991     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7992
7993     my ($target, $shared, $local);
7994
7995     foreach_storage_used_by_vm($conf, sub {
7996         my ($sid) = @_;
7997         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7998         my $dst = $scfg->{shared} ? \$shared : \$local;
7999         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8000     });
8001
8002     # second, use shared storage where VM has at least one disk
8003     # third, use local storage where VM has at least one disk
8004     # fall back to local storage
8005     $target = $shared // $local // 'local';
8006
8007     return $target;
8008 }
8009
8010 sub generate_uuid {
8011     my ($uuid, $uuid_str);
8012     UUID::generate($uuid);
8013     UUID::unparse($uuid, $uuid_str);
8014     return $uuid_str;
8015 }
8016
8017 sub generate_smbios1_uuid {
8018     return "uuid=".generate_uuid();
8019 }
8020
8021 sub nbd_stop {
8022     my ($vmid) = @_;
8023
8024     mon_cmd($vmid, 'nbd-server-stop');
8025 }
8026
8027 sub create_reboot_request {
8028     my ($vmid) = @_;
8029     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8030         or die "failed to create reboot trigger file: $!\n";
8031     close($fh);
8032 }
8033
8034 sub clear_reboot_request {
8035     my ($vmid) = @_;
8036     my $path = "/run/qemu-server/$vmid.reboot";
8037     my $res = 0;
8038
8039     $res = unlink($path);
8040     die "could not remove reboot request for $vmid: $!"
8041         if !$res && $! != POSIX::ENOENT;
8042
8043     return $res;
8044 }
8045
8046 sub bootorder_from_legacy {
8047     my ($conf, $bootcfg) = @_;
8048
8049     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8050     my $bootindex_hash = {};
8051     my $i = 1;
8052     foreach my $o (split(//, $boot)) {
8053         $bootindex_hash->{$o} = $i*100;
8054         $i++;
8055     }
8056
8057     my $bootorder = {};
8058
8059     PVE::QemuConfig->foreach_volume($conf, sub {
8060         my ($ds, $drive) = @_;
8061
8062         if (drive_is_cdrom ($drive, 1)) {
8063             if ($bootindex_hash->{d}) {
8064                 $bootorder->{$ds} = $bootindex_hash->{d};
8065                 $bootindex_hash->{d} += 1;
8066             }
8067         } elsif ($bootindex_hash->{c}) {
8068             $bootorder->{$ds} = $bootindex_hash->{c}
8069                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8070             $bootindex_hash->{c} += 1;
8071         }
8072     });
8073
8074     if ($bootindex_hash->{n}) {
8075         for (my $i = 0; $i < $MAX_NETS; $i++) {
8076             my $netname = "net$i";
8077             next if !$conf->{$netname};
8078             $bootorder->{$netname} = $bootindex_hash->{n};
8079             $bootindex_hash->{n} += 1;
8080         }
8081     }
8082
8083     return $bootorder;
8084 }
8085
8086 # Generate default device list for 'boot: order=' property. Matches legacy
8087 # default boot order, but with explicit device names. This is important, since
8088 # the fallback for when neither 'order' nor the old format is specified relies
8089 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8090 sub get_default_bootdevices {
8091     my ($conf) = @_;
8092
8093     my @ret = ();
8094
8095     # harddisk
8096     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8097     push @ret, $first if $first;
8098
8099     # cdrom
8100     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8101     push @ret, $first if $first;
8102
8103     # network
8104     for (my $i = 0; $i < $MAX_NETS; $i++) {
8105         my $netname = "net$i";
8106         next if !$conf->{$netname};
8107         push @ret, $netname;
8108         last;
8109     }
8110
8111     return \@ret;
8112 }
8113
8114 sub device_bootorder {
8115     my ($conf) = @_;
8116
8117     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8118
8119     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8120
8121     my $bootorder = {};
8122     if (!defined($boot) || $boot->{legacy}) {
8123         $bootorder = bootorder_from_legacy($conf, $boot);
8124     } elsif ($boot->{order}) {
8125         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8126         for my $dev (PVE::Tools::split_list($boot->{order})) {
8127             $bootorder->{$dev} = $i++;
8128         }
8129     }
8130
8131     return $bootorder;
8132 }
8133
8134 sub register_qmeventd_handle {
8135     my ($vmid) = @_;
8136
8137     my $fh;
8138     my $peer = "/var/run/qmeventd.sock";
8139     my $count = 0;
8140
8141     for (;;) {
8142         $count++;
8143         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8144         last if $fh;
8145         if ($! != EINTR && $! != EAGAIN) {
8146             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8147         }
8148         if ($count > 4) {
8149             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8150               . "after $count retries\n";
8151         }
8152         usleep(25000);
8153     }
8154
8155     # send handshake to mark VM as backing up
8156     print $fh to_json({vzdump => {vmid => "$vmid"}});
8157
8158     # return handle to be closed later when inhibit is no longer required
8159     return $fh;
8160 }
8161
8162 # bash completion helper
8163
8164 sub complete_backup_archives {
8165     my ($cmdname, $pname, $cvalue) = @_;
8166
8167     my $cfg = PVE::Storage::config();
8168
8169     my $storeid;
8170
8171     if ($cvalue =~ m/^([^:]+):/) {
8172         $storeid = $1;
8173     }
8174
8175     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8176
8177     my $res = [];
8178     foreach my $id (keys %$data) {
8179         foreach my $item (@{$data->{$id}}) {
8180             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8181             push @$res, $item->{volid} if defined($item->{volid});
8182         }
8183     }
8184
8185     return $res;
8186 }
8187
8188 my $complete_vmid_full = sub {
8189     my ($running) = @_;
8190
8191     my $idlist = vmstatus();
8192
8193     my $res = [];
8194
8195     foreach my $id (keys %$idlist) {
8196         my $d = $idlist->{$id};
8197         if (defined($running)) {
8198             next if $d->{template};
8199             next if $running && $d->{status} ne 'running';
8200             next if !$running && $d->{status} eq 'running';
8201         }
8202         push @$res, $id;
8203
8204     }
8205     return $res;
8206 };
8207
8208 sub complete_vmid {
8209     return &$complete_vmid_full();
8210 }
8211
8212 sub complete_vmid_stopped {
8213     return &$complete_vmid_full(0);
8214 }
8215
8216 sub complete_vmid_running {
8217     return &$complete_vmid_full(1);
8218 }
8219
8220 sub complete_storage {
8221
8222     my $cfg = PVE::Storage::config();
8223     my $ids = $cfg->{ids};
8224
8225     my $res = [];
8226     foreach my $sid (keys %$ids) {
8227         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8228         next if !$ids->{$sid}->{content}->{images};
8229         push @$res, $sid;
8230     }
8231
8232     return $res;
8233 }
8234
8235 sub complete_migration_storage {
8236     my ($cmd, $param, $current_value, $all_args) = @_;
8237
8238     my $targetnode = @$all_args[1];
8239
8240     my $cfg = PVE::Storage::config();
8241     my $ids = $cfg->{ids};
8242
8243     my $res = [];
8244     foreach my $sid (keys %$ids) {
8245         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8246         next if !$ids->{$sid}->{content}->{images};
8247         push @$res, $sid;
8248     }
8249
8250     return $res;
8251 }
8252
8253 sub vm_is_paused {
8254     my ($vmid) = @_;
8255     my $qmpstatus = eval {
8256         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8257         mon_cmd($vmid, "query-status");
8258     };
8259     warn "$@\n" if $@;
8260     return $qmpstatus && $qmpstatus->{status} eq "paused";
8261 }
8262
8263 sub check_volume_storage_type {
8264     my ($storecfg, $vol) = @_;
8265
8266     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8267     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8268     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8269
8270     die "storage '$storeid' does not support content-type '$vtype'\n"
8271         if !$scfg->{content}->{$vtype};
8272
8273     return 1;
8274 }
8275
8276 1;