PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file('/qemu-server/',
 106                   \&parse_vm_config,
 107                   \&write_vm_config);
 108
 109 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 110     description => "Some command save/restore state from this location.",
 111     type => 'string',
 112     maxLength => 128,
 113     optional => 1,
 114 });
 115
 116 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 117         description => "Specifies the Qemu machine type.",
 118         type => 'string',
 119         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 120         maxLength => 40,
 121         optional => 1,
 122 });
 123
 124 #no warnings 'redefine';
 125
 126 my $nodename_cache;
 127 sub nodename {
 128     $nodename_cache //= PVE::INotify::nodename();
 129     return $nodename_cache;
 130 }
 131
 132 my $watchdog_fmt = {
 133     model => {
 134         default_key => 1,
 135         type => 'string',
 136         enum => [qw(i6300esb ib700)],
 137         description => "Watchdog type to emulate.",
 138         default => 'i6300esb',
 139         optional => 1,
 140     },
 141     action => {
 142         type => 'string',
 143         enum => [qw(reset shutdown poweroff pause debug none)],
 144         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 145         optional => 1,
 146     },
 147 };
 148 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 149
 150 my $agent_fmt = {
 151     enabled => {
 152         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 153         type => 'boolean',
 154         default => 0,
 155         default_key => 1,
 156     },
 157     fstrim_cloned_disks => {
 158         description => "Run fstrim after moving a disk or migrating the VM.",
 159         type => 'boolean',
 160         optional => 1,
 161         default => 0
 162     },
 163     type => {
 164         description => "Select the agent type",
 165         type => 'string',
 166         default => 'virtio',
 167         optional => 1,
 168         enum => [qw(virtio isa)],
 169     },
 170 };
 171
 172 my $vga_fmt = {
 173     type => {
 174         description => "Select the VGA type.",
 175         type => 'string',
 176         default => 'std',
 177         optional => 1,
 178         default_key => 1,
 179         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 180     },
 181     memory => {
 182         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 183         type => 'integer',
 184         optional => 1,
 185         minimum => 4,
 186         maximum => 512,
 187     },
 188 };
 189
 190 my $ivshmem_fmt = {
 191     size => {
 192         type => 'integer',
 193         minimum => 1,
 194         description => "The size of the file in MB.",
 195     },
 196     name => {
 197         type => 'string',
 198         pattern => '[a-zA-Z0-9\-]+',
 199         optional => 1,
 200         format_description => 'string',
 201         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 202     },
 203 };
 204
 205 my $audio_fmt = {
 206     device => {
 207         type => 'string',
 208         enum => [qw(ich9-intel-hda intel-hda AC97)],
 209         description =>  "Configure an audio device."
 210     },
 211     driver =>  {
 212         type => 'string',
 213         enum => ['spice', 'none'],
 214         default => 'spice',
 215         optional => 1,
 216         description => "Driver backend for the audio device."
 217     },
 218 };
 219
 220 my $spice_enhancements_fmt = {
 221     foldersharing => {
 222         type => 'boolean',
 223         optional => 1,
 224         default => '0',
 225         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 226     },
 227     videostreaming =>  {
 228         type => 'string',
 229         enum => ['off', 'all', 'filter'],
 230         default => 'off',
 231         optional => 1,
 232         description => "Enable video streaming. Uses compression for detected video streams."
 233     },
 234 };
 235
 236 my $rng_fmt = {
 237     source => {
 238         type => 'string',
 239         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 240         default_key => 1,
 241         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 242             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 243             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 244             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 245             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 246             ." a hardware RNG from the host.",
 247     },
 248     max_bytes => {
 249         type => 'integer',
 250         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 251             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 252             ." `0` to disable limiting (potentially dangerous!).",
 253         optional => 1,
 254
 255         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 256         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 257         # reading from /dev/urandom
 258         default => 1024,
 259     },
 260     period => {
 261         type => 'integer',
 262         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 263             ." the guest to retrieve another 'max_bytes' of entropy.",
 264         optional => 1,
 265         default => 1000,
 266     },
 267 };
 268
 269 my $meta_info_fmt = {
 270     'ctime' => {
 271         type => 'integer',
 272         description => "The guest creation timestamp as UNIX epoch time",
 273         minimum => 0,
 274         optional => 1,
 275     },
 276     'creation-qemu' => {
 277         type => 'string',
 278         description => "The QEMU (machine) version from the time this VM was created.",
 279         pattern => '\d+(\.\d+)+',
 280         optional => 1,
 281     },
 282 };
 283
 284 my $confdesc = {
 285     onboot => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Specifies whether a VM will be started during system bootup.",
 289         default => 0,
 290     },
 291     autostart => {
 292         optional => 1,
 293         type => 'boolean',
 294         description => "Automatic restart after crash (currently ignored).",
 295         default => 0,
 296     },
 297     hotplug => {
 298         optional => 1,
 299         type => 'string', format => 'pve-hotplug-features',
 300         description => "Selectively enable hotplug features. This is a comma separated list of"
 301             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 302             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 303         default => 'network,disk,usb',
 304     },
 305     reboot => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 309         default => 1,
 310     },
 311     lock => {
 312         optional => 1,
 313         type => 'string',
 314         description => "Lock/unlock the VM.",
 315         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 316     },
 317     cpulimit => {
 318         optional => 1,
 319         type => 'number',
 320         description => "Limit of CPU usage.",
 321         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 322             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 323         minimum => 0,
 324         maximum => 128,
 325         default => 0,
 326     },
 327     cpuunits => {
 328         optional => 1,
 329         type => 'integer',
 330         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 331         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 332             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 333             ." weights of all the other running VMs.",
 334         minimum => 1,
 335         maximum => 262144,
 336         default => 'cgroup v1: 1024, cgroup v2: 100',
 337     },
 338     memory => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 342             ." you use the balloon device.",
 343         minimum => 16,
 344         default => 512,
 345     },
 346     balloon => {
 347         optional => 1,
 348         type => 'integer',
 349         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 350         minimum => 0,
 351     },
 352     shares => {
 353         optional => 1,
 354         type => 'integer',
 355         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 356             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 357             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 358         minimum => 0,
 359         maximum => 50000,
 360         default => 1000,
 361     },
 362     keyboard => {
 363         optional => 1,
 364         type => 'string',
 365         description => "Keyboard layout for VNC server. This option is generally not required and"
 366         ." is often better handled from within the guest OS.",
 367         enum => PVE::Tools::kvmkeymaplist(),
 368         default => undef,
 369     },
 370     name => {
 371         optional => 1,
 372         type => 'string', format => 'dns-name',
 373         description => "Set a name for the VM. Only used on the configuration web interface.",
 374     },
 375     scsihw => {
 376         optional => 1,
 377         type => 'string',
 378         description => "SCSI controller model",
 379         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 380         default => 'lsi',
 381     },
 382     description => {
 383         optional => 1,
 384         type => 'string',
 385         description => "Description for the VM. Shown in the web-interface VM's summary."
 386             ." This is saved as comment inside the configuration file.",
 387         maxLength => 1024 * 8,
 388     },
 389     ostype => {
 390         optional => 1,
 391         type => 'string',
 392         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 393         description => "Specify guest operating system.",
 394         verbose_description => <<EODESC,
 395 Specify guest operating system. This is used to enable special
 396 optimization/features for specific operating systems:
 397
 398 [horizontal]
 399 other;; unspecified OS
 400 wxp;; Microsoft Windows XP
 401 w2k;; Microsoft Windows 2000
 402 w2k3;; Microsoft Windows 2003
 403 w2k8;; Microsoft Windows 2008
 404 wvista;; Microsoft Windows Vista
 405 win7;; Microsoft Windows 7
 406 win8;; Microsoft Windows 8/2012/2012r2
 407 win10;; Microsoft Windows 10/2016/2019
 408 win11;; Microsoft Windows 11/2022
 409 l24;; Linux 2.4 Kernel
 410 l26;; Linux 2.6 - 5.X Kernel
 411 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 412 EODESC
 413     },
 414     boot => {
 415         optional => 1,
 416         type => 'string', format => 'pve-qm-boot',
 417         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 418             ." key or 'legacy=' is deprecated.",
 419     },
 420     bootdisk => {
 421         optional => 1,
 422         type => 'string', format => 'pve-qm-bootdisk',
 423         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 424         pattern => '(ide|sata|scsi|virtio)\d+',
 425     },
 426     smp => {
 427         optional => 1,
 428         type => 'integer',
 429         description => "The number of CPUs. Please use option -sockets instead.",
 430         minimum => 1,
 431         default => 1,
 432     },
 433     sockets => {
 434         optional => 1,
 435         type => 'integer',
 436         description => "The number of CPU sockets.",
 437         minimum => 1,
 438         default => 1,
 439     },
 440     cores => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "The number of cores per socket.",
 444         minimum => 1,
 445         default => 1,
 446     },
 447     numa => {
 448         optional => 1,
 449         type => 'boolean',
 450         description => "Enable/disable NUMA.",
 451         default => 0,
 452     },
 453     hugepages => {
 454         optional => 1,
 455         type => 'string',
 456         description => "Enable/disable hugepages memory.",
 457         enum => [qw(any 2 1024)],
 458     },
 459     keephugepages => {
 460         optional => 1,
 461         type => 'boolean',
 462         default => 0,
 463         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 464             ." after VM shutdown and can be used for subsequent starts.",
 465     },
 466     vcpus => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "Number of hotplugged vcpus.",
 470         minimum => 1,
 471         default => 0,
 472     },
 473     acpi => {
 474         optional => 1,
 475         type => 'boolean',
 476         description => "Enable/disable ACPI.",
 477         default => 1,
 478     },
 479     agent => {
 480         optional => 1,
 481         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 482         type => 'string',
 483         format => $agent_fmt,
 484     },
 485     kvm => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Enable/disable KVM hardware virtualization.",
 489         default => 1,
 490     },
 491     tdf => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable time drift fix.",
 495         default => 0,
 496     },
 497     localtime => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 501             ." the `ostype` indicates a Microsoft Windows OS.",
 502     },
 503     freeze => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 507     },
 508     vga => {
 509         optional => 1,
 510         type => 'string', format => $vga_fmt,
 511         description => "Configure the VGA hardware.",
 512         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 513             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 514             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 515             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 516             ." display server. For win* OS you can select how many independent displays you want,"
 517             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 518             ." using a serial device as terminal.",
 519     },
 520     watchdog => {
 521         optional => 1,
 522         type => 'string', format => 'pve-qm-watchdog',
 523         description => "Create a virtual hardware watchdog device.",
 524         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 525             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 526             ." else the watchdog will reset the guest (or execute the respective action specified)",
 527     },
 528     startdate => {
 529         optional => 1,
 530         type => 'string',
 531         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 532         description => "Set the initial date of the real time clock. Valid format for date are:"
 533             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 534         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 535         default => 'now',
 536     },
 537     startup =>  get_standard_option('pve-startup-order'),
 538     template => {
 539         optional => 1,
 540         type => 'boolean',
 541         description => "Enable/disable Template.",
 542         default => 0,
 543     },
 544     args => {
 545         optional => 1,
 546         type => 'string',
 547         description => "Arbitrary arguments passed to kvm.",
 548         verbose_description => <<EODESCR,
 549 Arbitrary arguments passed to kvm, for example:
 550
 551 args: -no-reboot -no-hpet
 552
 553 NOTE: this option is for experts only.
 554 EODESCR
 555     },
 556     tablet => {
 557         optional => 1,
 558         type => 'boolean',
 559         default => 1,
 560         description => "Enable/disable the USB tablet device.",
 561         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 562             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 563             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 564             ." may consider disabling this to save some context switches. This is turned off by"
 565             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 566     },
 567     migrate_speed => {
 568         optional => 1,
 569         type => 'integer',
 570         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 571         minimum => 0,
 572         default => 0,
 573     },
 574     migrate_downtime => {
 575         optional => 1,
 576         type => 'number',
 577         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 578         minimum => 0,
 579         default => 0.1,
 580     },
 581     cdrom => {
 582         optional => 1,
 583         type => 'string', format => 'pve-qm-ide',
 584         typetext => '<volume>',
 585         description => "This is an alias for option -ide2",
 586     },
 587     cpu => {
 588         optional => 1,
 589         description => "Emulated CPU type.",
 590         type => 'string',
 591         format => 'pve-vm-cpu-conf',
 592     },
 593     parent => get_standard_option('pve-snapshot-name', {
 594         optional => 1,
 595         description => "Parent snapshot name. This is used internally, and should not be modified.",
 596     }),
 597     snaptime => {
 598         optional => 1,
 599         description => "Timestamp for snapshots.",
 600         type => 'integer',
 601         minimum => 0,
 602     },
 603     vmstate => {
 604         optional => 1,
 605         type => 'string', format => 'pve-volume-id',
 606         description => "Reference to a volume which stores the VM state. This is used internally"
 607             ." for snapshots.",
 608     },
 609     vmstatestorage => get_standard_option('pve-storage-id', {
 610         description => "Default storage for VM state volumes/files.",
 611         optional => 1,
 612     }),
 613     runningmachine => get_standard_option('pve-qemu-machine', {
 614         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 615             ." for snapshots.",
 616     }),
 617     runningcpu => {
 618         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 619             ." internally for snapshots.",
 620         optional => 1,
 621         type => 'string',
 622         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 623         format_description => 'QEMU -cpu parameter'
 624     },
 625     machine => get_standard_option('pve-qemu-machine'),
 626     arch => {
 627         description => "Virtual processor architecture. Defaults to the host.",
 628         optional => 1,
 629         type => 'string',
 630         enum => [qw(x86_64 aarch64)],
 631     },
 632     smbios1 => {
 633         description => "Specify SMBIOS type 1 fields.",
 634         type => 'string', format => 'pve-qm-smbios1',
 635         maxLength => 512,
 636         optional => 1,
 637     },
 638     protection => {
 639         optional => 1,
 640         type => 'boolean',
 641         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 642             ." remove disk operations.",
 643         default => 0,
 644     },
 645     bios => {
 646         optional => 1,
 647         type => 'string',
 648         enum => [ qw(seabios ovmf) ],
 649         description => "Select BIOS implementation.",
 650         default => 'seabios',
 651     },
 652     vmgenid => {
 653         type => 'string',
 654         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 655         format_description => 'UUID',
 656         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 657             ." to disable explicitly.",
 658         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 659             ." value identifier to the guest OS. This allows to notify the guest operating system"
 660             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 661             ." execution or creation from a template). The guest operating system notices the"
 662             ." change, and is then able to react as appropriate by marking its copies of"
 663             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 664             ."Note that auto-creation only works when done through API/CLI create or update methods"
 665             .", but not when manually editing the config file.",
 666         default => "1 (autogenerated)",
 667         optional => 1,
 668     },
 669     hookscript => {
 670         type => 'string',
 671         format => 'pve-volume-id',
 672         optional => 1,
 673         description => "Script that will be executed during various steps in the vms lifetime.",
 674     },
 675     ivshmem => {
 676         type => 'string',
 677         format => $ivshmem_fmt,
 678         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 679             ." the host.",
 680         optional => 1,
 681     },
 682     audio0 => {
 683         type => 'string',
 684         format => $audio_fmt,
 685         description => "Configure a audio device, useful in combination with QXL/Spice.",
 686         optional => 1
 687     },
 688     spice_enhancements => {
 689         type => 'string',
 690         format => $spice_enhancements_fmt,
 691         description => "Configure additional enhancements for SPICE.",
 692         optional => 1
 693     },
 694     tags => {
 695         type => 'string', format => 'pve-tag-list',
 696         description => 'Tags of the VM. This is only meta information.',
 697         optional => 1,
 698     },
 699     rng0 => {
 700         type => 'string',
 701         format => $rng_fmt,
 702         description => "Configure a VirtIO-based Random Number Generator.",
 703         optional => 1,
 704     },
 705     meta => {
 706         type => 'string',
 707         format => $meta_info_fmt,
 708         description => "Some (read-only) meta-information about this guest.",
 709         optional => 1,
 710     },
 711     affinity => {
 712         type => 'string', format => 'pve-cpuset',
 713         description => "List of host cores used to execute guest processes.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.',
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.',
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1036 sub pve_verify_cpuset {
1037     my ($set_text, $noerr) = @_;
1038
1039     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1040
1041     if ($@) {
1042         return if $noerr;
1043         die "unable to parse cpuset option\n";
1044     }
1045
1046     return PVE::CpuSet->new($members)->short_string();
1047 }
1048
1049 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1050 sub verify_volume_id_or_qm_path {
1051     my ($volid, $noerr) = @_;
1052
1053     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1054
1055     return verify_volume_id_or_absolute_path($volid, $noerr);
1056 }
1057
1058 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1059 sub verify_volume_id_or_absolute_path {
1060     my ($volid, $noerr) = @_;
1061
1062     return $volid if $volid =~ m|^/|;
1063
1064     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1065     if ($@) {
1066         return if $noerr;
1067         die $@;
1068     }
1069     return $volid;
1070 }
1071
1072 my $usb_fmt = {
1073     host => {
1074         default_key => 1,
1075         type => 'string', format => 'pve-qm-usb-device',
1076         format_description => 'HOSTUSBDEVICE|spice',
1077         description => <<EODESCR,
1078 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1079
1080  'bus-port(.port)*' (decimal numbers) or
1081  'vendor_id:product_id' (hexadeciaml numbers) or
1082  'spice'
1083
1084 You can use the 'lsusb -t' command to list existing usb devices.
1085
1086 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1087 machines - use with special care.
1088
1089 The value 'spice' can be used to add a usb redirection devices for spice.
1090 EODESCR
1091     },
1092     usb3 => {
1093         optional => 1,
1094         type => 'boolean',
1095         description => "Specifies whether if given host option is a USB3 device or port."
1096             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1097             ." is irrelevant (all devices are plugged into a xhci controller).",
1098         default => 0,
1099     },
1100 };
1101
1102 my $usbdesc = {
1103     optional => 1,
1104     type => 'string', format => $usb_fmt,
1105     description => "Configure an USB device (n is 0 to 4).",
1106 };
1107 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1108
1109 my $serialdesc = {
1110         optional => 1,
1111         type => 'string',
1112         pattern => '(/dev/.+|socket)',
1113         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1114         verbose_description =>  <<EODESCR,
1115 Create a serial device inside the VM (n is 0 to 3), and pass through a
1116 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1117 host side (use 'qm terminal' to open a terminal connection).
1118
1119 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1120 use with special care.
1121
1122 CAUTION: Experimental! User reported problems with this option.
1123 EODESCR
1124 };
1125
1126 my $paralleldesc= {
1127         optional => 1,
1128         type => 'string',
1129         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1130         description =>  "Map host parallel devices (n is 0 to 2).",
1131         verbose_description =>  <<EODESCR,
1132 Map host parallel devices (n is 0 to 2).
1133
1134 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1135 machines - use with special care.
1136
1137 CAUTION: Experimental! User reported problems with this option.
1138 EODESCR
1139 };
1140
1141 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1142     $confdesc->{"parallel$i"} = $paralleldesc;
1143 }
1144
1145 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1146     $confdesc->{"serial$i"} = $serialdesc;
1147 }
1148
1149 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1150     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1151 }
1152
1153 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1154     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1155 }
1156
1157 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1158     $confdesc->{"usb$i"} = $usbdesc;
1159 }
1160
1161 my $boot_fmt = {
1162     legacy => {
1163         optional => 1,
1164         default_key => 1,
1165         type => 'string',
1166         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1167                      . " Deprecated, use 'order=' instead.",
1168         pattern => '[acdn]{1,4}',
1169         format_description => "[acdn]{1,4}",
1170
1171         # note: this is also the fallback if boot: is not given at all
1172         default => 'cdn',
1173     },
1174     order => {
1175         optional => 1,
1176         type => 'string',
1177         format => 'pve-qm-bootdev-list',
1178         format_description => "device[;device...]",
1179         description => <<EODESC,
1180 The guest will attempt to boot from devices in the order they appear here.
1181
1182 Disks, optical drives and passed-through storage USB devices will be directly
1183 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1184 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1185
1186 Note that only devices in this list will be marked as bootable and thus loaded
1187 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1188 (e.g. software-raid), you need to specify all of them here.
1189
1190 Overrides the deprecated 'legacy=[acdn]*' value when given.
1191 EODESC
1192     },
1193 };
1194 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1195
1196 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1197 sub verify_bootdev {
1198     my ($dev, $noerr) = @_;
1199
1200     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1201     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1202
1203     my $check = sub {
1204         my ($base) = @_;
1205         return 0 if $dev !~ m/^$base\d+$/;
1206         return 0 if !$confdesc->{$dev};
1207         return 1;
1208     };
1209
1210     return $dev if $check->("net");
1211     return $dev if $check->("usb");
1212     return $dev if $check->("hostpci");
1213
1214     return if $noerr;
1215     die "invalid boot device '$dev'\n";
1216 }
1217
1218 sub print_bootorder {
1219     my ($devs) = @_;
1220     return "" if !@$devs;
1221     my $data = { order => join(';', @$devs) };
1222     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1223 }
1224
1225 my $kvm_api_version = 0;
1226
1227 sub kvm_version {
1228     return $kvm_api_version if $kvm_api_version;
1229
1230     open my $fh, '<', '/dev/kvm' or return;
1231
1232     # 0xae00 => KVM_GET_API_VERSION
1233     $kvm_api_version = ioctl($fh, 0xae00, 0);
1234     close($fh);
1235
1236     return $kvm_api_version;
1237 }
1238
1239 my $kvm_user_version = {};
1240 my $kvm_mtime = {};
1241
1242 sub kvm_user_version {
1243     my ($binary) = @_;
1244
1245     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1246     my $st = stat($binary);
1247
1248     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1249     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1250         $cachedmtime == $st->mtime;
1251
1252     $kvm_user_version->{$binary} = 'unknown';
1253     $kvm_mtime->{$binary} = $st->mtime;
1254
1255     my $code = sub {
1256         my $line = shift;
1257         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1258             $kvm_user_version->{$binary} = $2;
1259         }
1260     };
1261
1262     eval { run_command([$binary, '--version'], outfunc => $code); };
1263     warn $@ if $@;
1264
1265     return $kvm_user_version->{$binary};
1266
1267 }
1268 my sub extract_version {
1269     my ($machine_type, $version) = @_;
1270     $version = kvm_user_version() if !defined($version);
1271     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1272 }
1273
1274 sub kernel_has_vhost_net {
1275     return -c '/dev/vhost-net';
1276 }
1277
1278 sub option_exists {
1279     my $key = shift;
1280     return defined($confdesc->{$key});
1281 }
1282
1283 my $cdrom_path;
1284 sub get_cdrom_path {
1285
1286     return  $cdrom_path if $cdrom_path;
1287
1288     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1289     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1290     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1291 }
1292
1293 sub get_iso_path {
1294     my ($storecfg, $vmid, $cdrom) = @_;
1295
1296     if ($cdrom eq 'cdrom') {
1297         return get_cdrom_path();
1298     } elsif ($cdrom eq 'none') {
1299         return '';
1300     } elsif ($cdrom =~ m|^/|) {
1301         return $cdrom;
1302     } else {
1303         return PVE::Storage::path($storecfg, $cdrom);
1304     }
1305 }
1306
1307 # try to convert old style file names to volume IDs
1308 sub filename_to_volume_id {
1309     my ($vmid, $file, $media) = @_;
1310
1311      if (!($file eq 'none' || $file eq 'cdrom' ||
1312           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1313
1314         return if $file =~ m|/|;
1315
1316         if ($media && $media eq 'cdrom') {
1317             $file = "local:iso/$file";
1318         } else {
1319             $file = "local:$vmid/$file";
1320         }
1321     }
1322
1323     return $file;
1324 }
1325
1326 sub verify_media_type {
1327     my ($opt, $vtype, $media) = @_;
1328
1329     return if !$media;
1330
1331     my $etype;
1332     if ($media eq 'disk') {
1333         $etype = 'images';
1334     } elsif ($media eq 'cdrom') {
1335         $etype = 'iso';
1336     } else {
1337         die "internal error";
1338     }
1339
1340     return if ($vtype eq $etype);
1341
1342     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1343 }
1344
1345 sub cleanup_drive_path {
1346     my ($opt, $storecfg, $drive) = @_;
1347
1348     # try to convert filesystem paths to volume IDs
1349
1350     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1351         ($drive->{file} !~ m|^/dev/.+|) &&
1352         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1353         ($drive->{file} !~ m/^\d+$/)) {
1354         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1355         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1356             if !$vtype;
1357         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1358         verify_media_type($opt, $vtype, $drive->{media});
1359         $drive->{file} = $volid;
1360     }
1361
1362     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1363 }
1364
1365 sub parse_hotplug_features {
1366     my ($data) = @_;
1367
1368     my $res = {};
1369
1370     return $res if $data eq '0';
1371
1372     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1373
1374     foreach my $feature (PVE::Tools::split_list($data)) {
1375         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1376             $res->{$1} = 1;
1377         } else {
1378             die "invalid hotplug feature '$feature'\n";
1379         }
1380     }
1381     return $res;
1382 }
1383
1384 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1385 sub pve_verify_hotplug_features {
1386     my ($value, $noerr) = @_;
1387
1388     return $value if parse_hotplug_features($value);
1389
1390     return if $noerr;
1391
1392     die "unable to parse hotplug option\n";
1393 }
1394
1395 sub scsi_inquiry {
1396     my($fh, $noerr) = @_;
1397
1398     my $SG_IO = 0x2285;
1399     my $SG_GET_VERSION_NUM = 0x2282;
1400
1401     my $versionbuf = "\x00" x 8;
1402     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1403     if (!$ret) {
1404         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1405         return;
1406     }
1407     my $version = unpack("I", $versionbuf);
1408     if ($version < 30000) {
1409         die "scsi generic interface too old\n"  if !$noerr;
1410         return;
1411     }
1412
1413     my $buf = "\x00" x 36;
1414     my $sensebuf = "\x00" x 8;
1415     my $cmd = pack("C x3 C x1", 0x12, 36);
1416
1417     # see /usr/include/scsi/sg.h
1418     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1419
1420     my $packet = pack(
1421         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1422     );
1423
1424     $ret = ioctl($fh, $SG_IO, $packet);
1425     if (!$ret) {
1426         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1427         return;
1428     }
1429
1430     my @res = unpack($sg_io_hdr_t, $packet);
1431     if ($res[17] || $res[18]) {
1432         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1433         return;
1434     }
1435
1436     my $res = {};
1437     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1438
1439     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1440     $res->{type} &= 0x1F;
1441
1442     return $res;
1443 }
1444
1445 sub path_is_scsi {
1446     my ($path) = @_;
1447
1448     my $fh = IO::File->new("+<$path") || return;
1449     my $res = scsi_inquiry($fh, 1);
1450     close($fh);
1451
1452     return $res;
1453 }
1454
1455 sub print_tabletdevice_full {
1456     my ($conf, $arch) = @_;
1457
1458     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1459
1460     # we use uhci for old VMs because tablet driver was buggy in older qemu
1461     my $usbbus;
1462     if ($q35 || $arch eq 'aarch64') {
1463         $usbbus = 'ehci';
1464     } else {
1465         $usbbus = 'uhci';
1466     }
1467
1468     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1469 }
1470
1471 sub print_keyboarddevice_full {
1472     my ($conf, $arch) = @_;
1473
1474     return if $arch ne 'aarch64';
1475
1476     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1477 }
1478
1479 my sub get_drive_id {
1480     my ($drive) = @_;
1481     return "$drive->{interface}$drive->{index}";
1482 }
1483
1484 sub print_drivedevice_full {
1485     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1486
1487     my $device = '';
1488     my $maxdev = 0;
1489
1490     my $drive_id = get_drive_id($drive);
1491     if ($drive->{interface} eq 'virtio') {
1492         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1493         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1494         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1495     } elsif ($drive->{interface} eq 'scsi') {
1496
1497         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1498         my $unit = $drive->{index} % $maxdev;
1499         my $devicetype = 'hd';
1500         my $path = '';
1501         if (drive_is_cdrom($drive)) {
1502             $devicetype = 'cd';
1503         } else {
1504             if ($drive->{file} =~ m|^/|) {
1505                 $path = $drive->{file};
1506                 if (my $info = path_is_scsi($path)) {
1507                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1508                         $devicetype = 'block';
1509                     } elsif ($info->{type} == 1) { # tape
1510                         $devicetype = 'generic';
1511                     }
1512                 }
1513             } else {
1514                  $path = PVE::Storage::path($storecfg, $drive->{file});
1515             }
1516
1517             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1518             my $version = extract_version($machine_type, kvm_user_version());
1519             if ($path =~ m/^iscsi\:\/\// &&
1520                !min_version($version, 4, 1)) {
1521                 $devicetype = 'generic';
1522             }
1523         }
1524
1525         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1526             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1527         } else {
1528             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1529                 .",lun=$drive->{index}";
1530         }
1531         $device .= ",drive=drive-$drive_id,id=$drive_id";
1532
1533         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1534             $device .= ",rotation_rate=1";
1535         }
1536         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1537
1538     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1539         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1540         my $controller = int($drive->{index} / $maxdev);
1541         my $unit = $drive->{index} % $maxdev;
1542         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1543
1544         $device = "ide-$devicetype";
1545         if ($drive->{interface} eq 'ide') {
1546             $device .= ",bus=ide.$controller,unit=$unit";
1547         } else {
1548             $device .= ",bus=ahci$controller.$unit";
1549         }
1550         $device .= ",drive=drive-$drive_id,id=$drive_id";
1551
1552         if ($devicetype eq 'hd') {
1553             if (my $model = $drive->{model}) {
1554                 $model = URI::Escape::uri_unescape($model);
1555                 $device .= ",model=$model";
1556             }
1557             if ($drive->{ssd}) {
1558                 $device .= ",rotation_rate=1";
1559             }
1560         }
1561         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1562     } elsif ($drive->{interface} eq 'usb') {
1563         die "implement me";
1564         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1565     } else {
1566         die "unsupported interface type";
1567     }
1568
1569     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1570
1571     if (my $serial = $drive->{serial}) {
1572         $serial = URI::Escape::uri_unescape($serial);
1573         $device .= ",serial=$serial";
1574     }
1575
1576
1577     return $device;
1578 }
1579
1580 sub get_initiator_name {
1581     my $initiator;
1582
1583     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1584     while (defined(my $line = <$fh>)) {
1585         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1586         $initiator = $1;
1587         last;
1588     }
1589     $fh->close();
1590
1591     return $initiator;
1592 }
1593
1594 sub print_drive_commandline_full {
1595     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1596
1597     my $path;
1598     my $volid = $drive->{file};
1599     my $format = $drive->{format};
1600     my $drive_id = get_drive_id($drive);
1601
1602     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1603     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1604
1605     if (drive_is_cdrom($drive)) {
1606         $path = get_iso_path($storecfg, $vmid, $volid);
1607         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1608     } else {
1609         if ($storeid) {
1610             $path = PVE::Storage::path($storecfg, $volid);
1611             $format //= qemu_img_format($scfg, $volname);
1612         } else {
1613             $path = $volid;
1614             $format //= "raw";
1615         }
1616    }
1617
1618    my $is_rbd = $path =~ m/^rbd:/;
1619
1620     my $opts = '';
1621     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1622     foreach my $o (@qemu_drive_options) {
1623         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1624     }
1625
1626     # snapshot only accepts on|off
1627     if (defined($drive->{snapshot})) {
1628         my $v = $drive->{snapshot} ? 'on' : 'off';
1629         $opts .= ",snapshot=$v";
1630     }
1631
1632     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1633         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1634     }
1635
1636     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1637         my ($dir, $qmpname) = @$type;
1638         if (my $v = $drive->{"mbps$dir"}) {
1639             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1640         }
1641         if (my $v = $drive->{"mbps${dir}_max"}) {
1642             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1643         }
1644         if (my $v = $drive->{"bps${dir}_max_length"}) {
1645             $opts .= ",throttling.bps$qmpname-max-length=$v";
1646         }
1647         if (my $v = $drive->{"iops${dir}"}) {
1648             $opts .= ",throttling.iops$qmpname=$v";
1649         }
1650         if (my $v = $drive->{"iops${dir}_max"}) {
1651             $opts .= ",throttling.iops$qmpname-max=$v";
1652         }
1653         if (my $v = $drive->{"iops${dir}_max_length"}) {
1654             $opts .= ",throttling.iops$qmpname-max-length=$v";
1655         }
1656     }
1657
1658     if ($pbs_name) {
1659         $format = "rbd" if $is_rbd;
1660         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1661             if !$format;
1662         $opts .= ",format=alloc-track,file.driver=$format";
1663     } elsif ($format) {
1664         $opts .= ",format=$format";
1665     }
1666
1667     my $cache_direct = 0;
1668
1669     if (my $cache = $drive->{cache}) {
1670         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1671     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1672         $opts .= ",cache=none";
1673         $cache_direct = 1;
1674     }
1675
1676     # io_uring with cache mode writeback or writethrough on krbd will hang...
1677     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1678
1679     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1680     # sometimes, just plain disable...
1681     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1682
1683     # io_uring causes problems when used with CIFS since kernel 5.15
1684     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1685     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1686
1687     if (!$drive->{aio}) {
1688         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1689             # io_uring supports all cache modes
1690             $opts .= ",aio=io_uring";
1691         } else {
1692             # aio native works only with O_DIRECT
1693             if($cache_direct) {
1694                 $opts .= ",aio=native";
1695             } else {
1696                 $opts .= ",aio=threads";
1697             }
1698         }
1699     }
1700
1701     if (!drive_is_cdrom($drive)) {
1702         my $detectzeroes;
1703         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1704             $detectzeroes = 'off';
1705         } elsif ($drive->{discard}) {
1706             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1707         } else {
1708             # This used to be our default with discard not being specified:
1709             $detectzeroes = 'on';
1710         }
1711
1712         # note: 'detect-zeroes' works per blockdev and we want it to persist
1713         # after the alloc-track is removed, so put it on 'file' directly
1714         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1715         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1716     }
1717
1718     if ($pbs_name) {
1719         $opts .= ",backing=$pbs_name";
1720         $opts .= ",auto-remove=on";
1721     }
1722
1723     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1724     my $file_param = "file";
1725     if ($pbs_name) {
1726         # non-rbd drivers require the underlying file to be a seperate block
1727         # node, so add a second .file indirection
1728         $file_param .= ".file" if !$is_rbd;
1729         $file_param .= ".filename";
1730     }
1731     my $pathinfo = $path ? "$file_param=$path," : '';
1732
1733     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1734 }
1735
1736 sub print_pbs_blockdev {
1737     my ($pbs_conf, $pbs_name) = @_;
1738     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1739     $blockdev .= ",repository=$pbs_conf->{repository}";
1740     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1741     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1742     $blockdev .= ",archive=$pbs_conf->{archive}";
1743     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1744     return $blockdev;
1745 }
1746
1747 sub print_netdevice_full {
1748     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1749
1750     my $device = $net->{model};
1751     if ($net->{model} eq 'virtio') {
1752          $device = 'virtio-net-pci';
1753      };
1754
1755     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1756     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1757     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1758         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1759         # and out of each queue plus one config interrupt and control vector queue
1760         my $vectors = $net->{queues} * 2 + 2;
1761         $tmpstr .= ",vectors=$vectors,mq=on";
1762     }
1763     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1764
1765     if (my $mtu = $net->{mtu}) {
1766         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1767             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1768             if ($mtu == 1) {
1769                  $mtu = $bridge_mtu;
1770             } elsif ($mtu < 576) {
1771                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1772             } elsif ($mtu > $bridge_mtu) {
1773                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1774             }
1775             $tmpstr .= ",host_mtu=$mtu";
1776         } else {
1777             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1778         }
1779     }
1780
1781     if ($use_old_bios_files) {
1782         my $romfile;
1783         if ($device eq 'virtio-net-pci') {
1784             $romfile = 'pxe-virtio.rom';
1785         } elsif ($device eq 'e1000') {
1786             $romfile = 'pxe-e1000.rom';
1787         } elsif ($device eq 'e1000e') {
1788             $romfile = 'pxe-e1000e.rom';
1789         } elsif ($device eq 'ne2k') {
1790             $romfile = 'pxe-ne2k_pci.rom';
1791         } elsif ($device eq 'pcnet') {
1792             $romfile = 'pxe-pcnet.rom';
1793         } elsif ($device eq 'rtl8139') {
1794             $romfile = 'pxe-rtl8139.rom';
1795         }
1796         $tmpstr .= ",romfile=$romfile" if $romfile;
1797     }
1798
1799     return $tmpstr;
1800 }
1801
1802 sub print_netdev_full {
1803     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1804
1805     my $i = '';
1806     if ($netid =~ m/^net(\d+)$/) {
1807         $i = int($1);
1808     }
1809
1810     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1811
1812     my $ifname = "tap${vmid}i$i";
1813
1814     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1815     die "interface name '$ifname' is too long (max 15 character)\n"
1816         if length($ifname) >= 16;
1817
1818     my $vhostparam = '';
1819     if (is_native($arch)) {
1820         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1821     }
1822
1823     my $vmname = $conf->{name} || "vm$vmid";
1824
1825     my $netdev = "";
1826     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1827
1828     if ($net->{bridge}) {
1829         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1830             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1831     } else {
1832         $netdev = "type=user,id=$netid,hostname=$vmname";
1833     }
1834
1835     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1836
1837     return $netdev;
1838 }
1839
1840 my $vga_map = {
1841     'cirrus' => 'cirrus-vga',
1842     'std' => 'VGA',
1843     'vmware' => 'vmware-svga',
1844     'virtio' => 'virtio-vga',
1845     'virtio-gl' => 'virtio-vga-gl',
1846 };
1847
1848 sub print_vga_device {
1849     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1850
1851     my $type = $vga_map->{$vga->{type}};
1852     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1853         $type = 'virtio-gpu';
1854     }
1855     my $vgamem_mb = $vga->{memory};
1856
1857     my $max_outputs = '';
1858     if ($qxlnum) {
1859         $type = $id ? 'qxl' : 'qxl-vga';
1860
1861         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1862             # set max outputs so linux can have up to 4 qxl displays with one device
1863             if (min_version($machine_version, 4, 1)) {
1864                 $max_outputs = ",max_outputs=4";
1865             }
1866         }
1867     }
1868
1869     die "no devicetype for $vga->{type}\n" if !$type;
1870
1871     my $memory = "";
1872     if ($vgamem_mb) {
1873         if ($vga->{type} =~ /^virtio/) {
1874             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1875             $memory = ",max_hostmem=$bytes";
1876         } elsif ($qxlnum) {
1877             # from https://www.spice-space.org/multiple-monitors.html
1878             $memory = ",vgamem_mb=$vga->{memory}";
1879             my $ram = $vgamem_mb * 4;
1880             my $vram = $vgamem_mb * 2;
1881             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1882         } else {
1883             $memory = ",vgamem_mb=$vga->{memory}";
1884         }
1885     } elsif ($qxlnum && $id) {
1886         $memory = ",ram_size=67108864,vram_size=33554432";
1887     }
1888
1889     my $edidoff = "";
1890     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1891         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1892     }
1893
1894     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1895     my $vgaid = "vga" . ($id // '');
1896     my $pciaddr;
1897     if ($q35 && $vgaid eq 'vga') {
1898         # the first display uses pcie.0 bus on q35 machines
1899         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1900     } else {
1901         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1902     }
1903
1904     if ($vga->{type} eq 'virtio-gl') {
1905         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1906         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1907             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1908
1909         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1910             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1911     }
1912
1913     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1914 }
1915
1916 sub parse_number_sets {
1917     my ($set) = @_;
1918     my $res = [];
1919     foreach my $part (split(/;/, $set)) {
1920         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1921             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1922             push @$res, [ $1, $2 ];
1923         } else {
1924             die "invalid range: $part\n";
1925         }
1926     }
1927     return $res;
1928 }
1929
1930 sub parse_numa {
1931     my ($data) = @_;
1932
1933     my $res = parse_property_string($numa_fmt, $data);
1934     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1935     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1936     return $res;
1937 }
1938
1939 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1940 sub parse_net {
1941     my ($data) = @_;
1942
1943     my $res = eval { parse_property_string($net_fmt, $data) };
1944     if ($@) {
1945         warn $@;
1946         return;
1947     }
1948     if (!defined($res->{macaddr})) {
1949         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1950         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1951     }
1952     return $res;
1953 }
1954
1955 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1956 sub parse_ipconfig {
1957     my ($data) = @_;
1958
1959     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1960     if ($@) {
1961         warn $@;
1962         return;
1963     }
1964
1965     if ($res->{gw} && !$res->{ip}) {
1966         warn 'gateway specified without specifying an IP address';
1967         return;
1968     }
1969     if ($res->{gw6} && !$res->{ip6}) {
1970         warn 'IPv6 gateway specified without specifying an IPv6 address';
1971         return;
1972     }
1973     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1974         warn 'gateway specified together with DHCP';
1975         return;
1976     }
1977     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1978         # gw6 + auto/dhcp
1979         warn "IPv6 gateway specified together with $res->{ip6} address";
1980         return;
1981     }
1982
1983     if (!$res->{ip} && !$res->{ip6}) {
1984         return { ip => 'dhcp', ip6 => 'dhcp' };
1985     }
1986
1987     return $res;
1988 }
1989
1990 sub print_net {
1991     my $net = shift;
1992
1993     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1994 }
1995
1996 sub add_random_macs {
1997     my ($settings) = @_;
1998
1999     foreach my $opt (keys %$settings) {
2000         next if $opt !~ m/^net(\d+)$/;
2001         my $net = parse_net($settings->{$opt});
2002         next if !$net;
2003         $settings->{$opt} = print_net($net);
2004     }
2005 }
2006
2007 sub vm_is_volid_owner {
2008     my ($storecfg, $vmid, $volid) = @_;
2009
2010     if ($volid !~  m|^/|) {
2011         my ($path, $owner);
2012         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2013         if ($owner && ($owner == $vmid)) {
2014             return 1;
2015         }
2016     }
2017
2018     return;
2019 }
2020
2021 sub vmconfig_register_unused_drive {
2022     my ($storecfg, $vmid, $conf, $drive) = @_;
2023
2024     if (drive_is_cloudinit($drive)) {
2025         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2026         warn $@ if $@;
2027         delete $conf->{cloudinit};
2028     } elsif (!drive_is_cdrom($drive)) {
2029         my $volid = $drive->{file};
2030         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2031             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2032         }
2033     }
2034 }
2035
2036 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2037 my $smbios1_fmt = {
2038     uuid => {
2039         type => 'string',
2040         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2041         format_description => 'UUID',
2042         description => "Set SMBIOS1 UUID.",
2043         optional => 1,
2044     },
2045     version => {
2046         type => 'string',
2047         pattern => '[A-Za-z0-9+\/]+={0,2}',
2048         format_description => 'Base64 encoded string',
2049         description => "Set SMBIOS1 version.",
2050         optional => 1,
2051     },
2052     serial => {
2053         type => 'string',
2054         pattern => '[A-Za-z0-9+\/]+={0,2}',
2055         format_description => 'Base64 encoded string',
2056         description => "Set SMBIOS1 serial number.",
2057         optional => 1,
2058     },
2059     manufacturer => {
2060         type => 'string',
2061         pattern => '[A-Za-z0-9+\/]+={0,2}',
2062         format_description => 'Base64 encoded string',
2063         description => "Set SMBIOS1 manufacturer.",
2064         optional => 1,
2065     },
2066     product => {
2067         type => 'string',
2068         pattern => '[A-Za-z0-9+\/]+={0,2}',
2069         format_description => 'Base64 encoded string',
2070         description => "Set SMBIOS1 product ID.",
2071         optional => 1,
2072     },
2073     sku => {
2074         type => 'string',
2075         pattern => '[A-Za-z0-9+\/]+={0,2}',
2076         format_description => 'Base64 encoded string',
2077         description => "Set SMBIOS1 SKU string.",
2078         optional => 1,
2079     },
2080     family => {
2081         type => 'string',
2082         pattern => '[A-Za-z0-9+\/]+={0,2}',
2083         format_description => 'Base64 encoded string',
2084         description => "Set SMBIOS1 family string.",
2085         optional => 1,
2086     },
2087     base64 => {
2088         type => 'boolean',
2089         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2090         optional => 1,
2091     },
2092 };
2093
2094 sub parse_smbios1 {
2095     my ($data) = @_;
2096
2097     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2098     warn $@ if $@;
2099     return $res;
2100 }
2101
2102 sub print_smbios1 {
2103     my ($smbios1) = @_;
2104     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2105 }
2106
2107 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2108
2109 sub parse_watchdog {
2110     my ($value) = @_;
2111
2112     return if !$value;
2113
2114     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2115     warn $@ if $@;
2116     return $res;
2117 }
2118
2119 sub parse_guest_agent {
2120     my ($conf) = @_;
2121
2122     return {} if !defined($conf->{agent});
2123
2124     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2125     warn $@ if $@;
2126
2127     # if the agent is disabled ignore the other potentially set properties
2128     return {} if !$res->{enabled};
2129     return $res;
2130 }
2131
2132 sub get_qga_key {
2133     my ($conf, $key) = @_;
2134     return undef if !defined($conf->{agent});
2135
2136     my $agent = parse_guest_agent($conf);
2137     return $agent->{$key};
2138 }
2139
2140 sub parse_vga {
2141     my ($value) = @_;
2142
2143     return {} if !$value;
2144     my $res = eval { parse_property_string($vga_fmt, $value) };
2145     warn $@ if $@;
2146     return $res;
2147 }
2148
2149 sub parse_rng {
2150     my ($value) = @_;
2151
2152     return if !$value;
2153
2154     my $res = eval { parse_property_string($rng_fmt, $value) };
2155     warn $@ if $@;
2156     return $res;
2157 }
2158
2159 sub parse_meta_info {
2160     my ($value) = @_;
2161
2162     return if !$value;
2163
2164     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2165     warn $@ if $@;
2166     return $res;
2167 }
2168
2169 sub new_meta_info_string {
2170     my () = @_; # for now do not allow to override any value
2171
2172     return PVE::JSONSchema::print_property_string(
2173         {
2174             'creation-qemu' => kvm_user_version(),
2175             ctime => "". int(time()),
2176         },
2177         $meta_info_fmt
2178     );
2179 }
2180
2181 sub qemu_created_version_fixups {
2182     my ($conf, $forcemachine, $kvmver) = @_;
2183
2184     my $meta = parse_meta_info($conf->{meta}) // {};
2185     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2186
2187     # check if we need to apply some handling for VMs that always use the latest machine version but
2188     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2189     # would be required (we do not want to pin machine version for non-windows OS type)
2190     if (
2191         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2192         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2193         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2194         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2195     ) {
2196         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2197         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2198             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2199             # and thus with the predictable interface naming of systemd
2200             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2201         }
2202     }
2203     return;
2204 }
2205
2206 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2207 sub verify_usb_device {
2208     my ($value, $noerr) = @_;
2209
2210     return $value if parse_usb_device($value);
2211
2212     return if $noerr;
2213
2214     die "unable to parse usb device\n";
2215 }
2216
2217 # add JSON properties for create and set function
2218 sub json_config_properties {
2219     my ($prop, $with_disk_alloc) = @_;
2220
2221     my $skip_json_config_opts = {
2222         parent => 1,
2223         snaptime => 1,
2224         vmstate => 1,
2225         runningmachine => 1,
2226         runningcpu => 1,
2227         meta => 1,
2228     };
2229
2230     foreach my $opt (keys %$confdesc) {
2231         next if $skip_json_config_opts->{$opt};
2232
2233         if ($with_disk_alloc && is_valid_drivename($opt)) {
2234             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2235         } else {
2236             $prop->{$opt} = $confdesc->{$opt};
2237         }
2238     }
2239
2240     return $prop;
2241 }
2242
2243 # Properties that we can read from an OVF file
2244 sub json_ovf_properties {
2245     my $prop = {};
2246
2247     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2248         $prop->{$device} = {
2249             type => 'string',
2250             format => 'pve-volume-id-or-absolute-path',
2251             description => "Disk image that gets imported to $device",
2252             optional => 1,
2253         };
2254     }
2255
2256     $prop->{cores} = {
2257         type => 'integer',
2258         description => "The number of CPU cores.",
2259         optional => 1,
2260     };
2261     $prop->{memory} = {
2262         type => 'integer',
2263         description => "Amount of RAM for the VM in MB.",
2264         optional => 1,
2265     };
2266     $prop->{name} = {
2267         type => 'string',
2268         description => "Name of the VM.",
2269         optional => 1,
2270     };
2271
2272     return $prop;
2273 }
2274
2275 # return copy of $confdesc_cloudinit to generate documentation
2276 sub cloudinit_config_properties {
2277
2278     return dclone($confdesc_cloudinit);
2279 }
2280
2281 sub check_type {
2282     my ($key, $value) = @_;
2283
2284     die "unknown setting '$key'\n" if !$confdesc->{$key};
2285
2286     my $type = $confdesc->{$key}->{type};
2287
2288     if (!defined($value)) {
2289         die "got undefined value\n";
2290     }
2291
2292     if ($value =~ m/[\n\r]/) {
2293         die "property contains a line feed\n";
2294     }
2295
2296     if ($type eq 'boolean') {
2297         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2298         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2299         die "type check ('boolean') failed - got '$value'\n";
2300     } elsif ($type eq 'integer') {
2301         return int($1) if $value =~ m/^(\d+)$/;
2302         die "type check ('integer') failed - got '$value'\n";
2303     } elsif ($type eq 'number') {
2304         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2305         die "type check ('number') failed - got '$value'\n";
2306     } elsif ($type eq 'string') {
2307         if (my $fmt = $confdesc->{$key}->{format}) {
2308             PVE::JSONSchema::check_format($fmt, $value);
2309             return $value;
2310         }
2311         $value =~ s/^\"(.*)\"$/$1/;
2312         return $value;
2313     } else {
2314         die "internal error"
2315     }
2316 }
2317
2318 sub destroy_vm {
2319     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2320
2321     my $conf = PVE::QemuConfig->load_config($vmid);
2322
2323     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2324
2325     if ($conf->{template}) {
2326         # check if any base image is still used by a linked clone
2327         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2328                 my ($ds, $drive) = @_;
2329                 return if drive_is_cdrom($drive);
2330
2331                 my $volid = $drive->{file};
2332                 return if !$volid || $volid =~ m|^/|;
2333
2334                 die "base volume '$volid' is still in use by linked cloned\n"
2335                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2336
2337         });
2338     }
2339
2340     my $volids = {};
2341     my $remove_owned_drive = sub {
2342         my ($ds, $drive) = @_;
2343         return if drive_is_cdrom($drive, 1);
2344
2345         my $volid = $drive->{file};
2346         return if !$volid || $volid =~ m|^/|;
2347         return if $volids->{$volid};
2348
2349         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2350         return if !$path || !$owner || ($owner != $vmid);
2351
2352         $volids->{$volid} = 1;
2353         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2354         warn "Could not remove disk '$volid', check manually: $@" if $@;
2355     };
2356
2357     # only remove disks owned by this VM (referenced in the config)
2358     my $include_opts = {
2359         include_unused => 1,
2360         extra_keys => ['vmstate'],
2361     };
2362     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2363
2364     for my $snap (values %{$conf->{snapshots}}) {
2365         next if !defined($snap->{vmstate});
2366         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2367         next if !defined($drive);
2368         $remove_owned_drive->('vmstate', $drive);
2369     }
2370
2371     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2372
2373     if ($purge_unreferenced) { # also remove unreferenced disk
2374         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2375         PVE::Storage::foreach_volid($vmdisks, sub {
2376             my ($volid, $sid, $volname, $d) = @_;
2377             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2378             warn $@ if $@;
2379         });
2380     }
2381
2382     if (defined $replacement_conf) {
2383         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2384     } else {
2385         PVE::QemuConfig->destroy_config($vmid);
2386     }
2387 }
2388
2389 sub parse_vm_config {
2390     my ($filename, $raw, $strict) = @_;
2391
2392     return if !defined($raw);
2393
2394     my $res = {
2395         digest => Digest::SHA::sha1_hex($raw),
2396         snapshots => {},
2397         pending => {},
2398         cloudinit => {},
2399     };
2400
2401     my $handle_error = sub {
2402         my ($msg) = @_;
2403
2404         if ($strict) {
2405             die $msg;
2406         } else {
2407             warn $msg;
2408         }
2409     };
2410
2411     $filename =~ m|/qemu-server/(\d+)\.conf$|
2412         || die "got strange filename '$filename'";
2413
2414     my $vmid = $1;
2415
2416     my $conf = $res;
2417     my $descr;
2418     my $section = '';
2419
2420     my @lines = split(/\n/, $raw);
2421     foreach my $line (@lines) {
2422         next if $line =~ m/^\s*$/;
2423
2424         if ($line =~ m/^\[PENDING\]\s*$/i) {
2425             $section = 'pending';
2426             if (defined($descr)) {
2427                 $descr =~ s/\s+$//;
2428                 $conf->{description} = $descr;
2429             }
2430             $descr = undef;
2431             $conf = $res->{$section} = {};
2432             next;
2433         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2434             $section = 'cloudinit';
2435             $descr = undef;
2436             $conf = $res->{$section} = {};
2437             next;
2438
2439         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2440             $section = $1;
2441             if (defined($descr)) {
2442                 $descr =~ s/\s+$//;
2443                 $conf->{description} = $descr;
2444             }
2445             $descr = undef;
2446             $conf = $res->{snapshots}->{$section} = {};
2447             next;
2448         }
2449
2450         if ($line =~ m/^\#(.*)$/) {
2451             $descr = '' if !defined($descr);
2452             $descr .= PVE::Tools::decode_text($1) . "\n";
2453             next;
2454         }
2455
2456         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2457             $descr = '' if !defined($descr);
2458             $descr .= PVE::Tools::decode_text($2);
2459         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2460             $conf->{snapstate} = $1;
2461         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2462             my $key = $1;
2463             my $value = $2;
2464             $conf->{$key} = $value;
2465         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2466             my $value = $1;
2467             if ($section eq 'pending') {
2468                 $conf->{delete} = $value; # we parse this later
2469             } else {
2470                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2471             }
2472         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2473             my $key = $1;
2474             my $value = $2;
2475             eval { $value = check_type($key, $value); };
2476             if ($@) {
2477                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2478             } else {
2479                 $key = 'ide2' if $key eq 'cdrom';
2480                 my $fmt = $confdesc->{$key}->{format};
2481                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2482                     my $v = parse_drive($key, $value);
2483                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2484                         $v->{file} = $volid;
2485                         $value = print_drive($v);
2486                     } else {
2487                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2488                         next;
2489                     }
2490                 }
2491
2492                 $conf->{$key} = $value;
2493             }
2494         } else {
2495             $handle_error->("vm $vmid - unable to parse config: $line\n");
2496         }
2497     }
2498
2499     if (defined($descr)) {
2500         $descr =~ s/\s+$//;
2501         $conf->{description} = $descr;
2502     }
2503     delete $res->{snapstate}; # just to be sure
2504
2505     return $res;
2506 }
2507
2508 sub write_vm_config {
2509     my ($filename, $conf) = @_;
2510
2511     delete $conf->{snapstate}; # just to be sure
2512
2513     if ($conf->{cdrom}) {
2514         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2515         $conf->{ide2} = $conf->{cdrom};
2516         delete $conf->{cdrom};
2517     }
2518
2519     # we do not use 'smp' any longer
2520     if ($conf->{sockets}) {
2521         delete $conf->{smp};
2522     } elsif ($conf->{smp}) {
2523         $conf->{sockets} = $conf->{smp};
2524         delete $conf->{cores};
2525         delete $conf->{smp};
2526     }
2527
2528     my $used_volids = {};
2529
2530     my $cleanup_config = sub {
2531         my ($cref, $pending, $snapname) = @_;
2532
2533         foreach my $key (keys %$cref) {
2534             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2535                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2536             my $value = $cref->{$key};
2537             if ($key eq 'delete') {
2538                 die "propertry 'delete' is only allowed in [PENDING]\n"
2539                     if !$pending;
2540                 # fixme: check syntax?
2541                 next;
2542             }
2543             eval { $value = check_type($key, $value); };
2544             die "unable to parse value of '$key' - $@" if $@;
2545
2546             $cref->{$key} = $value;
2547
2548             if (!$snapname && is_valid_drivename($key)) {
2549                 my $drive = parse_drive($key, $value);
2550                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2551             }
2552         }
2553     };
2554
2555     &$cleanup_config($conf);
2556
2557     &$cleanup_config($conf->{pending}, 1);
2558
2559     &$cleanup_config($conf->{cloudinit});
2560
2561     foreach my $snapname (keys %{$conf->{snapshots}}) {
2562         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2563         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2564     }
2565
2566     # remove 'unusedX' settings if we re-add a volume
2567     foreach my $key (keys %$conf) {
2568         my $value = $conf->{$key};
2569         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2570             delete $conf->{$key};
2571         }
2572     }
2573
2574     my $generate_raw_config = sub {
2575         my ($conf, $pending) = @_;
2576
2577         my $raw = '';
2578
2579         # add description as comment to top of file
2580         if (defined(my $descr = $conf->{description})) {
2581             if ($descr) {
2582                 foreach my $cl (split(/\n/, $descr)) {
2583                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2584                 }
2585             } else {
2586                 $raw .= "#\n" if $pending;
2587             }
2588         }
2589
2590         foreach my $key (sort keys %$conf) {
2591             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2592             $raw .= "$key: $conf->{$key}\n";
2593         }
2594         return $raw;
2595     };
2596
2597     my $raw = &$generate_raw_config($conf);
2598
2599     if (scalar(keys %{$conf->{pending}})){
2600         $raw .= "\n[PENDING]\n";
2601         $raw .= &$generate_raw_config($conf->{pending}, 1);
2602     }
2603
2604     if (scalar(keys %{$conf->{cloudinit}})){
2605         $raw .= "\n[special:cloudinit]\n";
2606         $raw .= &$generate_raw_config($conf->{cloudinit});
2607     }
2608
2609     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2610         $raw .= "\n[$snapname]\n";
2611         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2612     }
2613
2614     return $raw;
2615 }
2616
2617 sub load_defaults {
2618
2619     my $res = {};
2620
2621     # we use static defaults from our JSON schema configuration
2622     foreach my $key (keys %$confdesc) {
2623         if (defined(my $default = $confdesc->{$key}->{default})) {
2624             $res->{$key} = $default;
2625         }
2626     }
2627
2628     return $res;
2629 }
2630
2631 sub config_list {
2632     my $vmlist = PVE::Cluster::get_vmlist();
2633     my $res = {};
2634     return $res if !$vmlist || !$vmlist->{ids};
2635     my $ids = $vmlist->{ids};
2636     my $nodename = nodename();
2637
2638     foreach my $vmid (keys %$ids) {
2639         my $d = $ids->{$vmid};
2640         next if !$d->{node} || $d->{node} ne $nodename;
2641         next if !$d->{type} || $d->{type} ne 'qemu';
2642         $res->{$vmid}->{exists} = 1;
2643     }
2644     return $res;
2645 }
2646
2647 # test if VM uses local resources (to prevent migration)
2648 sub check_local_resources {
2649     my ($conf, $noerr) = @_;
2650
2651     my @loc_res = ();
2652
2653     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2654     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2655
2656     push @loc_res, "ivshmem" if $conf->{ivshmem};
2657
2658     foreach my $k (keys %$conf) {
2659         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2660         # sockets are safe: they will recreated be on the target side post-migrate
2661         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2662         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2663     }
2664
2665     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2666
2667     return \@loc_res;
2668 }
2669
2670 # check if used storages are available on all nodes (use by migrate)
2671 sub check_storage_availability {
2672     my ($storecfg, $conf, $node) = @_;
2673
2674     PVE::QemuConfig->foreach_volume($conf, sub {
2675         my ($ds, $drive) = @_;
2676
2677         my $volid = $drive->{file};
2678         return if !$volid;
2679
2680         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2681         return if !$sid;
2682
2683         # check if storage is available on both nodes
2684         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2685         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2686
2687         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2688
2689         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2690             if !$scfg->{content}->{$vtype};
2691    });
2692 }
2693
2694 # list nodes where all VM images are available (used by has_feature API)
2695 sub shared_nodes {
2696     my ($conf, $storecfg) = @_;
2697
2698     my $nodelist = PVE::Cluster::get_nodelist();
2699     my $nodehash = { map { $_ => 1 } @$nodelist };
2700     my $nodename = nodename();
2701
2702     PVE::QemuConfig->foreach_volume($conf, sub {
2703         my ($ds, $drive) = @_;
2704
2705         my $volid = $drive->{file};
2706         return if !$volid;
2707
2708         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2709         if ($storeid) {
2710             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2711             if ($scfg->{disable}) {
2712                 $nodehash = {};
2713             } elsif (my $avail = $scfg->{nodes}) {
2714                 foreach my $node (keys %$nodehash) {
2715                     delete $nodehash->{$node} if !$avail->{$node};
2716                 }
2717             } elsif (!$scfg->{shared}) {
2718                 foreach my $node (keys %$nodehash) {
2719                     delete $nodehash->{$node} if $node ne $nodename
2720                 }
2721             }
2722         }
2723     });
2724
2725     return $nodehash
2726 }
2727
2728 sub check_local_storage_availability {
2729     my ($conf, $storecfg) = @_;
2730
2731     my $nodelist = PVE::Cluster::get_nodelist();
2732     my $nodehash = { map { $_ => {} } @$nodelist };
2733
2734     PVE::QemuConfig->foreach_volume($conf, sub {
2735         my ($ds, $drive) = @_;
2736
2737         my $volid = $drive->{file};
2738         return if !$volid;
2739
2740         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2741         if ($storeid) {
2742             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2743
2744             if ($scfg->{disable}) {
2745                 foreach my $node (keys %$nodehash) {
2746                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2747                 }
2748             } elsif (my $avail = $scfg->{nodes}) {
2749                 foreach my $node (keys %$nodehash) {
2750                     if (!$avail->{$node}) {
2751                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2752                     }
2753                 }
2754             }
2755         }
2756     });
2757
2758     foreach my $node (values %$nodehash) {
2759         if (my $unavail = $node->{unavailable_storages}) {
2760             $node->{unavailable_storages} = [ sort keys %$unavail ];
2761         }
2762     }
2763
2764     return $nodehash
2765 }
2766
2767 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2768 sub check_running {
2769     my ($vmid, $nocheck, $node) = @_;
2770
2771     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2772     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2773 }
2774
2775 sub vzlist {
2776
2777     my $vzlist = config_list();
2778
2779     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2780
2781     while (defined(my $de = $fd->read)) {
2782         next if $de !~ m/^(\d+)\.pid$/;
2783         my $vmid = $1;
2784         next if !defined($vzlist->{$vmid});
2785         if (my $pid = check_running($vmid)) {
2786             $vzlist->{$vmid}->{pid} = $pid;
2787         }
2788     }
2789
2790     return $vzlist;
2791 }
2792
2793 our $vmstatus_return_properties = {
2794     vmid => get_standard_option('pve-vmid'),
2795     status => {
2796         description => "Qemu process status.",
2797         type => 'string',
2798         enum => ['stopped', 'running'],
2799     },
2800     maxmem => {
2801         description => "Maximum memory in bytes.",
2802         type => 'integer',
2803         optional => 1,
2804         renderer => 'bytes',
2805     },
2806     maxdisk => {
2807         description => "Root disk size in bytes.",
2808         type => 'integer',
2809         optional => 1,
2810         renderer => 'bytes',
2811     },
2812     name => {
2813         description => "VM name.",
2814         type => 'string',
2815         optional => 1,
2816     },
2817     qmpstatus => {
2818         description => "Qemu QMP agent status.",
2819         type => 'string',
2820         optional => 1,
2821     },
2822     pid => {
2823         description => "PID of running qemu process.",
2824         type => 'integer',
2825         optional => 1,
2826     },
2827     uptime => {
2828         description => "Uptime.",
2829         type => 'integer',
2830         optional => 1,
2831         renderer => 'duration',
2832     },
2833     cpus => {
2834         description => "Maximum usable CPUs.",
2835         type => 'number',
2836         optional => 1,
2837     },
2838     lock => {
2839         description => "The current config lock, if any.",
2840         type => 'string',
2841         optional => 1,
2842     },
2843     tags => {
2844         description  => "The current configured tags, if any",
2845         type => 'string',
2846         optional => 1,
2847     },
2848     'running-machine' => {
2849         description => "The currently running machine type (if running).",
2850         type => 'string',
2851         optional => 1,
2852     },
2853     'running-qemu' => {
2854         description => "The currently running QEMU version (if running).",
2855         type => 'string',
2856         optional => 1,
2857     },
2858 };
2859
2860 my $last_proc_pid_stat;
2861
2862 # get VM status information
2863 # This must be fast and should not block ($full == false)
2864 # We only query KVM using QMP if $full == true (this can be slow)
2865 sub vmstatus {
2866     my ($opt_vmid, $full) = @_;
2867
2868     my $res = {};
2869
2870     my $storecfg = PVE::Storage::config();
2871
2872     my $list = vzlist();
2873     my $defaults = load_defaults();
2874
2875     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2876
2877     my $cpucount = $cpuinfo->{cpus} || 1;
2878
2879     foreach my $vmid (keys %$list) {
2880         next if $opt_vmid && ($vmid ne $opt_vmid);
2881
2882         my $conf = PVE::QemuConfig->load_config($vmid);
2883
2884         my $d = { vmid => int($vmid) };
2885         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2886
2887         # fixme: better status?
2888         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2889
2890         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2891         if (defined($size)) {
2892             $d->{disk} = 0; # no info available
2893             $d->{maxdisk} = $size;
2894         } else {
2895             $d->{disk} = 0;
2896             $d->{maxdisk} = 0;
2897         }
2898
2899         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2900             * ($conf->{cores} || $defaults->{cores});
2901         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2902         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2903
2904         $d->{name} = $conf->{name} || "VM $vmid";
2905         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2906             : $defaults->{memory}*(1024*1024);
2907
2908         if ($conf->{balloon}) {
2909             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2910             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2911                 : $defaults->{shares};
2912         }
2913
2914         $d->{uptime} = 0;
2915         $d->{cpu} = 0;
2916         $d->{mem} = 0;
2917
2918         $d->{netout} = 0;
2919         $d->{netin} = 0;
2920
2921         $d->{diskread} = 0;
2922         $d->{diskwrite} = 0;
2923
2924         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2925
2926         $d->{serial} = 1 if conf_has_serial($conf);
2927         $d->{lock} = $conf->{lock} if $conf->{lock};
2928         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2929
2930         $res->{$vmid} = $d;
2931     }
2932
2933     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2934     foreach my $dev (keys %$netdev) {
2935         next if $dev !~ m/^tap([1-9]\d*)i/;
2936         my $vmid = $1;
2937         my $d = $res->{$vmid};
2938         next if !$d;
2939
2940         $d->{netout} += $netdev->{$dev}->{receive};
2941         $d->{netin} += $netdev->{$dev}->{transmit};
2942
2943         if ($full) {
2944             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2945             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2946         }
2947
2948     }
2949
2950     my $ctime = gettimeofday;
2951
2952     foreach my $vmid (keys %$list) {
2953
2954         my $d = $res->{$vmid};
2955         my $pid = $d->{pid};
2956         next if !$pid;
2957
2958         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2959         next if !$pstat; # not running
2960
2961         my $used = $pstat->{utime} + $pstat->{stime};
2962
2963         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2964
2965         if ($pstat->{vsize}) {
2966             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2967         }
2968
2969         my $old = $last_proc_pid_stat->{$pid};
2970         if (!$old) {
2971             $last_proc_pid_stat->{$pid} = {
2972                 time => $ctime,
2973                 used => $used,
2974                 cpu => 0,
2975             };
2976             next;
2977         }
2978
2979         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2980
2981         if ($dtime > 1000) {
2982             my $dutime = $used -  $old->{used};
2983
2984             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2985             $last_proc_pid_stat->{$pid} = {
2986                 time => $ctime,
2987                 used => $used,
2988                 cpu => $d->{cpu},
2989             };
2990         } else {
2991             $d->{cpu} = $old->{cpu};
2992         }
2993     }
2994
2995     return $res if !$full;
2996
2997     my $qmpclient = PVE::QMPClient->new();
2998
2999     my $ballooncb = sub {
3000         my ($vmid, $resp) = @_;
3001
3002         my $info = $resp->{'return'};
3003         return if !$info->{max_mem};
3004
3005         my $d = $res->{$vmid};
3006
3007         # use memory assigned to VM
3008         $d->{maxmem} = $info->{max_mem};
3009         $d->{balloon} = $info->{actual};
3010
3011         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3012             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3013             $d->{freemem} = $info->{free_mem};
3014         }
3015
3016         $d->{ballooninfo} = $info;
3017     };
3018
3019     my $blockstatscb = sub {
3020         my ($vmid, $resp) = @_;
3021         my $data = $resp->{'return'} || [];
3022         my $totalrdbytes = 0;
3023         my $totalwrbytes = 0;
3024
3025         for my $blockstat (@$data) {
3026             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3027             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3028
3029             $blockstat->{device} =~ s/drive-//;
3030             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3031         }
3032         $res->{$vmid}->{diskread} = $totalrdbytes;
3033         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3034     };
3035
3036     my $machinecb = sub {
3037         my ($vmid, $resp) = @_;
3038         my $data = $resp->{'return'} || [];
3039
3040         $res->{$vmid}->{'running-machine'} =
3041             PVE::QemuServer::Machine::current_from_query_machines($data);
3042     };
3043
3044     my $versioncb = sub {
3045         my ($vmid, $resp) = @_;
3046         my $data = $resp->{'return'} // {};
3047         my $version = 'unknown';
3048
3049         if (my $v = $data->{qemu}) {
3050             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3051         }
3052
3053         $res->{$vmid}->{'running-qemu'} = $version;
3054     };
3055
3056     my $statuscb = sub {
3057         my ($vmid, $resp) = @_;
3058
3059         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3060         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3061         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3062         # this fails if ballon driver is not loaded, so this must be
3063         # the last commnand (following command are aborted if this fails).
3064         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3065
3066         my $status = 'unknown';
3067         if (!defined($status = $resp->{'return'}->{status})) {
3068             warn "unable to get VM status\n";
3069             return;
3070         }
3071
3072         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3073     };
3074
3075     foreach my $vmid (keys %$list) {
3076         next if $opt_vmid && ($vmid ne $opt_vmid);
3077         next if !$res->{$vmid}->{pid}; # not running
3078         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3079     }
3080
3081     $qmpclient->queue_execute(undef, 2);
3082
3083     foreach my $vmid (keys %$list) {
3084         next if $opt_vmid && ($vmid ne $opt_vmid);
3085         next if !$res->{$vmid}->{pid}; #not running
3086
3087         # we can't use the $qmpclient since it might have already aborted on
3088         # 'query-balloon', but this might also fail for older versions...
3089         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3090         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3091     }
3092
3093     foreach my $vmid (keys %$list) {
3094         next if $opt_vmid && ($vmid ne $opt_vmid);
3095         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3096     }
3097
3098     return $res;
3099 }
3100
3101 sub conf_has_serial {
3102     my ($conf) = @_;
3103
3104     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3105         if ($conf->{"serial$i"}) {
3106             return 1;
3107         }
3108     }
3109
3110     return 0;
3111 }
3112
3113 sub conf_has_audio {
3114     my ($conf, $id) = @_;
3115
3116     $id //= 0;
3117     my $audio = $conf->{"audio$id"};
3118     return if !defined($audio);
3119
3120     my $audioproperties = parse_property_string($audio_fmt, $audio);
3121     my $audiodriver = $audioproperties->{driver} // 'spice';
3122
3123     return {
3124         dev => $audioproperties->{device},
3125         dev_id => "audiodev$id",
3126         backend => $audiodriver,
3127         backend_id => "$audiodriver-backend${id}",
3128     };
3129 }
3130
3131 sub audio_devs {
3132     my ($audio, $audiopciaddr, $machine_version) = @_;
3133
3134     my $devs = [];
3135
3136     my $id = $audio->{dev_id};
3137     my $audiodev = "";
3138     if (min_version($machine_version, 4, 2)) {
3139         $audiodev = ",audiodev=$audio->{backend_id}";
3140     }
3141
3142     if ($audio->{dev} eq 'AC97') {
3143         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3144     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3145         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3146         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3147         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3148     } else {
3149         die "unkown audio device '$audio->{dev}', implement me!";
3150     }
3151
3152     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3153
3154     return $devs;
3155 }
3156
3157 sub get_tpm_paths {
3158     my ($vmid) = @_;
3159     return {
3160         socket => "/var/run/qemu-server/$vmid.swtpm",
3161         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3162     };
3163 }
3164
3165 sub add_tpm_device {
3166     my ($vmid, $devices, $conf) = @_;
3167
3168     return if !$conf->{tpmstate0};
3169
3170     my $paths = get_tpm_paths($vmid);
3171
3172     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3173     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3174     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3175 }
3176
3177 sub start_swtpm {
3178     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3179
3180     return if !$tpmdrive;
3181
3182     my $state;
3183     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3184     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3185     if ($storeid) {
3186         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3187     } else {
3188         $state = $tpm->{file};
3189     }
3190
3191     my $paths = get_tpm_paths($vmid);
3192
3193     # during migration, we will get state from remote
3194     #
3195     if (!$migration) {
3196         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3197         my $setup_cmd = [
3198             "swtpm_setup",
3199             "--tpmstate",
3200             "file://$state",
3201             "--createek",
3202             "--create-ek-cert",
3203             "--create-platform-cert",
3204             "--lock-nvram",
3205             "--config",
3206             "/etc/swtpm_setup.conf", # do not use XDG configs
3207             "--runas",
3208             "0", # force creation as root, error if not possible
3209             "--not-overwrite", # ignore existing state, do not modify
3210         ];
3211
3212         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3213         # TPM 2.0 supports ECC crypto, use if possible
3214         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3215
3216         run_command($setup_cmd, outfunc => sub {
3217             print "swtpm_setup: $1\n";
3218         });
3219     }
3220
3221     my $emulator_cmd = [
3222         "swtpm",
3223         "socket",
3224         "--tpmstate",
3225         "backend-uri=file://$state,mode=0600",
3226         "--ctrl",
3227         "type=unixio,path=$paths->{socket},mode=0600",
3228         "--pid",
3229         "file=$paths->{pid}",
3230         "--terminate", # terminate on QEMU disconnect
3231         "--daemon",
3232     ];
3233     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3234     run_command($emulator_cmd, outfunc => sub { print $1; });
3235
3236     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3237     while (! -e $paths->{pid}) {
3238         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3239         usleep(50_000);
3240     }
3241
3242     # return untainted PID of swtpm daemon so it can be killed on error
3243     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3244     return $1;
3245 }
3246
3247 sub vga_conf_has_spice {
3248     my ($vga) = @_;
3249
3250     my $vgaconf = parse_vga($vga);
3251     my $vgatype = $vgaconf->{type};
3252     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3253
3254     return $1 || 1;
3255 }
3256
3257 sub is_native($) {
3258     my ($arch) = @_;
3259     return get_host_arch() eq $arch;
3260 }
3261
3262 sub get_vm_arch {
3263     my ($conf) = @_;
3264     return $conf->{arch} // get_host_arch();
3265 }
3266
3267 my $default_machines = {
3268     x86_64 => 'pc',
3269     aarch64 => 'virt',
3270 };
3271
3272 sub get_installed_machine_version {
3273     my ($kvmversion) = @_;
3274     $kvmversion = kvm_user_version() if !defined($kvmversion);
3275     $kvmversion =~ m/^(\d+\.\d+)/;
3276     return $1;
3277 }
3278
3279 sub windows_get_pinned_machine_version {
3280     my ($machine, $base_version, $kvmversion) = @_;
3281
3282     my $pin_version = $base_version;
3283     if (!defined($base_version) ||
3284         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3285     ) {
3286         $pin_version = get_installed_machine_version($kvmversion);
3287     }
3288     if (!$machine || $machine eq 'pc') {
3289         $machine = "pc-i440fx-$pin_version";
3290     } elsif ($machine eq 'q35') {
3291         $machine = "pc-q35-$pin_version";
3292     } elsif ($machine eq 'virt') {
3293         $machine = "virt-$pin_version";
3294     } else {
3295         warn "unknown machine type '$machine', not touching that!\n";
3296     }
3297
3298     return $machine;
3299 }
3300
3301 sub get_vm_machine {
3302     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3303
3304     my $machine = $forcemachine || $conf->{machine};
3305
3306     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3307         $kvmversion //= kvm_user_version();
3308         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3309         # layout which confuses windows quite a bit and may result in various regressions..
3310         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3311         if (windows_version($conf->{ostype})) {
3312             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3313         }
3314         $arch //= 'x86_64';
3315         $machine ||= $default_machines->{$arch};
3316         if ($add_pve_version) {
3317             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3318             $machine .= "+pve$pvever";
3319         }
3320     }
3321
3322     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3323         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3324         $machine = $1 if $is_pxe;
3325
3326         # for version-pinned machines that do not include a pve-version (e.g.
3327         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3328         $machine .= '+pve0';
3329
3330         $machine .= '.pxe' if $is_pxe;
3331     }
3332
3333     return $machine;
3334 }
3335
3336 sub get_ovmf_files($$$) {
3337     my ($arch, $efidisk, $smm) = @_;
3338
3339     my $types = $OVMF->{$arch}
3340         or die "no OVMF images known for architecture '$arch'\n";
3341
3342     my $type = 'default';
3343     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3344         $type = $smm ? "4m" : "4m-no-smm";
3345         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3346     }
3347
3348     return $types->{$type}->@*;
3349 }
3350
3351 my $Arch2Qemu = {
3352     aarch64 => '/usr/bin/qemu-system-aarch64',
3353     x86_64 => '/usr/bin/qemu-system-x86_64',
3354 };
3355 sub get_command_for_arch($) {
3356     my ($arch) = @_;
3357     return '/usr/bin/kvm' if is_native($arch);
3358
3359     my $cmd = $Arch2Qemu->{$arch}
3360         or die "don't know how to emulate architecture '$arch'\n";
3361     return $cmd;
3362 }
3363
3364 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3365 # to use in a QEMU command line (-cpu element), first array_intersect the result
3366 # of query_supported_ with query_understood_. This is necessary because:
3367 #
3368 # a) query_understood_ returns flags the host cannot use and
3369 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3370 #    flags, but CPU settings - with most of them being flags. Those settings
3371 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3372 #
3373 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3374 # expensive. If you need the value returned from this, you can get it much
3375 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3376 # $accel being 'kvm' or 'tcg'.
3377 #
3378 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3379 # changes, automatically populating pmxcfs.
3380 #
3381 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3382 # since kvm and tcg machines support different flags
3383 #
3384 sub query_supported_cpu_flags {
3385     my ($arch) = @_;
3386
3387     $arch //= get_host_arch();
3388     my $default_machine = $default_machines->{$arch};
3389
3390     my $flags = {};
3391
3392     # FIXME: Once this is merged, the code below should work for ARM as well:
3393     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3394     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3395         $arch eq "aarch64";
3396
3397     my $kvm_supported = defined(kvm_version());
3398     my $qemu_cmd = get_command_for_arch($arch);
3399     my $fakevmid = -1;
3400     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3401
3402     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3403     my $query_supported_run_qemu = sub {
3404         my ($kvm) = @_;
3405
3406         my $flags = {};
3407         my $cmd = [
3408             $qemu_cmd,
3409             '-machine', $default_machine,
3410             '-display', 'none',
3411             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3412             '-mon', 'chardev=qmp,mode=control',
3413             '-pidfile', $pidfile,
3414             '-S', '-daemonize'
3415         ];
3416
3417         if (!$kvm) {
3418             push @$cmd, '-accel', 'tcg';
3419         }
3420
3421         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3422         die "QEMU flag querying VM exited with code " . $rc if $rc;
3423
3424         eval {
3425             my $cmd_result = mon_cmd(
3426                 $fakevmid,
3427                 'query-cpu-model-expansion',
3428                 type => 'full',
3429                 model => { name => 'host' }
3430             );
3431
3432             my $props = $cmd_result->{model}->{props};
3433             foreach my $prop (keys %$props) {
3434                 next if $props->{$prop} ne '1';
3435                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3436                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3437                 # We only keep those with underscores, to match /proc/cpuinfo
3438                 $prop =~ s/\.|-/_/g;
3439                 $flags->{$prop} = 1;
3440             }
3441         };
3442         my $err = $@;
3443
3444         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3445         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3446
3447         die $err if $err;
3448
3449         return [ sort keys %$flags ];
3450     };
3451
3452     # We need to query QEMU twice, since KVM and TCG have different supported flags
3453     PVE::QemuConfig->lock_config($fakevmid, sub {
3454         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3455         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3456
3457         if ($kvm_supported) {
3458             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3459             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3460         }
3461     });
3462
3463     return $flags;
3464 }
3465
3466 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3467 my $understood_cpu_flag_dir = "/usr/share/kvm";
3468 sub query_understood_cpu_flags {
3469     my $arch = get_host_arch();
3470     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3471
3472     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3473         if ! -e $filepath;
3474
3475     my $raw = file_get_contents($filepath);
3476     $raw =~ s/^\s+|\s+$//g;
3477     my @flags = split(/\s+/, $raw);
3478
3479     return \@flags;
3480 }
3481
3482 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3483 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3484 my sub should_disable_smm {
3485     my ($conf, $vga) = @_;
3486
3487     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3488         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3489 }
3490
3491 sub config_to_command {
3492     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3493         $pbs_backing) = @_;
3494
3495     my $cmd = [];
3496     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3497     my $devices = [];
3498     my $bridges = {};
3499     my $ostype = $conf->{ostype};
3500     my $winversion = windows_version($ostype);
3501     my $kvm = $conf->{kvm};
3502     my $nodename = nodename();
3503
3504     my $arch = get_vm_arch($conf);
3505     my $kvm_binary = get_command_for_arch($arch);
3506     my $kvmver = kvm_user_version($kvm_binary);
3507
3508     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3509         $kvmver //= "undefined";
3510         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3511     }
3512
3513     my $add_pve_version = min_version($kvmver, 4, 1);
3514
3515     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3516     my $machine_version = extract_version($machine_type, $kvmver);
3517     $kvm //= 1 if is_native($arch);
3518
3519     $machine_version =~ m/(\d+)\.(\d+)/;
3520     my ($machine_major, $machine_minor) = ($1, $2);
3521
3522     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3523         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3524     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3525         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3526             ." please upgrade node '$nodename'\n"
3527     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3528         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3529         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3530             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3531             ." node '$nodename'\n";
3532     }
3533
3534     # if a specific +pve version is required for a feature, use $version_guard
3535     # instead of min_version to allow machines to be run with the minimum
3536     # required version
3537     my $required_pve_version = 0;
3538     my $version_guard = sub {
3539         my ($major, $minor, $pve) = @_;
3540         return 0 if !min_version($machine_version, $major, $minor, $pve);
3541         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3542         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3543         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3544         return 1;
3545     };
3546
3547     if ($kvm && !defined kvm_version()) {
3548         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3549             ." or enable in BIOS.\n";
3550     }
3551
3552     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3553     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3554     my $use_old_bios_files = undef;
3555     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3556
3557     if ($conf->{affinity}) {
3558         push @$cmd, "/usr/bin/taskset";
3559         push @$cmd, "--cpu-list";
3560         push @$cmd, "--all-tasks";
3561         push @$cmd, $conf->{affinity};
3562     }
3563
3564     push @$cmd, $kvm_binary;
3565
3566     push @$cmd, '-id', $vmid;
3567
3568     my $vmname = $conf->{name} || "vm$vmid";
3569
3570     push @$cmd, '-name', "$vmname,debug-threads=on";
3571
3572     push @$cmd, '-no-shutdown';
3573
3574     my $use_virtio = 0;
3575
3576     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3577     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3578     push @$cmd, '-mon', "chardev=qmp,mode=control";
3579
3580     if (min_version($machine_version, 2, 12)) {
3581         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3582         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3583     }
3584
3585     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3586
3587     push @$cmd, '-daemonize';
3588
3589     if ($conf->{smbios1}) {
3590         my $smbios_conf = parse_smbios1($conf->{smbios1});
3591         if ($smbios_conf->{base64}) {
3592             # Do not pass base64 flag to qemu
3593             delete $smbios_conf->{base64};
3594             my $smbios_string = "";
3595             foreach my $key (keys %$smbios_conf) {
3596                 my $value;
3597                 if ($key eq "uuid") {
3598                     $value = $smbios_conf->{uuid}
3599                 } else {
3600                     $value = decode_base64($smbios_conf->{$key});
3601                 }
3602                 # qemu accepts any binary data, only commas need escaping by double comma
3603                 $value =~ s/,/,,/g;
3604                 $smbios_string .= "," . $key . "=" . $value if $value;
3605             }
3606             push @$cmd, '-smbios', "type=1" . $smbios_string;
3607         } else {
3608             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3609         }
3610     }
3611
3612     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3613         my $d;
3614         if (my $efidisk = $conf->{efidisk0}) {
3615             $d = parse_drive('efidisk0', $efidisk);
3616         }
3617
3618         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3619         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3620
3621         my ($path, $format);
3622         my $read_only_str = '';
3623         if ($d) {
3624             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3625             $format = $d->{format};
3626             if ($storeid) {
3627                 $path = PVE::Storage::path($storecfg, $d->{file});
3628                 if (!defined($format)) {
3629                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3630                     $format = qemu_img_format($scfg, $volname);
3631                 }
3632             } else {
3633                 $path = $d->{file};
3634                 die "efidisk format must be specified\n"
3635                     if !defined($format);
3636             }
3637
3638             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3639         } else {
3640             log_warn("no efidisk configured! Using temporary efivars disk.");
3641             $path = "/tmp/$vmid-ovmf.fd";
3642             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3643             $format = 'raw';
3644         }
3645
3646         my $size_str = "";
3647
3648         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3649             $size_str = ",size=" . (-s $ovmf_vars);
3650         }
3651
3652         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3653         my $cache = "";
3654         if ($path =~ m/^rbd:/) {
3655                 $cache = ',cache=writeback';
3656                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3657         }
3658
3659         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3660         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3661     }
3662
3663     if ($q35) { # tell QEMU to load q35 config early
3664         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3665         if (min_version($machine_version, 4, 0)) {
3666             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3667         } else {
3668             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3669         }
3670     }
3671
3672     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3673         push @$cmd, $fixups->@*;
3674     }
3675
3676     if ($conf->{vmgenid}) {
3677         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3678     }
3679
3680     # add usb controllers
3681     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3682         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3683     push @$devices, @usbcontrollers if @usbcontrollers;
3684     my $vga = parse_vga($conf->{vga});
3685
3686     my $qxlnum = vga_conf_has_spice($conf->{vga});
3687     $vga->{type} = 'qxl' if $qxlnum;
3688
3689     if (!$vga->{type}) {
3690         if ($arch eq 'aarch64') {
3691             $vga->{type} = 'virtio';
3692         } elsif (min_version($machine_version, 2, 9)) {
3693             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3694         } else {
3695             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3696         }
3697     }
3698
3699     # enable absolute mouse coordinates (needed by vnc)
3700     my $tablet = $conf->{tablet};
3701     if (!defined($tablet)) {
3702         $tablet = $defaults->{tablet};
3703         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3704         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3705     }
3706
3707     if ($tablet) {
3708         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3709         my $kbd = print_keyboarddevice_full($conf, $arch);
3710         push @$devices, '-device', $kbd if defined($kbd);
3711     }
3712
3713     my $bootorder = device_bootorder($conf);
3714
3715     # host pci device passthrough
3716     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3717         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3718
3719     # usb devices
3720     my $usb_dev_features = {};
3721     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3722
3723     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3724         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3725     push @$devices, @usbdevices if @usbdevices;
3726
3727     # serial devices
3728     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3729         my $path = $conf->{"serial$i"} or next;
3730         if ($path eq 'socket') {
3731             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3732             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3733             # On aarch64, serial0 is the UART device. Qemu only allows
3734             # connecting UART devices via the '-serial' command line, as
3735             # the device has a fixed slot on the hardware...
3736             if ($arch eq 'aarch64' && $i == 0) {
3737                 push @$devices, '-serial', "chardev:serial$i";
3738             } else {
3739                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3740             }
3741         } else {
3742             die "no such serial device\n" if ! -c $path;
3743             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3744             push @$devices, '-device', "isa-serial,chardev=serial$i";
3745         }
3746     }
3747
3748     # parallel devices
3749     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3750         if (my $path = $conf->{"parallel$i"}) {
3751             die "no such parallel device\n" if ! -c $path;
3752             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3753             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3754             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3755         }
3756     }
3757
3758     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3759         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3760         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3761         push @$devices, @$audio_devs;
3762     }
3763
3764     add_tpm_device($vmid, $devices, $conf);
3765
3766     my $sockets = 1;
3767     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3768     $sockets = $conf->{sockets} if  $conf->{sockets};
3769
3770     my $cores = $conf->{cores} || 1;
3771
3772     my $maxcpus = $sockets * $cores;
3773
3774     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3775
3776     my $allowed_vcpus = $cpuinfo->{cpus};
3777
3778     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3779
3780     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3781         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3782         for (my $i = 2; $i <= $vcpus; $i++)  {
3783             my $cpustr = print_cpu_device($conf,$i);
3784             push @$cmd, '-device', $cpustr;
3785         }
3786
3787     } else {
3788
3789         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3790     }
3791     push @$cmd, '-nodefaults';
3792
3793     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3794
3795     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3796
3797     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3798
3799     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3800         push @$devices, '-device', print_vga_device(
3801             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3802
3803         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3804
3805         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3806         push @$cmd,  '-vnc', "unix:$socket,password=on";
3807     } else {
3808         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3809         push @$cmd, '-nographic';
3810     }
3811
3812     # time drift fix
3813     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3814     my $useLocaltime = $conf->{localtime};
3815
3816     if ($winversion >= 5) { # windows
3817         $useLocaltime = 1 if !defined($conf->{localtime});
3818
3819         # use time drift fix when acpi is enabled
3820         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3821             $tdf = 1 if !defined($conf->{tdf});
3822         }
3823     }
3824
3825     if ($winversion >= 6) {
3826         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3827         push @$cmd, '-no-hpet';
3828     }
3829
3830     push @$rtcFlags, 'driftfix=slew' if $tdf;
3831
3832     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3833         push @$rtcFlags, "base=$conf->{startdate}";
3834     } elsif ($useLocaltime) {
3835         push @$rtcFlags, 'base=localtime';
3836     }
3837
3838     if ($forcecpu) {
3839         push @$cmd, '-cpu', $forcecpu;
3840     } else {
3841         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3842     }
3843
3844     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3845
3846     push @$cmd, '-S' if $conf->{freeze};
3847
3848     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3849
3850     my $guest_agent = parse_guest_agent($conf);
3851
3852     if ($guest_agent->{enabled}) {
3853         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3854         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3855
3856         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3857             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3858             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3859             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3860         } elsif ($guest_agent->{type} eq 'isa') {
3861             push @$devices, '-device', "isa-serial,chardev=qga0";
3862         }
3863     }
3864
3865     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3866     if ($rng && $version_guard->(4, 1, 2)) {
3867         check_rng_source($rng->{source});
3868
3869         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3870         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3871         my $limiter_str = "";
3872         if ($max_bytes) {
3873             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3874         }
3875
3876         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3877         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3878         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3879     }
3880
3881     my $spice_port;
3882
3883     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3884         if ($qxlnum > 1) {
3885             if ($winversion){
3886                 for (my $i = 1; $i < $qxlnum; $i++){
3887                     push @$devices, '-device', print_vga_device(
3888                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3889                 }
3890             } else {
3891                 # assume other OS works like Linux
3892                 my ($ram, $vram) = ("134217728", "67108864");
3893                 if ($vga->{memory}) {
3894                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3895                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3896                 }
3897                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3898                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3899             }
3900         }
3901
3902         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3903
3904         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3905         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3906         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3907
3908         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3909         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3910         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3911
3912         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3913         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3914
3915         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3916         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3917         if ($spice_enhancement->{foldersharing}) {
3918             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3919             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3920         }
3921
3922         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3923         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3924             if $spice_enhancement->{videostreaming};
3925
3926         push @$devices, '-spice', "$spice_opts";
3927     }
3928
3929     # enable balloon by default, unless explicitly disabled
3930     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3931         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3932         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3933         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3934         push @$devices, '-device', $ballooncmd;
3935     }
3936
3937     if ($conf->{watchdog}) {
3938         my $wdopts = parse_watchdog($conf->{watchdog});
3939         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3940         my $watchdog = $wdopts->{model} || 'i6300esb';
3941         push @$devices, '-device', "$watchdog$pciaddr";
3942         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3943     }
3944
3945     my $vollist = [];
3946     my $scsicontroller = {};
3947     my $ahcicontroller = {};
3948     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3949
3950     # Add iscsi initiator name if available
3951     if (my $initiator = get_initiator_name()) {
3952         push @$devices, '-iscsi', "initiator-name=$initiator";
3953     }
3954
3955     PVE::QemuConfig->foreach_volume($conf, sub {
3956         my ($ds, $drive) = @_;
3957
3958         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3959             check_volume_storage_type($storecfg, $drive->{file});
3960             push @$vollist, $drive->{file};
3961         }
3962
3963         # ignore efidisk here, already added in bios/fw handling code above
3964         return if $drive->{interface} eq 'efidisk';
3965         # similar for TPM
3966         return if $drive->{interface} eq 'tpmstate';
3967
3968         $use_virtio = 1 if $ds =~ m/^virtio/;
3969
3970         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3971
3972         if ($drive->{interface} eq 'virtio'){
3973            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3974         }
3975
3976         if ($drive->{interface} eq 'scsi') {
3977
3978             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3979
3980             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3981                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3982
3983             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3984             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3985
3986             my $iothread = '';
3987             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3988                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3989                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3990             } elsif ($drive->{iothread}) {
3991                 log_warn(
3992                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3993                 );
3994             }
3995
3996             my $queues = '';
3997             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3998                 $queues = ",num_queues=$drive->{queues}";
3999             }
4000
4001             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4002                 if !$scsicontroller->{$controller};
4003             $scsicontroller->{$controller}=1;
4004         }
4005
4006         if ($drive->{interface} eq 'sata') {
4007             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4008             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4009             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4010                 if !$ahcicontroller->{$controller};
4011             $ahcicontroller->{$controller}=1;
4012         }
4013
4014         my $pbs_conf = $pbs_backing->{$ds};
4015         my $pbs_name = undef;
4016         if ($pbs_conf) {
4017             $pbs_name = "drive-$ds-pbs";
4018             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4019         }
4020
4021         my $drive_cmd = print_drive_commandline_full(
4022             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4023
4024         # extra protection for templates, but SATA and IDE don't support it..
4025         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4026
4027         push @$devices, '-drive',$drive_cmd;
4028         push @$devices, '-device', print_drivedevice_full(
4029             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4030     });
4031
4032     for (my $i = 0; $i < $MAX_NETS; $i++) {
4033         my $netname = "net$i";
4034
4035         next if !$conf->{$netname};
4036         my $d = parse_net($conf->{$netname});
4037         next if !$d;
4038
4039         $use_virtio = 1 if $d->{model} eq 'virtio';
4040
4041         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4042
4043         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4044         push @$devices, '-netdev', $netdevfull;
4045
4046         my $netdevicefull = print_netdevice_full(
4047             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4048
4049         push @$devices, '-device', $netdevicefull;
4050     }
4051
4052     if ($conf->{ivshmem}) {
4053         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4054
4055         my $bus;
4056         if ($q35) {
4057             $bus = print_pcie_addr("ivshmem");
4058         } else {
4059             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4060         }
4061
4062         my $ivshmem_name = $ivshmem->{name} // $vmid;
4063         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4064
4065         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4066         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4067             .",size=$ivshmem->{size}M";
4068     }
4069
4070     # pci.4 is nested in pci.1
4071     $bridges->{1} = 1 if $bridges->{4};
4072
4073     if (!$q35) { # add pci bridges
4074         if (min_version($machine_version, 2, 3)) {
4075            $bridges->{1} = 1;
4076            $bridges->{2} = 1;
4077         }
4078         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4079     }
4080
4081     for my $k (sort {$b cmp $a} keys %$bridges) {
4082         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4083
4084         my $k_name = $k;
4085         if ($k == 2 && $legacy_igd) {
4086             $k_name = "$k-igd";
4087         }
4088         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4089         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4090
4091         if ($q35) { # add after -readconfig pve-q35.cfg
4092             splice @$devices, 2, 0, '-device', $devstr;
4093         } else {
4094             unshift @$devices, '-device', $devstr if $k > 0;
4095         }
4096     }
4097
4098     if (!$kvm) {
4099         push @$machineFlags, 'accel=tcg';
4100     }
4101
4102     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4103
4104     my $machine_type_min = $machine_type;
4105     if ($add_pve_version) {
4106         $machine_type_min =~ s/\+pve\d+$//;
4107         $machine_type_min .= "+pve$required_pve_version";
4108     }
4109     push @$machineFlags, "type=${machine_type_min}";
4110
4111     push @$cmd, @$devices;
4112     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4113     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4114     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4115
4116     if (my $vmstate = $conf->{vmstate}) {
4117         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4118         push @$vollist, $vmstate;
4119         push @$cmd, '-loadstate', $statepath;
4120         print "activating and using '$vmstate' as vmstate\n";
4121     }
4122
4123     if (PVE::QemuConfig->is_template($conf)) {
4124         # needed to workaround base volumes being read-only
4125         push @$cmd, '-snapshot';
4126     }
4127
4128     # add custom args
4129     if ($conf->{args}) {
4130         my $aa = PVE::Tools::split_args($conf->{args});
4131         push @$cmd, @$aa;
4132     }
4133
4134     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4135 }
4136
4137 sub check_rng_source {
4138     my ($source) = @_;
4139
4140     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4141     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4142         if ! -e $source;
4143
4144     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4145     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4146         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4147         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4148         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4149             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4150             ." to the host.\n";
4151     }
4152 }
4153
4154 sub spice_port {
4155     my ($vmid) = @_;
4156
4157     my $res = mon_cmd($vmid, 'query-spice');
4158
4159     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4160 }
4161
4162 sub vm_devices_list {
4163     my ($vmid) = @_;
4164
4165     my $res = mon_cmd($vmid, 'query-pci');
4166     my $devices_to_check = [];
4167     my $devices = {};
4168     foreach my $pcibus (@$res) {
4169         push @$devices_to_check, @{$pcibus->{devices}},
4170     }
4171
4172     while (@$devices_to_check) {
4173         my $to_check = [];
4174         for my $d (@$devices_to_check) {
4175             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4176             next if !$d->{'pci_bridge'};
4177
4178             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4179             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4180         }
4181         $devices_to_check = $to_check;
4182     }
4183
4184     my $resblock = mon_cmd($vmid, 'query-block');
4185     foreach my $block (@$resblock) {
4186         if($block->{device} =~ m/^drive-(\S+)/){
4187                 $devices->{$1} = 1;
4188         }
4189     }
4190
4191     my $resmice = mon_cmd($vmid, 'query-mice');
4192     foreach my $mice (@$resmice) {
4193         if ($mice->{name} eq 'QEMU HID Tablet') {
4194             $devices->{tablet} = 1;
4195             last;
4196         }
4197     }
4198
4199     # for usb devices there is no query-usb
4200     # but we can iterate over the entries in
4201     # qom-list path=/machine/peripheral
4202     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4203     foreach my $per (@$resperipheral) {
4204         if ($per->{name} =~ m/^usb\d+$/) {
4205             $devices->{$per->{name}} = 1;
4206         }
4207     }
4208
4209     return $devices;
4210 }
4211
4212 sub vm_deviceplug {
4213     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4214
4215     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4216
4217     my $devices_list = vm_devices_list($vmid);
4218     return 1 if defined($devices_list->{$deviceid});
4219
4220     # add PCI bridge if we need it for the device
4221     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4222
4223     if ($deviceid eq 'tablet') {
4224         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4225     } elsif ($deviceid eq 'keyboard') {
4226         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4227     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4228         die "usb hotplug currently not reliable\n";
4229         # since we can't reliably hot unplug all added usb devices and usb
4230         # passthrough breaks live migration we disable usb hotplugging for now
4231         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4232     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4233         qemu_iothread_add($vmid, $deviceid, $device);
4234
4235         qemu_driveadd($storecfg, $vmid, $device);
4236         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4237
4238         qemu_deviceadd($vmid, $devicefull);
4239         eval { qemu_deviceaddverify($vmid, $deviceid); };
4240         if (my $err = $@) {
4241             eval { qemu_drivedel($vmid, $deviceid); };
4242             warn $@ if $@;
4243             die $err;
4244         }
4245     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4246         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4247         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4248         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4249
4250         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4251
4252         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4253             qemu_iothread_add($vmid, $deviceid, $device);
4254             $devicefull .= ",iothread=iothread-$deviceid";
4255         }
4256
4257         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4258             $devicefull .= ",num_queues=$device->{queues}";
4259         }
4260
4261         qemu_deviceadd($vmid, $devicefull);
4262         qemu_deviceaddverify($vmid, $deviceid);
4263     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4264         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4265         qemu_driveadd($storecfg, $vmid, $device);
4266
4267         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4268         eval { qemu_deviceadd($vmid, $devicefull); };
4269         if (my $err = $@) {
4270             eval { qemu_drivedel($vmid, $deviceid); };
4271             warn $@ if $@;
4272             die $err;
4273         }
4274     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4275         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4276
4277         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4278         my $use_old_bios_files = undef;
4279         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4280
4281         my $netdevicefull = print_netdevice_full(
4282             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4283         qemu_deviceadd($vmid, $netdevicefull);
4284         eval {
4285             qemu_deviceaddverify($vmid, $deviceid);
4286             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4287         };
4288         if (my $err = $@) {
4289             eval { qemu_netdevdel($vmid, $deviceid); };
4290             warn $@ if $@;
4291             die $err;
4292         }
4293     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4294         my $bridgeid = $2;
4295         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4296         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4297
4298         qemu_deviceadd($vmid, $devicefull);
4299         qemu_deviceaddverify($vmid, $deviceid);
4300     } else {
4301         die "can't hotplug device '$deviceid'\n";
4302     }
4303
4304     return 1;
4305 }
4306
4307 # fixme: this should raise exceptions on error!
4308 sub vm_deviceunplug {
4309     my ($vmid, $conf, $deviceid) = @_;
4310
4311     my $devices_list = vm_devices_list($vmid);
4312     return 1 if !defined($devices_list->{$deviceid});
4313
4314     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4315     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4316
4317     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4318         qemu_devicedel($vmid, $deviceid);
4319     } elsif ($deviceid =~ m/^usb\d+$/) {
4320         die "usb hotplug currently not reliable\n";
4321         # when unplugging usb devices this way, there may be remaining usb
4322         # controllers/hubs so we disable it for now
4323         #qemu_devicedel($vmid, $deviceid);
4324         #qemu_devicedelverify($vmid, $deviceid);
4325     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4326         my $device = parse_drive($deviceid, $conf->{$deviceid});
4327
4328         qemu_devicedel($vmid, $deviceid);
4329         qemu_devicedelverify($vmid, $deviceid);
4330         qemu_drivedel($vmid, $deviceid);
4331         qemu_iothread_del($vmid, $deviceid, $device);
4332     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4333         qemu_devicedel($vmid, $deviceid);
4334         qemu_devicedelverify($vmid, $deviceid);
4335     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4336         my $device = parse_drive($deviceid, $conf->{$deviceid});
4337
4338         qemu_devicedel($vmid, $deviceid);
4339         qemu_devicedelverify($vmid, $deviceid);
4340         qemu_drivedel($vmid, $deviceid);
4341         qemu_deletescsihw($conf, $vmid, $deviceid);
4342
4343         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4344             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4345     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348         qemu_netdevdel($vmid, $deviceid);
4349     } else {
4350         die "can't unplug device '$deviceid'\n";
4351     }
4352
4353     return 1;
4354 }
4355
4356 sub qemu_deviceadd {
4357     my ($vmid, $devicefull) = @_;
4358
4359     $devicefull = "driver=".$devicefull;
4360     my %options =  split(/[=,]/, $devicefull);
4361
4362     mon_cmd($vmid, "device_add" , %options);
4363 }
4364
4365 sub qemu_devicedel {
4366     my ($vmid, $deviceid) = @_;
4367
4368     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4369 }
4370
4371 sub qemu_iothread_add {
4372     my ($vmid, $deviceid, $device) = @_;
4373
4374     if ($device->{iothread}) {
4375         my $iothreads = vm_iothreads_list($vmid);
4376         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4377     }
4378 }
4379
4380 sub qemu_iothread_del {
4381     my ($vmid, $deviceid, $device) = @_;
4382
4383     if ($device->{iothread}) {
4384         my $iothreads = vm_iothreads_list($vmid);
4385         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4386     }
4387 }
4388
4389 sub qemu_objectadd {
4390     my ($vmid, $objectid, $qomtype) = @_;
4391
4392     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4393
4394     return 1;
4395 }
4396
4397 sub qemu_objectdel {
4398     my ($vmid, $objectid) = @_;
4399
4400     mon_cmd($vmid, "object-del", id => $objectid);
4401
4402     return 1;
4403 }
4404
4405 sub qemu_driveadd {
4406     my ($storecfg, $vmid, $device) = @_;
4407
4408     my $kvmver = get_running_qemu_version($vmid);
4409     my $io_uring = min_version($kvmver, 6, 0);
4410     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4411     $drive =~ s/\\/\\\\/g;
4412     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4413
4414     # If the command succeeds qemu prints: "OK"
4415     return 1 if $ret =~ m/OK/s;
4416
4417     die "adding drive failed: $ret\n";
4418 }
4419
4420 sub qemu_drivedel {
4421     my ($vmid, $deviceid) = @_;
4422
4423     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4424     $ret =~ s/^\s+//;
4425
4426     return 1 if $ret eq "";
4427
4428     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4429     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4430
4431     die "deleting drive $deviceid failed : $ret\n";
4432 }
4433
4434 sub qemu_deviceaddverify {
4435     my ($vmid, $deviceid) = @_;
4436
4437     for (my $i = 0; $i <= 5; $i++) {
4438          my $devices_list = vm_devices_list($vmid);
4439          return 1 if defined($devices_list->{$deviceid});
4440          sleep 1;
4441     }
4442
4443     die "error on hotplug device '$deviceid'\n";
4444 }
4445
4446
4447 sub qemu_devicedelverify {
4448     my ($vmid, $deviceid) = @_;
4449
4450     # need to verify that the device is correctly removed as device_del
4451     # is async and empty return is not reliable
4452
4453     for (my $i = 0; $i <= 5; $i++) {
4454          my $devices_list = vm_devices_list($vmid);
4455          return 1 if !defined($devices_list->{$deviceid});
4456          sleep 1;
4457     }
4458
4459     die "error on hot-unplugging device '$deviceid'\n";
4460 }
4461
4462 sub qemu_findorcreatescsihw {
4463     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4464
4465     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4466
4467     my $scsihwid="$controller_prefix$controller";
4468     my $devices_list = vm_devices_list($vmid);
4469
4470     if (!defined($devices_list->{$scsihwid})) {
4471         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4472     }
4473
4474     return 1;
4475 }
4476
4477 sub qemu_deletescsihw {
4478     my ($conf, $vmid, $opt) = @_;
4479
4480     my $device = parse_drive($opt, $conf->{$opt});
4481
4482     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4483         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4484         return 1;
4485     }
4486
4487     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4488
4489     my $devices_list = vm_devices_list($vmid);
4490     foreach my $opt (keys %{$devices_list}) {
4491         if (is_valid_drivename($opt)) {
4492             my $drive = parse_drive($opt, $conf->{$opt});
4493             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4494                 return 1;
4495             }
4496         }
4497     }
4498
4499     my $scsihwid="scsihw$controller";
4500
4501     vm_deviceunplug($vmid, $conf, $scsihwid);
4502
4503     return 1;
4504 }
4505
4506 sub qemu_add_pci_bridge {
4507     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4508
4509     my $bridges = {};
4510
4511     my $bridgeid;
4512
4513     print_pci_addr($device, $bridges, $arch, $machine_type);
4514
4515     while (my ($k, $v) = each %$bridges) {
4516         $bridgeid = $k;
4517     }
4518     return 1 if !defined($bridgeid) || $bridgeid < 1;
4519
4520     my $bridge = "pci.$bridgeid";
4521     my $devices_list = vm_devices_list($vmid);
4522
4523     if (!defined($devices_list->{$bridge})) {
4524         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4525     }
4526
4527     return 1;
4528 }
4529
4530 sub qemu_set_link_status {
4531     my ($vmid, $device, $up) = @_;
4532
4533     mon_cmd($vmid, "set_link", name => $device,
4534                up => $up ? JSON::true : JSON::false);
4535 }
4536
4537 sub qemu_netdevadd {
4538     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4539
4540     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4541     my %options =  split(/[=,]/, $netdev);
4542
4543     if (defined(my $vhost = $options{vhost})) {
4544         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4545     }
4546
4547     if (defined(my $queues = $options{queues})) {
4548         $options{queues} = $queues + 0;
4549     }
4550
4551     mon_cmd($vmid, "netdev_add",  %options);
4552     return 1;
4553 }
4554
4555 sub qemu_netdevdel {
4556     my ($vmid, $deviceid) = @_;
4557
4558     mon_cmd($vmid, "netdev_del", id => $deviceid);
4559 }
4560
4561 sub qemu_usb_hotplug {
4562     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4563
4564     return if !$device;
4565
4566     # remove the old one first
4567     vm_deviceunplug($vmid, $conf, $deviceid);
4568
4569     # check if xhci controller is necessary and available
4570     if ($device->{usb3}) {
4571
4572         my $devicelist = vm_devices_list($vmid);
4573
4574         if (!$devicelist->{xhci}) {
4575             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4576             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4577         }
4578     }
4579     my $d = parse_usb_device($device->{host});
4580     $d->{usb3} = $device->{usb3};
4581
4582     # add the new one
4583     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4584 }
4585
4586 sub qemu_cpu_hotplug {
4587     my ($vmid, $conf, $vcpus) = @_;
4588
4589     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4590
4591     my $sockets = 1;
4592     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4593     $sockets = $conf->{sockets} if  $conf->{sockets};
4594     my $cores = $conf->{cores} || 1;
4595     my $maxcpus = $sockets * $cores;
4596
4597     $vcpus = $maxcpus if !$vcpus;
4598
4599     die "you can't add more vcpus than maxcpus\n"
4600         if $vcpus > $maxcpus;
4601
4602     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4603
4604     if ($vcpus < $currentvcpus) {
4605
4606         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4607
4608             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4609                 qemu_devicedel($vmid, "cpu$i");
4610                 my $retry = 0;
4611                 my $currentrunningvcpus = undef;
4612                 while (1) {
4613                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4614                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4615                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4616                     $retry++;
4617                     sleep 1;
4618                 }
4619                 #update conf after each succesfull cpu unplug
4620                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4621                 PVE::QemuConfig->write_config($vmid, $conf);
4622             }
4623         } else {
4624             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4625         }
4626
4627         return;
4628     }
4629
4630     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4631     die "vcpus in running vm does not match its configuration\n"
4632         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4633
4634     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4635
4636         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4637             my $cpustr = print_cpu_device($conf, $i);
4638             qemu_deviceadd($vmid, $cpustr);
4639
4640             my $retry = 0;
4641             my $currentrunningvcpus = undef;
4642             while (1) {
4643                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4644                 last if scalar(@{$currentrunningvcpus}) == $i;
4645                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4646                 sleep 1;
4647                 $retry++;
4648             }
4649             #update conf after each succesfull cpu hotplug
4650             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4651             PVE::QemuConfig->write_config($vmid, $conf);
4652         }
4653     } else {
4654
4655         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4656             mon_cmd($vmid, "cpu-add", id => int($i));
4657         }
4658     }
4659 }
4660
4661 sub qemu_block_set_io_throttle {
4662     my ($vmid, $deviceid,
4663         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4664         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4665         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4666         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4667
4668     return if !check_running($vmid) ;
4669
4670     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4671         bps => int($bps),
4672         bps_rd => int($bps_rd),
4673         bps_wr => int($bps_wr),
4674         iops => int($iops),
4675         iops_rd => int($iops_rd),
4676         iops_wr => int($iops_wr),
4677         bps_max => int($bps_max),
4678         bps_rd_max => int($bps_rd_max),
4679         bps_wr_max => int($bps_wr_max),
4680         iops_max => int($iops_max),
4681         iops_rd_max => int($iops_rd_max),
4682         iops_wr_max => int($iops_wr_max),
4683         bps_max_length => int($bps_max_length),
4684         bps_rd_max_length => int($bps_rd_max_length),
4685         bps_wr_max_length => int($bps_wr_max_length),
4686         iops_max_length => int($iops_max_length),
4687         iops_rd_max_length => int($iops_rd_max_length),
4688         iops_wr_max_length => int($iops_wr_max_length),
4689     );
4690
4691 }
4692
4693 sub qemu_block_resize {
4694     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4695
4696     my $running = check_running($vmid);
4697
4698     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4699
4700     return if !$running;
4701
4702     my $padding = (1024 - $size % 1024) % 1024;
4703     $size = $size + $padding;
4704
4705     mon_cmd(
4706         $vmid,
4707         "block_resize",
4708         device => $deviceid,
4709         size => int($size),
4710         timeout => 60,
4711     );
4712 }
4713
4714 sub qemu_volume_snapshot {
4715     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4716
4717     my $running = check_running($vmid);
4718
4719     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4720         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4721     } else {
4722         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4723     }
4724 }
4725
4726 sub qemu_volume_snapshot_delete {
4727     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4728
4729     my $running = check_running($vmid);
4730
4731     if($running) {
4732
4733         $running = undef;
4734         my $conf = PVE::QemuConfig->load_config($vmid);
4735         PVE::QemuConfig->foreach_volume($conf, sub {
4736             my ($ds, $drive) = @_;
4737             $running = 1 if $drive->{file} eq $volid;
4738         });
4739     }
4740
4741     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4742         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4743     } else {
4744         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4745     }
4746 }
4747
4748 sub set_migration_caps {
4749     my ($vmid, $savevm) = @_;
4750
4751     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4752
4753     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4754     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4755
4756     my $cap_ref = [];
4757
4758     my $enabled_cap = {
4759         "auto-converge" => 1,
4760         "xbzrle" => 1,
4761         "x-rdma-pin-all" => 0,
4762         "zero-blocks" => 0,
4763         "compress" => 0,
4764         "dirty-bitmaps" => $dirty_bitmaps,
4765     };
4766
4767     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4768
4769     for my $supported_capability (@$supported_capabilities) {
4770         push @$cap_ref, {
4771             capability => $supported_capability->{capability},
4772             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4773         };
4774     }
4775
4776     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4777 }
4778
4779 sub foreach_volid {
4780     my ($conf, $func, @param) = @_;
4781
4782     my $volhash = {};
4783
4784     my $test_volid = sub {
4785         my ($key, $drive, $snapname) = @_;
4786
4787         my $volid = $drive->{file};
4788         return if !$volid;
4789
4790         $volhash->{$volid}->{cdrom} //= 1;
4791         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4792
4793         my $replicate = $drive->{replicate} // 1;
4794         $volhash->{$volid}->{replicate} //= 0;
4795         $volhash->{$volid}->{replicate} = 1 if $replicate;
4796
4797         $volhash->{$volid}->{shared} //= 0;
4798         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4799
4800         $volhash->{$volid}->{referenced_in_config} //= 0;
4801         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4802
4803         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4804             if defined($snapname);
4805
4806         my $size = $drive->{size};
4807         $volhash->{$volid}->{size} //= $size if $size;
4808
4809         $volhash->{$volid}->{is_vmstate} //= 0;
4810         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4811
4812         $volhash->{$volid}->{is_tpmstate} //= 0;
4813         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4814
4815         $volhash->{$volid}->{is_unused} //= 0;
4816         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4817
4818         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4819     };
4820
4821     my $include_opts = {
4822         extra_keys => ['vmstate'],
4823         include_unused => 1,
4824     };
4825
4826     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4827     foreach my $snapname (keys %{$conf->{snapshots}}) {
4828         my $snap = $conf->{snapshots}->{$snapname};
4829         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4830     }
4831
4832     foreach my $volid (keys %$volhash) {
4833         &$func($volid, $volhash->{$volid}, @param);
4834     }
4835 }
4836
4837 my $fast_plug_option = {
4838     'lock' => 1,
4839     'name' => 1,
4840     'onboot' => 1,
4841     'shares' => 1,
4842     'startup' => 1,
4843     'description' => 1,
4844     'protection' => 1,
4845     'vmstatestorage' => 1,
4846     'hookscript' => 1,
4847     'tags' => 1,
4848 };
4849
4850 for my $opt (keys %$confdesc_cloudinit) {
4851     $fast_plug_option->{$opt} = 1;
4852 };
4853
4854 # hotplug changes in [PENDING]
4855 # $selection hash can be used to only apply specified options, for
4856 # example: { cores => 1 } (only apply changed 'cores')
4857 # $errors ref is used to return error messages
4858 sub vmconfig_hotplug_pending {
4859     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4860
4861     my $defaults = load_defaults();
4862     my $arch = get_vm_arch($conf);
4863     my $machine_type = get_vm_machine($conf, undef, $arch);
4864
4865     # commit values which do not have any impact on running VM first
4866     # Note: those option cannot raise errors, we we do not care about
4867     # $selection and always apply them.
4868
4869     my $add_error = sub {
4870         my ($opt, $msg) = @_;
4871         $errors->{$opt} = "hotplug problem - $msg";
4872     };
4873
4874     my $changes = 0;
4875     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4876         if ($fast_plug_option->{$opt}) {
4877             $conf->{$opt} = $conf->{pending}->{$opt};
4878             delete $conf->{pending}->{$opt};
4879             $changes = 1;
4880         }
4881     }
4882
4883     if ($changes) {
4884         PVE::QemuConfig->write_config($vmid, $conf);
4885     }
4886
4887     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4888
4889     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4890     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4891     foreach my $opt (sort keys %$pending_delete_hash) {
4892         next if $selection && !$selection->{$opt};
4893         my $force = $pending_delete_hash->{$opt}->{force};
4894         eval {
4895             if ($opt eq 'hotplug') {
4896                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4897             } elsif ($opt eq 'tablet') {
4898                 die "skip\n" if !$hotplug_features->{usb};
4899                 if ($defaults->{tablet}) {
4900                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4901                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4902                         if $arch eq 'aarch64';
4903                 } else {
4904                     vm_deviceunplug($vmid, $conf, 'tablet');
4905                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4906                 }
4907             } elsif ($opt =~ m/^usb\d+/) {
4908                 die "skip\n";
4909                 # since we cannot reliably hot unplug usb devices we are disabling it
4910                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4911                 #vm_deviceunplug($vmid, $conf, $opt);
4912             } elsif ($opt eq 'vcpus') {
4913                 die "skip\n" if !$hotplug_features->{cpu};
4914                 qemu_cpu_hotplug($vmid, $conf, undef);
4915             } elsif ($opt eq 'balloon') {
4916                 # enable balloon device is not hotpluggable
4917                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4918                 # here we reset the ballooning value to memory
4919                 my $balloon = $conf->{memory} || $defaults->{memory};
4920                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4921             } elsif ($fast_plug_option->{$opt}) {
4922                 # do nothing
4923             } elsif ($opt =~ m/^net(\d+)$/) {
4924                 die "skip\n" if !$hotplug_features->{network};
4925                 vm_deviceunplug($vmid, $conf, $opt);
4926             } elsif (is_valid_drivename($opt)) {
4927                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4928                 vm_deviceunplug($vmid, $conf, $opt);
4929                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4930             } elsif ($opt =~ m/^memory$/) {
4931                 die "skip\n" if !$hotplug_features->{memory};
4932                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4933             } elsif ($opt eq 'cpuunits') {
4934                 $cgroup->change_cpu_shares(undef);
4935             } elsif ($opt eq 'cpulimit') {
4936                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4937             } else {
4938                 die "skip\n";
4939             }
4940         };
4941         if (my $err = $@) {
4942             &$add_error($opt, $err) if $err ne "skip\n";
4943         } else {
4944             delete $conf->{$opt};
4945             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4946         }
4947     }
4948
4949     foreach my $opt (keys %{$conf->{pending}}) {
4950         next if $selection && !$selection->{$opt};
4951         my $value = $conf->{pending}->{$opt};
4952         eval {
4953             if ($opt eq 'hotplug') {
4954                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4955             } elsif ($opt eq 'tablet') {
4956                 die "skip\n" if !$hotplug_features->{usb};
4957                 if ($value == 1) {
4958                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4959                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4960                         if $arch eq 'aarch64';
4961                 } elsif ($value == 0) {
4962                     vm_deviceunplug($vmid, $conf, 'tablet');
4963                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4964                 }
4965             } elsif ($opt =~ m/^usb\d+$/) {
4966                 die "skip\n";
4967                 # since we cannot reliably hot unplug usb devices we disable it for now
4968                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4969                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4970                 #die "skip\n" if !$d;
4971                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4972             } elsif ($opt eq 'vcpus') {
4973                 die "skip\n" if !$hotplug_features->{cpu};
4974                 qemu_cpu_hotplug($vmid, $conf, $value);
4975             } elsif ($opt eq 'balloon') {
4976                 # enable/disable balloning device is not hotpluggable
4977                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4978                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4979                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4980
4981                 # allow manual ballooning if shares is set to zero
4982                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4983                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4984                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4985                 }
4986             } elsif ($opt =~ m/^net(\d+)$/) {
4987                 # some changes can be done without hotplug
4988                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4989                                     $vmid, $opt, $value, $arch, $machine_type);
4990             } elsif (is_valid_drivename($opt)) {
4991                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4992                 # some changes can be done without hotplug
4993                 my $drive = parse_drive($opt, $value);
4994                 if (drive_is_cloudinit($drive)) {
4995                     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4996                 }
4997                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4998                                      $vmid, $opt, $value, $arch, $machine_type);
4999             } elsif ($opt =~ m/^memory$/) { #dimms
5000                 die "skip\n" if !$hotplug_features->{memory};
5001                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5002             } elsif ($opt eq 'cpuunits') {
5003                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5004                 $cgroup->change_cpu_shares($new_cpuunits);
5005             } elsif ($opt eq 'cpulimit') {
5006                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5007                 $cgroup->change_cpu_quota($cpulimit, 100000);
5008             } elsif ($opt eq 'agent') {
5009                 vmconfig_update_agent($conf, $opt, $value);
5010             } else {
5011                 die "skip\n";  # skip non-hot-pluggable options
5012             }
5013         };
5014         if (my $err = $@) {
5015             &$add_error($opt, $err) if $err ne "skip\n";
5016         } else {
5017             $conf->{$opt} = $value;
5018             delete $conf->{pending}->{$opt};
5019         }
5020     }
5021     PVE::QemuConfig->write_config($vmid, $conf);
5022
5023     if($hotplug_features->{cloudinit}) {
5024         my $pending = PVE::QemuServer::Cloudinit::get_pending_config($conf, $vmid);
5025         my $regenerate = undef;
5026         for my $item (@$pending) {
5027             $regenerate = 1 if defined($item->{delete}) or defined($item->{pending});
5028         }
5029         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid) if $regenerate;
5030     }
5031 }
5032
5033 sub try_deallocate_drive {
5034     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5035
5036     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5037         my $volid = $drive->{file};
5038         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5039             my $sid = PVE::Storage::parse_volume_id($volid);
5040             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5041
5042             # check if the disk is really unused
5043             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5044                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5045             PVE::Storage::vdisk_free($storecfg, $volid);
5046             return 1;
5047         } else {
5048             # If vm is not owner of this disk remove from config
5049             return 1;
5050         }
5051     }
5052
5053     return;
5054 }
5055
5056 sub vmconfig_delete_or_detach_drive {
5057     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5058
5059     my $drive = parse_drive($opt, $conf->{$opt});
5060
5061     my $rpcenv = PVE::RPCEnvironment::get();
5062     my $authuser = $rpcenv->get_user();
5063
5064     if ($force) {
5065         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5066         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5067     } else {
5068         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5069     }
5070 }
5071
5072
5073
5074 sub vmconfig_apply_pending {
5075     my ($vmid, $conf, $storecfg, $errors) = @_;
5076
5077     return if !scalar(keys %{$conf->{pending}});
5078
5079     my $add_apply_error = sub {
5080         my ($opt, $msg) = @_;
5081         my $err_msg = "unable to apply pending change $opt : $msg";
5082         $errors->{$opt} = $err_msg;
5083         warn $err_msg;
5084     };
5085
5086     # cold plug
5087
5088     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5089     foreach my $opt (sort keys %$pending_delete_hash) {
5090         my $force = $pending_delete_hash->{$opt}->{force};
5091         eval {
5092             if ($opt =~ m/^unused/) {
5093                 die "internal error";
5094             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5095                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5096             }
5097         };
5098         if (my $err = $@) {
5099             $add_apply_error->($opt, $err);
5100         } else {
5101             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5102             delete $conf->{$opt};
5103         }
5104     }
5105
5106     PVE::QemuConfig->cleanup_pending($conf);
5107
5108     my $generate_cloudnit = undef;
5109
5110     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5111         next if $opt eq 'delete'; # just to be sure
5112         eval {
5113             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5114                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5115             }
5116         };
5117         if (my $err = $@) {
5118             $add_apply_error->($opt, $err);
5119         } else {
5120
5121             if (is_valid_drivename($opt)) {
5122                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5123                 $generate_cloudnit = 1 if drive_is_cloudinit($drive);
5124             }
5125
5126             $conf->{$opt} = delete $conf->{pending}->{$opt};
5127         }
5128     }
5129
5130     # write all changes at once to avoid unnecessary i/o
5131     PVE::QemuConfig->write_config($vmid, $conf);
5132     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if $generate_cloudnit;
5133 }
5134
5135 sub vmconfig_update_net {
5136     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5137
5138     my $newnet = parse_net($value);
5139
5140     if ($conf->{$opt}) {
5141         my $oldnet = parse_net($conf->{$opt});
5142
5143         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5144             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5145             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5146             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5147
5148             # for non online change, we try to hot-unplug
5149             die "skip\n" if !$hotplug;
5150             vm_deviceunplug($vmid, $conf, $opt);
5151         } else {
5152
5153             die "internal error" if $opt !~ m/net(\d+)/;
5154             my $iface = "tap${vmid}i$1";
5155
5156             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5157                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5158                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5159                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5160                 PVE::Network::tap_unplug($iface);
5161
5162                 if ($have_sdn) {
5163                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5164                 } else {
5165                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5166                 }
5167             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5168                 # Rate can be applied on its own but any change above needs to
5169                 # include the rate in tap_plug since OVS resets everything.
5170                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5171             }
5172
5173             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5174                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5175             }
5176
5177             return 1;
5178         }
5179     }
5180
5181     if ($hotplug) {
5182         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5183     } else {
5184         die "skip\n";
5185     }
5186 }
5187
5188 sub vmconfig_update_agent {
5189     my ($conf, $opt, $value) = @_;
5190
5191     die "skip\n" if !$conf->{$opt};
5192
5193     my $hotplug_options = { fstrim_cloned_disks => 1 };
5194
5195     my $old_agent = parse_guest_agent($conf);
5196     my $agent = parse_guest_agent({$opt => $value});
5197
5198     for my $option (keys %$agent) { # added/changed options
5199         next if defined($hotplug_options->{$option});
5200         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5201     }
5202
5203     for my $option (keys %$old_agent) { # removed options
5204         next if defined($hotplug_options->{$option});
5205         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5206     }
5207
5208     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5209 }
5210
5211 sub vmconfig_update_disk {
5212     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5213
5214     my $drive = parse_drive($opt, $value);
5215
5216     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5217         my $media = $drive->{media} || 'disk';
5218         my $oldmedia = $old_drive->{media} || 'disk';
5219         die "unable to change media type\n" if $media ne $oldmedia;
5220
5221         if (!drive_is_cdrom($old_drive)) {
5222
5223             if ($drive->{file} ne $old_drive->{file}) {
5224
5225                 die "skip\n" if !$hotplug;
5226
5227                 # unplug and register as unused
5228                 vm_deviceunplug($vmid, $conf, $opt);
5229                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5230
5231             } else {
5232                 # update existing disk
5233
5234                 # skip non hotpluggable value
5235                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5236                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5237                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5238                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5239                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5240                     die "skip\n";
5241                 }
5242
5243                 # apply throttle
5244                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5245                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5246                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5247                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5248                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5249                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5250                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5251                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5252                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5253                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5254                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5255                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5256                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5257                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5258                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5259                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5260                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5261                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5262
5263                     qemu_block_set_io_throttle(
5264                         $vmid,"drive-$opt",
5265                         ($drive->{mbps} || 0)*1024*1024,
5266                         ($drive->{mbps_rd} || 0)*1024*1024,
5267                         ($drive->{mbps_wr} || 0)*1024*1024,
5268                          $drive->{iops} || 0,
5269                          $drive->{iops_rd} || 0,
5270                          $drive->{iops_wr} || 0,
5271                         ($drive->{mbps_max} || 0)*1024*1024,
5272                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5273                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5274                          $drive->{iops_max} || 0,
5275                          $drive->{iops_rd_max} || 0,
5276                          $drive->{iops_wr_max} || 0,
5277                          $drive->{bps_max_length} || 1,
5278                          $drive->{bps_rd_max_length} || 1,
5279                          $drive->{bps_wr_max_length} || 1,
5280                          $drive->{iops_max_length} || 1,
5281                          $drive->{iops_rd_max_length} || 1,
5282                          $drive->{iops_wr_max_length} || 1,
5283                     );
5284
5285                 }
5286
5287                 return 1;
5288             }
5289
5290         } else { # cdrom
5291
5292             if ($drive->{file} eq 'none') {
5293                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5294                 if (drive_is_cloudinit($old_drive)) {
5295                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5296                 }
5297             } else {
5298                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5299
5300                 # force eject if locked
5301                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5302
5303                 if ($path) {
5304                     mon_cmd($vmid, "blockdev-change-medium",
5305                         id => "$opt", filename => "$path");
5306                 }
5307             }
5308
5309             return 1;
5310         }
5311     }
5312
5313     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5314     # hotplug new disks
5315     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5316     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5317 }
5318
5319 sub vmconfig_update_cloudinit_drive {
5320     my ($storecfg, $conf, $vmid) = @_;
5321
5322     my $cloudinit_ds = undef;
5323     my $cloudinit_drive = undef;
5324
5325     PVE::QemuConfig->foreach_volume($conf, sub {
5326         my ($ds, $drive) = @_;
5327         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5328             $cloudinit_ds = $ds;
5329             $cloudinit_drive = $drive;
5330         }
5331     });
5332
5333     return if !$cloudinit_drive;
5334
5335     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5336     my $running = PVE::QemuServer::check_running($vmid);
5337
5338     if ($running) {
5339         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5340         if ($path) {
5341             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5342             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5343         }
5344     }
5345 }
5346
5347 # called in locked context by incoming migration
5348 sub vm_migrate_get_nbd_disks {
5349     my ($storecfg, $conf, $replicated_volumes) = @_;
5350
5351     my $local_volumes = {};
5352     PVE::QemuConfig->foreach_volume($conf, sub {
5353         my ($ds, $drive) = @_;
5354
5355         return if drive_is_cdrom($drive);
5356         return if $ds eq 'tpmstate0';
5357
5358         my $volid = $drive->{file};
5359
5360         return if !$volid;
5361
5362         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5363
5364         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5365         return if $scfg->{shared};
5366
5367         # replicated disks re-use existing state via bitmap
5368         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5369         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5370     });
5371     return $local_volumes;
5372 }
5373
5374 # called in locked context by incoming migration
5375 sub vm_migrate_alloc_nbd_disks {
5376     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5377
5378     my $nbd = {};
5379     foreach my $opt (sort keys %$source_volumes) {
5380         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5381
5382         if ($use_existing) {
5383             $nbd->{$opt}->{drivestr} = print_drive($drive);
5384             $nbd->{$opt}->{volid} = $volid;
5385             $nbd->{$opt}->{replicated} = 1;
5386             next;
5387         }
5388
5389         # storage mapping + volname = regular migration
5390         # storage mapping + format = remote migration
5391         # order of precedence, filtered by whether storage supports it:
5392         # 1. explicit requested format
5393         # 2. format of current volume
5394         # 3. default format of storage
5395         if (!$storagemap->{identity}) {
5396             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5397             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5398             if (!$format || !grep { $format eq $_ } @$validFormats) {
5399                 if ($volname) {
5400                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5401                     my $fileFormat = qemu_img_format($scfg, $volname);
5402                     $format = $fileFormat
5403                         if grep { $fileFormat eq $_ } @$validFormats;
5404                 }
5405                 $format //= $defFormat;
5406             }
5407         } else {
5408             # can't happen for remote migration, so $volname is always defined
5409             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5410             $format = qemu_img_format($scfg, $volname);
5411         }
5412
5413         my $size = $drive->{size} / 1024;
5414         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5415         my $newdrive = $drive;
5416         $newdrive->{format} = $format;
5417         $newdrive->{file} = $newvolid;
5418         my $drivestr = print_drive($newdrive);
5419         $nbd->{$opt}->{drivestr} = $drivestr;
5420         $nbd->{$opt}->{volid} = $newvolid;
5421     }
5422
5423     return $nbd;
5424 }
5425
5426 # see vm_start_nolock for parameters, additionally:
5427 # migrate_opts:
5428 #   storagemap = parsed storage map for allocating NBD disks
5429 sub vm_start {
5430     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5431
5432     return PVE::QemuConfig->lock_config($vmid, sub {
5433         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5434
5435         die "you can't start a vm if it's a template\n"
5436             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5437
5438         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5439         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5440
5441         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5442
5443         if ($has_backup_lock && $running) {
5444             # a backup is currently running, attempt to start the guest in the
5445             # existing QEMU instance
5446             return vm_resume($vmid);
5447         }
5448
5449         PVE::QemuConfig->check_lock($conf)
5450             if !($params->{skiplock} || $has_suspended_lock);
5451
5452         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5453
5454         die "VM $vmid already running\n" if $running;
5455
5456         if (my $storagemap = $migrate_opts->{storagemap}) {
5457             my $replicated = $migrate_opts->{replicated_volumes};
5458             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5459             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5460
5461             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5462                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5463             }
5464         }
5465
5466         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5467     });
5468 }
5469
5470
5471 # params:
5472 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5473 #   skiplock => 0/1, skip checking for config lock
5474 #   skiptemplate => 0/1, skip checking whether VM is template
5475 #   forcemachine => to force Qemu machine (rollback/migration)
5476 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5477 #   timeout => in seconds
5478 #   paused => start VM in paused state (backup)
5479 #   resume => resume from hibernation
5480 #   pbs-backing => {
5481 #      sata0 => {
5482 #         repository
5483 #         snapshot
5484 #         keyfile
5485 #         archive
5486 #      },
5487 #      virtio2 => ...
5488 #   }
5489 # migrate_opts:
5490 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5491 #   migratedfrom => source node
5492 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5493 #   network => CIDR of migration network
5494 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5495 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5496 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5497 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5498 #       contained in config
5499 sub vm_start_nolock {
5500     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5501
5502     my $statefile = $params->{statefile};
5503     my $resume = $params->{resume};
5504
5505     my $migratedfrom = $migrate_opts->{migratedfrom};
5506     my $migration_type = $migrate_opts->{type};
5507
5508     my $res = {};
5509
5510     # clean up leftover reboot request files
5511     eval { clear_reboot_request($vmid); };
5512     warn $@ if $@;
5513
5514     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5515         vmconfig_apply_pending($vmid, $conf, $storecfg);
5516         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5517     }
5518
5519     # don't regenerate the ISO if the VM is started as part of a live migration
5520     # this way we can reuse the old ISO with the correct config
5521     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5522
5523     # override offline migrated volumes, conf is out of date still
5524     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5525         for my $key (sort keys $offline_volumes->%*) {
5526             my $parsed = parse_drive($key, $conf->{$key});
5527             $parsed->{file} = $offline_volumes->{$key};
5528             $conf->{$key} = print_drive($parsed);
5529         }
5530     }
5531
5532     my $defaults = load_defaults();
5533
5534     # set environment variable useful inside network script
5535     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5536
5537     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5538
5539     my $forcemachine = $params->{forcemachine};
5540     my $forcecpu = $params->{forcecpu};
5541     if ($resume) {
5542         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5543         $forcemachine = $conf->{runningmachine};
5544         $forcecpu = $conf->{runningcpu};
5545         print "Resuming suspended VM\n";
5546     }
5547
5548     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5549         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5550
5551     my $migration_ip;
5552     my $get_migration_ip = sub {
5553         my ($nodename) = @_;
5554
5555         return $migration_ip if defined($migration_ip);
5556
5557         my $cidr = $migrate_opts->{network};
5558
5559         if (!defined($cidr)) {
5560             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5561             $cidr = $dc_conf->{migration}->{network};
5562         }
5563
5564         if (defined($cidr)) {
5565             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5566
5567             die "could not get IP: no address configured on local " .
5568                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5569
5570             die "could not get IP: multiple addresses configured on local " .
5571                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5572
5573             $migration_ip = @$ips[0];
5574         }
5575
5576         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5577             if !defined($migration_ip);
5578
5579         return $migration_ip;
5580     };
5581
5582     my $migrate_uri;
5583     if ($statefile) {
5584         if ($statefile eq 'tcp') {
5585             my $localip = "localhost";
5586             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5587             my $nodename = nodename();
5588
5589             if (!defined($migration_type)) {
5590                 if (defined($datacenterconf->{migration}->{type})) {
5591                     $migration_type = $datacenterconf->{migration}->{type};
5592                 } else {
5593                     $migration_type = 'secure';
5594                 }
5595             }
5596
5597             if ($migration_type eq 'insecure') {
5598                 $localip = $get_migration_ip->($nodename);
5599                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5600             }
5601
5602             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5603             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5604             $migrate_uri = "tcp:${localip}:${migrate_port}";
5605             push @$cmd, '-incoming', $migrate_uri;
5606             push @$cmd, '-S';
5607
5608         } elsif ($statefile eq 'unix') {
5609             # should be default for secure migrations as a ssh TCP forward
5610             # tunnel is not deterministic reliable ready and fails regurarly
5611             # to set up in time, so use UNIX socket forwards
5612             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5613             unlink $socket_addr;
5614
5615             $migrate_uri = "unix:$socket_addr";
5616
5617             push @$cmd, '-incoming', $migrate_uri;
5618             push @$cmd, '-S';
5619
5620         } elsif (-e $statefile) {
5621             push @$cmd, '-loadstate', $statefile;
5622         } else {
5623             my $statepath = PVE::Storage::path($storecfg, $statefile);
5624             push @$vollist, $statefile;
5625             push @$cmd, '-loadstate', $statepath;
5626         }
5627     } elsif ($params->{paused}) {
5628         push @$cmd, '-S';
5629     }
5630
5631     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5632
5633     my $pci_devices = {}; # host pci devices
5634     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5635         my $dev = $conf->{"hostpci$i"} or next;
5636         $pci_devices->{$i} = parse_hostpci($dev);
5637     }
5638
5639     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5640     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5641
5642     # map to a flat list of pci ids
5643     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5644
5645     # reserve all PCI IDs before actually doing anything with them
5646     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5647
5648     eval {
5649         my $uuid;
5650         for my $id (sort keys %$pci_devices) {
5651             my $d = $pci_devices->{$id};
5652             for my $dev ($d->{pciid}->@*) {
5653                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5654
5655                 # nvidia grid needs the uuid of the mdev as qemu parameter
5656                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5657                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5658                 }
5659             }
5660         }
5661         push @$cmd, '-uuid', $uuid if defined($uuid);
5662     };
5663     if (my $err = $@) {
5664         eval { cleanup_pci_devices($vmid, $conf) };
5665         warn $@ if $@;
5666         die $err;
5667     }
5668
5669     PVE::Storage::activate_volumes($storecfg, $vollist);
5670
5671     eval {
5672         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5673     };
5674     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5675     # timeout should be more than enough here...
5676     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5677
5678     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5679
5680     my %run_params = (
5681         timeout => $statefile ? undef : $start_timeout,
5682         umask => 0077,
5683         noerr => 1,
5684     );
5685
5686     # when migrating, prefix QEMU output so other side can pick up any
5687     # errors that might occur and show the user
5688     if ($migratedfrom) {
5689         $run_params{quiet} = 1;
5690         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5691     }
5692
5693     my %systemd_properties = (
5694         Slice => 'qemu.slice',
5695         KillMode => 'process',
5696         SendSIGKILL => 0,
5697         TimeoutStopUSec => ULONG_MAX, # infinity
5698     );
5699
5700     if (PVE::CGroup::cgroup_mode() == 2) {
5701         $systemd_properties{CPUWeight} = $cpuunits;
5702     } else {
5703         $systemd_properties{CPUShares} = $cpuunits;
5704     }
5705
5706     if (my $cpulimit = $conf->{cpulimit}) {
5707         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5708     }
5709     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5710
5711     my $run_qemu = sub {
5712         PVE::Tools::run_fork sub {
5713             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5714
5715             my $tpmpid;
5716             if (my $tpm = $conf->{tpmstate0}) {
5717                 # start the TPM emulator so QEMU can connect on start
5718                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5719             }
5720
5721             my $exitcode = run_command($cmd, %run_params);
5722             if ($exitcode) {
5723                 if ($tpmpid) {
5724                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5725                     kill 'TERM', $tpmpid;
5726                 }
5727                 die "QEMU exited with code $exitcode\n";
5728             }
5729         };
5730     };
5731
5732     if ($conf->{hugepages}) {
5733
5734         my $code = sub {
5735             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5736             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5737
5738             PVE::QemuServer::Memory::hugepages_mount();
5739             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5740
5741             eval { $run_qemu->() };
5742             if (my $err = $@) {
5743                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5744                     if !$conf->{keephugepages};
5745                 die $err;
5746             }
5747
5748             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5749                 if !$conf->{keephugepages};
5750         };
5751         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5752
5753     } else {
5754         eval { $run_qemu->() };
5755     }
5756
5757     if (my $err = $@) {
5758         # deactivate volumes if start fails
5759         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5760         warn $@ if $@;
5761         eval { cleanup_pci_devices($vmid, $conf) };
5762         warn $@ if $@;
5763
5764         die "start failed: $err";
5765     }
5766
5767     # re-reserve all PCI IDs now that we can know the actual VM PID
5768     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5769     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5770     warn $@ if $@;
5771
5772     print "migration listens on $migrate_uri\n" if $migrate_uri;
5773     $res->{migrate_uri} = $migrate_uri;
5774
5775     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5776         eval { mon_cmd($vmid, "cont"); };
5777         warn $@ if $@;
5778     }
5779
5780     #start nbd server for storage migration
5781     if (my $nbd = $migrate_opts->{nbd}) {
5782         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5783
5784         my $migrate_storage_uri;
5785         # nbd_protocol_version > 0 for unix socket support
5786         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5787             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5788             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5789             $migrate_storage_uri = "nbd:unix:$socket_path";
5790         } else {
5791             my $nodename = nodename();
5792             my $localip = $get_migration_ip->($nodename);
5793             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5794             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5795
5796             mon_cmd($vmid, "nbd-server-start", addr => {
5797                 type => 'inet',
5798                 data => {
5799                     host => "${localip}",
5800                     port => "${storage_migrate_port}",
5801                 },
5802             });
5803             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5804             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5805         }
5806
5807         $res->{migrate_storage_uri} = $migrate_storage_uri;
5808
5809         foreach my $opt (sort keys %$nbd) {
5810             my $drivestr = $nbd->{$opt}->{drivestr};
5811             my $volid = $nbd->{$opt}->{volid};
5812             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5813             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5814             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5815             print "re-using replicated volume: $opt - $volid\n"
5816                 if $nbd->{$opt}->{replicated};
5817
5818             $res->{drives}->{$opt} = $nbd->{$opt};
5819             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5820         }
5821     }
5822
5823     if ($migratedfrom) {
5824         eval {
5825             set_migration_caps($vmid);
5826         };
5827         warn $@ if $@;
5828
5829         if ($spice_port) {
5830             print "spice listens on port $spice_port\n";
5831             $res->{spice_port} = $spice_port;
5832             if ($migrate_opts->{spice_ticket}) {
5833                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5834                     $migrate_opts->{spice_ticket});
5835                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5836             }
5837         }
5838
5839     } else {
5840         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5841             if !$statefile && $conf->{balloon};
5842
5843         foreach my $opt (keys %$conf) {
5844             next if $opt !~  m/^net\d+$/;
5845             my $nicconf = parse_net($conf->{$opt});
5846             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5847         }
5848     }
5849
5850     mon_cmd($vmid, 'qom-set',
5851                 path => "machine/peripheral/balloon0",
5852                 property => "guest-stats-polling-interval",
5853                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5854
5855     if ($resume) {
5856         print "Resumed VM, removing state\n";
5857         if (my $vmstate = $conf->{vmstate}) {
5858             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5859             PVE::Storage::vdisk_free($storecfg, $vmstate);
5860         }
5861         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5862         PVE::QemuConfig->write_config($vmid, $conf);
5863     }
5864
5865     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5866
5867     return $res;
5868 }
5869
5870 sub vm_commandline {
5871     my ($storecfg, $vmid, $snapname) = @_;
5872
5873     my $conf = PVE::QemuConfig->load_config($vmid);
5874
5875     my ($forcemachine, $forcecpu);
5876     if ($snapname) {
5877         my $snapshot = $conf->{snapshots}->{$snapname};
5878         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5879
5880         # check for machine or CPU overrides in snapshot
5881         $forcemachine = $snapshot->{runningmachine};
5882         $forcecpu = $snapshot->{runningcpu};
5883
5884         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5885
5886         $conf = $snapshot;
5887     }
5888
5889     my $defaults = load_defaults();
5890
5891     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5892
5893     return PVE::Tools::cmd2string($cmd);
5894 }
5895
5896 sub vm_reset {
5897     my ($vmid, $skiplock) = @_;
5898
5899     PVE::QemuConfig->lock_config($vmid, sub {
5900
5901         my $conf = PVE::QemuConfig->load_config($vmid);
5902
5903         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5904
5905         mon_cmd($vmid, "system_reset");
5906     });
5907 }
5908
5909 sub get_vm_volumes {
5910     my ($conf) = @_;
5911
5912     my $vollist = [];
5913     foreach_volid($conf, sub {
5914         my ($volid, $attr) = @_;
5915
5916         return if $volid =~ m|^/|;
5917
5918         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5919         return if !$sid;
5920
5921         push @$vollist, $volid;
5922     });
5923
5924     return $vollist;
5925 }
5926
5927 sub cleanup_pci_devices {
5928     my ($vmid, $conf) = @_;
5929
5930     foreach my $key (keys %$conf) {
5931         next if $key !~ m/^hostpci(\d+)$/;
5932         my $hostpciindex = $1;
5933         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5934         my $d = parse_hostpci($conf->{$key});
5935         if ($d->{mdev}) {
5936             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
5937             # don't want to break ABI just for this two liner
5938             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
5939             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
5940         }
5941     }
5942     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
5943 }
5944
5945 sub vm_stop_cleanup {
5946     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5947
5948     eval {
5949
5950         if (!$keepActive) {
5951             my $vollist = get_vm_volumes($conf);
5952             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5953
5954             if (my $tpmdrive = $conf->{tpmstate0}) {
5955                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5956                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5957                 if ($storeid) {
5958                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5959                 }
5960             }
5961         }
5962
5963         foreach my $ext (qw(mon qmp pid vnc qga)) {
5964             unlink "/var/run/qemu-server/${vmid}.$ext";
5965         }
5966
5967         if ($conf->{ivshmem}) {
5968             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5969             # just delete it for now, VMs which have this already open do not
5970             # are affected, but new VMs will get a separated one. If this
5971             # becomes an issue we either add some sort of ref-counting or just
5972             # add a "don't delete on stop" flag to the ivshmem format.
5973             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5974         }
5975
5976         cleanup_pci_devices($vmid, $conf);
5977
5978         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5979     };
5980     warn $@ if $@; # avoid errors - just warn
5981 }
5982
5983 # call only in locked context
5984 sub _do_vm_stop {
5985     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5986
5987     my $pid = check_running($vmid, $nocheck);
5988     return if !$pid;
5989
5990     my $conf;
5991     if (!$nocheck) {
5992         $conf = PVE::QemuConfig->load_config($vmid);
5993         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5994         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5995             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5996             $timeout = $opts->{down} if $opts->{down};
5997         }
5998         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5999     }
6000
6001     eval {
6002         if ($shutdown) {
6003             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6004                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6005             } else {
6006                 mon_cmd($vmid, "system_powerdown");
6007             }
6008         } else {
6009             mon_cmd($vmid, "quit");
6010         }
6011     };
6012     my $err = $@;
6013
6014     if (!$err) {
6015         $timeout = 60 if !defined($timeout);
6016
6017         my $count = 0;
6018         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6019             $count++;
6020             sleep 1;
6021         }
6022
6023         if ($count >= $timeout) {
6024             if ($force) {
6025                 warn "VM still running - terminating now with SIGTERM\n";
6026                 kill 15, $pid;
6027             } else {
6028                 die "VM quit/powerdown failed - got timeout\n";
6029             }
6030         } else {
6031             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6032             return;
6033         }
6034     } else {
6035        if (!check_running($vmid, $nocheck)) {
6036             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6037             return;
6038        }
6039        if ($force) {
6040             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6041             kill 15, $pid;
6042         } else {
6043             die "VM quit/powerdown failed\n";
6044         }
6045     }
6046
6047     # wait again
6048     $timeout = 10;
6049
6050     my $count = 0;
6051     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6052         $count++;
6053         sleep 1;
6054     }
6055
6056     if ($count >= $timeout) {
6057         warn "VM still running - terminating now with SIGKILL\n";
6058         kill 9, $pid;
6059         sleep 1;
6060     }
6061
6062     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6063 }
6064
6065 # Note: use $nocheck to skip tests if VM configuration file exists.
6066 # We need that when migration VMs to other nodes (files already moved)
6067 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6068 sub vm_stop {
6069     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6070
6071     $force = 1 if !defined($force) && !$shutdown;
6072
6073     if ($migratedfrom){
6074         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6075         kill 15, $pid if $pid;
6076         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6077         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6078         return;
6079     }
6080
6081     PVE::QemuConfig->lock_config($vmid, sub {
6082         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6083    });
6084 }
6085
6086 sub vm_reboot {
6087     my ($vmid, $timeout) = @_;
6088
6089     PVE::QemuConfig->lock_config($vmid, sub {
6090         eval {
6091
6092             # only reboot if running, as qmeventd starts it again on a stop event
6093             return if !check_running($vmid);
6094
6095             create_reboot_request($vmid);
6096
6097             my $storecfg = PVE::Storage::config();
6098             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6099
6100         };
6101         if (my $err = $@) {
6102             # avoid that the next normal shutdown will be confused for a reboot
6103             clear_reboot_request($vmid);
6104             die $err;
6105         }
6106    });
6107 }
6108
6109 # note: if using the statestorage parameter, the caller has to check privileges
6110 sub vm_suspend {
6111     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6112
6113     my $conf;
6114     my $path;
6115     my $storecfg;
6116     my $vmstate;
6117
6118     PVE::QemuConfig->lock_config($vmid, sub {
6119
6120         $conf = PVE::QemuConfig->load_config($vmid);
6121
6122         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6123         PVE::QemuConfig->check_lock($conf)
6124             if !($skiplock || $is_backing_up);
6125
6126         die "cannot suspend to disk during backup\n"
6127             if $is_backing_up && $includestate;
6128
6129         if ($includestate) {
6130             $conf->{lock} = 'suspending';
6131             my $date = strftime("%Y-%m-%d", localtime(time()));
6132             $storecfg = PVE::Storage::config();
6133             if (!$statestorage) {
6134                 $statestorage = find_vmstate_storage($conf, $storecfg);
6135                 # check permissions for the storage
6136                 my $rpcenv = PVE::RPCEnvironment::get();
6137                 if ($rpcenv->{type} ne 'cli') {
6138                     my $authuser = $rpcenv->get_user();
6139                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6140                 }
6141             }
6142
6143
6144             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6145                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6146             $path = PVE::Storage::path($storecfg, $vmstate);
6147             PVE::QemuConfig->write_config($vmid, $conf);
6148         } else {
6149             mon_cmd($vmid, "stop");
6150         }
6151     });
6152
6153     if ($includestate) {
6154         # save vm state
6155         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6156
6157         eval {
6158             set_migration_caps($vmid, 1);
6159             mon_cmd($vmid, "savevm-start", statefile => $path);
6160             for(;;) {
6161                 my $state = mon_cmd($vmid, "query-savevm");
6162                 if (!$state->{status}) {
6163                     die "savevm not active\n";
6164                 } elsif ($state->{status} eq 'active') {
6165                     sleep(1);
6166                     next;
6167                 } elsif ($state->{status} eq 'completed') {
6168                     print "State saved, quitting\n";
6169                     last;
6170                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6171                     die "query-savevm failed with error '$state->{error}'\n"
6172                 } else {
6173                     die "query-savevm returned status '$state->{status}'\n";
6174                 }
6175             }
6176         };
6177         my $err = $@;
6178
6179         PVE::QemuConfig->lock_config($vmid, sub {
6180             $conf = PVE::QemuConfig->load_config($vmid);
6181             if ($err) {
6182                 # cleanup, but leave suspending lock, to indicate something went wrong
6183                 eval {
6184                     mon_cmd($vmid, "savevm-end");
6185                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6186                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6187                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6188                     PVE::QemuConfig->write_config($vmid, $conf);
6189                 };
6190                 warn $@ if $@;
6191                 die $err;
6192             }
6193
6194             die "lock changed unexpectedly\n"
6195                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6196
6197             mon_cmd($vmid, "quit");
6198             $conf->{lock} = 'suspended';
6199             PVE::QemuConfig->write_config($vmid, $conf);
6200         });
6201     }
6202 }
6203
6204 sub vm_resume {
6205     my ($vmid, $skiplock, $nocheck) = @_;
6206
6207     PVE::QemuConfig->lock_config($vmid, sub {
6208         my $res = mon_cmd($vmid, 'query-status');
6209         my $resume_cmd = 'cont';
6210         my $reset = 0;
6211
6212         if ($res->{status}) {
6213             return if $res->{status} eq 'running'; # job done, go home
6214             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6215             $reset = 1 if $res->{status} eq 'shutdown';
6216         }
6217
6218         if (!$nocheck) {
6219
6220             my $conf = PVE::QemuConfig->load_config($vmid);
6221
6222             PVE::QemuConfig->check_lock($conf)
6223                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6224         }
6225
6226         if ($reset) {
6227             # required if a VM shuts down during a backup and we get a resume
6228             # request before the backup finishes for example
6229             mon_cmd($vmid, "system_reset");
6230         }
6231         mon_cmd($vmid, $resume_cmd);
6232     });
6233 }
6234
6235 sub vm_sendkey {
6236     my ($vmid, $skiplock, $key) = @_;
6237
6238     PVE::QemuConfig->lock_config($vmid, sub {
6239
6240         my $conf = PVE::QemuConfig->load_config($vmid);
6241
6242         # there is no qmp command, so we use the human monitor command
6243         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6244         die $res if $res ne '';
6245     });
6246 }
6247
6248 # vzdump restore implementaion
6249
6250 sub tar_archive_read_firstfile {
6251     my $archive = shift;
6252
6253     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6254
6255     # try to detect archive type first
6256     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6257         die "unable to open file '$archive'\n";
6258     my $firstfile = <$fh>;
6259     kill 15, $pid;
6260     close $fh;
6261
6262     die "ERROR: archive contaions no data\n" if !$firstfile;
6263     chomp $firstfile;
6264
6265     return $firstfile;
6266 }
6267
6268 sub tar_restore_cleanup {
6269     my ($storecfg, $statfile) = @_;
6270
6271     print STDERR "starting cleanup\n";
6272
6273     if (my $fd = IO::File->new($statfile, "r")) {
6274         while (defined(my $line = <$fd>)) {
6275             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6276                 my $volid = $2;
6277                 eval {
6278                     if ($volid =~ m|^/|) {
6279                         unlink $volid || die 'unlink failed\n';
6280                     } else {
6281                         PVE::Storage::vdisk_free($storecfg, $volid);
6282                     }
6283                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6284                 };
6285                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6286             } else {
6287                 print STDERR "unable to parse line in statfile - $line";
6288             }
6289         }
6290         $fd->close();
6291     }
6292 }
6293
6294 sub restore_file_archive {
6295     my ($archive, $vmid, $user, $opts) = @_;
6296
6297     return restore_vma_archive($archive, $vmid, $user, $opts)
6298         if $archive eq '-';
6299
6300     my $info = PVE::Storage::archive_info($archive);
6301     my $format = $opts->{format} // $info->{format};
6302     my $comp = $info->{compression};
6303
6304     # try to detect archive format
6305     if ($format eq 'tar') {
6306         return restore_tar_archive($archive, $vmid, $user, $opts);
6307     } else {
6308         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6309     }
6310 }
6311
6312 # hepler to remove disks that will not be used after restore
6313 my $restore_cleanup_oldconf = sub {
6314     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6315
6316     my $kept_disks = {};
6317
6318     PVE::QemuConfig->foreach_volume($oldconf, sub {
6319         my ($ds, $drive) = @_;
6320
6321         return if drive_is_cdrom($drive, 1);
6322
6323         my $volid = $drive->{file};
6324         return if !$volid || $volid =~ m|^/|;
6325
6326         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6327         return if !$path || !$owner || ($owner != $vmid);
6328
6329         # Note: only delete disk we want to restore
6330         # other volumes will become unused
6331         if ($virtdev_hash->{$ds}) {
6332             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6333             if (my $err = $@) {
6334                 warn $err;
6335             }
6336         } else {
6337             $kept_disks->{$volid} = 1;
6338         }
6339     });
6340
6341     # after the restore we have no snapshots anymore
6342     for my $snapname (keys $oldconf->{snapshots}->%*) {
6343         my $snap = $oldconf->{snapshots}->{$snapname};
6344         if ($snap->{vmstate}) {
6345             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6346             if (my $err = $@) {
6347                 warn $err;
6348             }
6349         }
6350
6351         for my $volid (keys $kept_disks->%*) {
6352             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6353             warn $@ if $@;
6354         }
6355     }
6356 };
6357
6358 # Helper to parse vzdump backup device hints
6359 #
6360 # $rpcenv: Environment, used to ckeck storage permissions
6361 # $user: User ID, to check storage permissions
6362 # $storecfg: Storage configuration
6363 # $fh: the file handle for reading the configuration
6364 # $devinfo: should contain device sizes for all backu-up'ed devices
6365 # $options: backup options (pool, default storage)
6366 #
6367 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6368 my $parse_backup_hints = sub {
6369     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6370
6371     my $check_storage = sub { # assert if an image can be allocate
6372         my ($storeid, $scfg) = @_;
6373         die "Content type 'images' is not available on storage '$storeid'\n"
6374             if !$scfg->{content}->{images};
6375         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6376             if $user ne 'root@pam';
6377     };
6378
6379     my $virtdev_hash = {};
6380     while (defined(my $line = <$fh>)) {
6381         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6382             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6383             die "archive does not contain data for drive '$virtdev'\n"
6384                 if !$devinfo->{$devname};
6385
6386             if (defined($options->{storage})) {
6387                 $storeid = $options->{storage} || 'local';
6388             } elsif (!$storeid) {
6389                 $storeid = 'local';
6390             }
6391             $format = 'raw' if !$format;
6392             $devinfo->{$devname}->{devname} = $devname;
6393             $devinfo->{$devname}->{virtdev} = $virtdev;
6394             $devinfo->{$devname}->{format} = $format;
6395             $devinfo->{$devname}->{storeid} = $storeid;
6396
6397             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6398             $check_storage->($storeid, $scfg); # permission and content type check
6399
6400             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6401         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6402             my $virtdev = $1;
6403             my $drive = parse_drive($virtdev, $2);
6404
6405             if (drive_is_cloudinit($drive)) {
6406                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6407                 $storeid = $options->{storage} if defined ($options->{storage});
6408                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6409                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6410
6411                 $check_storage->($storeid, $scfg); # permission and content type check
6412
6413                 $virtdev_hash->{$virtdev} = {
6414                     format => $format,
6415                     storeid => $storeid,
6416                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6417                     is_cloudinit => 1,
6418                 };
6419             }
6420         }
6421     }
6422
6423     return $virtdev_hash;
6424 };
6425
6426 # Helper to allocate and activate all volumes required for a restore
6427 #
6428 # $storecfg: Storage configuration
6429 # $virtdev_hash: as returned by parse_backup_hints()
6430 #
6431 # Returns: { $virtdev => $volid }
6432 my $restore_allocate_devices = sub {
6433     my ($storecfg, $virtdev_hash, $vmid) = @_;
6434
6435     my $map = {};
6436     foreach my $virtdev (sort keys %$virtdev_hash) {
6437         my $d = $virtdev_hash->{$virtdev};
6438         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6439         my $storeid = $d->{storeid};
6440         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6441
6442         # test if requested format is supported
6443         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6444         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6445         $d->{format} = $defFormat if !$supported;
6446
6447         my $name;
6448         if ($d->{is_cloudinit}) {
6449             $name = "vm-$vmid-cloudinit";
6450             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6451             if ($scfg->{path}) {
6452                 $name .= ".$d->{format}";
6453             }
6454         }
6455
6456         my $volid = PVE::Storage::vdisk_alloc(
6457             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6458
6459         print STDERR "new volume ID is '$volid'\n";
6460         $d->{volid} = $volid;
6461
6462         PVE::Storage::activate_volumes($storecfg, [$volid]);
6463
6464         $map->{$virtdev} = $volid;
6465     }
6466
6467     return $map;
6468 };
6469
6470 sub restore_update_config_line {
6471     my ($cookie, $map, $line, $unique) = @_;
6472
6473     return '' if $line =~ m/^\#qmdump\#/;
6474     return '' if $line =~ m/^\#vzdump\#/;
6475     return '' if $line =~ m/^lock:/;
6476     return '' if $line =~ m/^unused\d+:/;
6477     return '' if $line =~ m/^parent:/;
6478
6479     my $res = '';
6480
6481     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6482     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6483         # try to convert old 1.X settings
6484         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6485         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6486             my ($model, $macaddr) = split(/\=/, $devconfig);
6487             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6488             my $net = {
6489                 model => $model,
6490                 bridge => "vmbr$ind",
6491                 macaddr => $macaddr,
6492             };
6493             my $netstr = print_net($net);
6494
6495             $res .= "net$cookie->{netcount}: $netstr\n";
6496             $cookie->{netcount}++;
6497         }
6498     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6499         my ($id, $netstr) = ($1, $2);
6500         my $net = parse_net($netstr);
6501         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6502         $netstr = print_net($net);
6503         $res .= "$id: $netstr\n";
6504     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6505         my $virtdev = $1;
6506         my $value = $3;
6507         my $di = parse_drive($virtdev, $value);
6508         if (defined($di->{backup}) && !$di->{backup}) {
6509             $res .= "#$line";
6510         } elsif ($map->{$virtdev}) {
6511             delete $di->{format}; # format can change on restore
6512             $di->{file} = $map->{$virtdev};
6513             $value = print_drive($di);
6514             $res .= "$virtdev: $value\n";
6515         } else {
6516             $res .= $line;
6517         }
6518     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6519         my $vmgenid = $1;
6520         if ($vmgenid ne '0') {
6521             # always generate a new vmgenid if there was a valid one setup
6522             $vmgenid = generate_uuid();
6523         }
6524         $res .= "vmgenid: $vmgenid\n";
6525     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6526         my ($uuid, $uuid_str);
6527         UUID::generate($uuid);
6528         UUID::unparse($uuid, $uuid_str);
6529         my $smbios1 = parse_smbios1($2);
6530         $smbios1->{uuid} = $uuid_str;
6531         $res .= $1.print_smbios1($smbios1)."\n";
6532     } else {
6533         $res .= $line;
6534     }
6535
6536     return $res;
6537 }
6538
6539 my $restore_deactivate_volumes = sub {
6540     my ($storecfg, $virtdev_hash) = @_;
6541
6542     my $vollist = [];
6543     for my $dev (values $virtdev_hash->%*) {
6544         push $vollist->@*, $dev->{volid} if $dev->{volid};
6545     }
6546
6547     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6548     print STDERR $@ if $@;
6549 };
6550
6551 my $restore_destroy_volumes = sub {
6552     my ($storecfg, $virtdev_hash) = @_;
6553
6554     for my $dev (values $virtdev_hash->%*) {
6555         my $volid = $dev->{volid} or next;
6556         eval {
6557             PVE::Storage::vdisk_free($storecfg, $volid);
6558             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6559         };
6560         print STDERR "unable to cleanup '$volid' - $@" if $@;
6561     }
6562 };
6563
6564 my $restore_merge_config = sub {
6565     my ($filename, $backup_conf_raw, $override_conf) = @_;
6566
6567     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6568     for my $key (keys $override_conf->%*) {
6569         $backup_conf->{$key} = $override_conf->{$key};
6570     }
6571
6572     return $backup_conf;
6573 };
6574
6575 sub scan_volids {
6576     my ($cfg, $vmid) = @_;
6577
6578     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6579
6580     my $volid_hash = {};
6581     foreach my $storeid (keys %$info) {
6582         foreach my $item (@{$info->{$storeid}}) {
6583             next if !($item->{volid} && $item->{size});
6584             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6585             $volid_hash->{$item->{volid}} = $item;
6586         }
6587     }
6588
6589     return $volid_hash;
6590 }
6591
6592 sub update_disk_config {
6593     my ($vmid, $conf, $volid_hash) = @_;
6594
6595     my $changes;
6596     my $prefix = "VM $vmid";
6597
6598     # used and unused disks
6599     my $referenced = {};
6600
6601     # Note: it is allowed to define multiple storages with same path (alias), so
6602     # we need to check both 'volid' and real 'path' (two different volid can point
6603     # to the same path).
6604
6605     my $referencedpath = {};
6606
6607     # update size info
6608     PVE::QemuConfig->foreach_volume($conf, sub {
6609         my ($opt, $drive) = @_;
6610
6611         my $volid = $drive->{file};
6612         return if !$volid;
6613         my $volume = $volid_hash->{$volid};
6614
6615         # mark volid as "in-use" for next step
6616         $referenced->{$volid} = 1;
6617         if ($volume && (my $path = $volume->{path})) {
6618             $referencedpath->{$path} = 1;
6619         }
6620
6621         return if drive_is_cdrom($drive);
6622         return if !$volume;
6623
6624         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6625         if (defined($updated)) {
6626             $changes = 1;
6627             $conf->{$opt} = print_drive($updated);
6628             print "$prefix ($opt): $msg\n";
6629         }
6630     });
6631
6632     # remove 'unusedX' entry if volume is used
6633     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6634         my ($opt, $drive) = @_;
6635
6636         my $volid = $drive->{file};
6637         return if !$volid;
6638
6639         my $path;
6640         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6641         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6642             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6643             $changes = 1;
6644             delete $conf->{$opt};
6645         }
6646
6647         $referenced->{$volid} = 1;
6648         $referencedpath->{$path} = 1 if $path;
6649     });
6650
6651     foreach my $volid (sort keys %$volid_hash) {
6652         next if $volid =~ m/vm-$vmid-state-/;
6653         next if $referenced->{$volid};
6654         my $path = $volid_hash->{$volid}->{path};
6655         next if !$path; # just to be sure
6656         next if $referencedpath->{$path};
6657         $changes = 1;
6658         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6659         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6660         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6661     }
6662
6663     return $changes;
6664 }
6665
6666 sub rescan {
6667     my ($vmid, $nolock, $dryrun) = @_;
6668
6669     my $cfg = PVE::Storage::config();
6670
6671     print "rescan volumes...\n";
6672     my $volid_hash = scan_volids($cfg, $vmid);
6673
6674     my $updatefn =  sub {
6675         my ($vmid) = @_;
6676
6677         my $conf = PVE::QemuConfig->load_config($vmid);
6678
6679         PVE::QemuConfig->check_lock($conf);
6680
6681         my $vm_volids = {};
6682         foreach my $volid (keys %$volid_hash) {
6683             my $info = $volid_hash->{$volid};
6684             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6685         }
6686
6687         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6688
6689         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6690     };
6691
6692     if (defined($vmid)) {
6693         if ($nolock) {
6694             &$updatefn($vmid);
6695         } else {
6696             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6697         }
6698     } else {
6699         my $vmlist = config_list();
6700         foreach my $vmid (keys %$vmlist) {
6701             if ($nolock) {
6702                 &$updatefn($vmid);
6703             } else {
6704                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6705             }
6706         }
6707     }
6708 }
6709
6710 sub restore_proxmox_backup_archive {
6711     my ($archive, $vmid, $user, $options) = @_;
6712
6713     my $storecfg = PVE::Storage::config();
6714
6715     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6716     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6717
6718     my $fingerprint = $scfg->{fingerprint};
6719     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6720
6721     my $repo = PVE::PBSClient::get_repository($scfg);
6722     my $namespace = $scfg->{namespace};
6723
6724     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6725     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6726     local $ENV{PBS_PASSWORD} = $password;
6727     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6728
6729     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6730         PVE::Storage::parse_volname($storecfg, $archive);
6731
6732     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6733
6734     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6735
6736     my $tmpdir = "/var/tmp/vzdumptmp$$";
6737     rmtree $tmpdir;
6738     mkpath $tmpdir;
6739
6740     my $conffile = PVE::QemuConfig->config_file($vmid);
6741      # disable interrupts (always do cleanups)
6742     local $SIG{INT} =
6743         local $SIG{TERM} =
6744         local $SIG{QUIT} =
6745         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6746
6747     # Note: $oldconf is undef if VM does not exists
6748     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6749     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6750     my $new_conf_raw = '';
6751
6752     my $rpcenv = PVE::RPCEnvironment::get();
6753     my $devinfo = {}; # info about drives included in backup
6754     my $virtdev_hash = {}; # info about allocated drives
6755
6756     eval {
6757         # enable interrupts
6758         local $SIG{INT} =
6759             local $SIG{TERM} =
6760             local $SIG{QUIT} =
6761             local $SIG{HUP} =
6762             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6763
6764         my $cfgfn = "$tmpdir/qemu-server.conf";
6765         my $firewall_config_fn = "$tmpdir/fw.conf";
6766         my $index_fn = "$tmpdir/index.json";
6767
6768         my $cmd = "restore";
6769
6770         my $param = [$pbs_backup_name, "index.json", $index_fn];
6771         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6772         my $index = PVE::Tools::file_get_contents($index_fn);
6773         $index = decode_json($index);
6774
6775         foreach my $info (@{$index->{files}}) {
6776             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6777                 my $devname = $1;
6778                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6779                     $devinfo->{$devname}->{size} = $1;
6780                 } else {
6781                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6782                 }
6783             }
6784         }
6785
6786         my $is_qemu_server_backup = scalar(
6787             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6788         );
6789         if (!$is_qemu_server_backup) {
6790             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6791         }
6792         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6793
6794         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6795         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6796
6797         if ($has_firewall_config) {
6798             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6799             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6800
6801             my $pve_firewall_dir = '/etc/pve/firewall';
6802             mkdir $pve_firewall_dir; # make sure the dir exists
6803             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6804         }
6805
6806         my $fh = IO::File->new($cfgfn, "r") ||
6807             die "unable to read qemu-server.conf - $!\n";
6808
6809         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6810
6811         # fixme: rate limit?
6812
6813         # create empty/temp config
6814         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6815
6816         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6817
6818         # allocate volumes
6819         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6820
6821         foreach my $virtdev (sort keys %$virtdev_hash) {
6822             my $d = $virtdev_hash->{$virtdev};
6823             next if $d->{is_cloudinit}; # no need to restore cloudinit
6824
6825             # this fails if storage is unavailable
6826             my $volid = $d->{volid};
6827             my $path = PVE::Storage::path($storecfg, $volid);
6828
6829             # for live-restore we only want to preload the efidisk and TPM state
6830             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6831
6832             my @ns_arg;
6833             if (defined(my $ns = $scfg->{namespace})) {
6834                 @ns_arg = ('--ns', $ns);
6835             }
6836
6837             my $pbs_restore_cmd = [
6838                 '/usr/bin/pbs-restore',
6839                 '--repository', $repo,
6840                 @ns_arg,
6841                 $pbs_backup_name,
6842                 "$d->{devname}.img.fidx",
6843                 $path,
6844                 '--verbose',
6845                 ];
6846
6847             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6848             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6849
6850             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6851                 push @$pbs_restore_cmd, '--skip-zero';
6852             }
6853
6854             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6855             print "restore proxmox backup image: $dbg_cmdstring\n";
6856             run_command($pbs_restore_cmd);
6857         }
6858
6859         $fh->seek(0, 0) || die "seek failed - $!\n";
6860
6861         my $cookie = { netcount => 0 };
6862         while (defined(my $line = <$fh>)) {
6863             $new_conf_raw .= restore_update_config_line(
6864                 $cookie,
6865                 $map,
6866                 $line,
6867                 $options->{unique},
6868             );
6869         }
6870
6871         $fh->close();
6872     };
6873     my $err = $@;
6874
6875     if ($err || !$options->{live}) {
6876         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6877     }
6878
6879     rmtree $tmpdir;
6880
6881     if ($err) {
6882         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6883         die $err;
6884     }
6885
6886     if ($options->{live}) {
6887         # keep lock during live-restore
6888         $new_conf_raw .= "\nlock: create";
6889     }
6890
6891     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6892     PVE::QemuConfig->write_config($vmid, $new_conf);
6893
6894     eval { rescan($vmid, 1); };
6895     warn $@ if $@;
6896
6897     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6898
6899     if ($options->{live}) {
6900         # enable interrupts
6901         local $SIG{INT} =
6902             local $SIG{TERM} =
6903             local $SIG{QUIT} =
6904             local $SIG{HUP} =
6905             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6906
6907         my $conf = PVE::QemuConfig->load_config($vmid);
6908         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6909
6910         # these special drives are already restored before start
6911         delete $devinfo->{'drive-efidisk0'};
6912         delete $devinfo->{'drive-tpmstate0-backup'};
6913
6914         my $pbs_opts = {
6915             repo => $repo,
6916             keyfile => $keyfile,
6917             snapshot => $pbs_backup_name,
6918             namespace => $namespace,
6919         };
6920         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6921
6922         PVE::QemuConfig->remove_lock($vmid, "create");
6923     }
6924 }
6925
6926 sub pbs_live_restore {
6927     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6928
6929     print "starting VM for live-restore\n";
6930     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6931
6932     my $pbs_backing = {};
6933     for my $ds (keys %$restored_disks) {
6934         $ds =~ m/^drive-(.*)$/;
6935         my $confname = $1;
6936         $pbs_backing->{$confname} = {
6937             repository => $opts->{repo},
6938             snapshot => $opts->{snapshot},
6939             archive => "$ds.img.fidx",
6940         };
6941         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6942         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6943
6944         my $drive = parse_drive($confname, $conf->{$confname});
6945         print "restoring '$ds' to '$drive->{file}'\n";
6946     }
6947
6948     my $drives_streamed = 0;
6949     eval {
6950         # make sure HA doesn't interrupt our restore by stopping the VM
6951         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6952             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6953         }
6954
6955         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6956         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6957         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6958
6959         my $qmeventd_fd = register_qmeventd_handle($vmid);
6960
6961         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6962         # this will effectively collapse the backing image chain consisting of
6963         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6964         # removes itself once all backing images vanish with 'auto-remove=on')
6965         my $jobs = {};
6966         for my $ds (sort keys %$restored_disks) {
6967             my $job_id = "restore-$ds";
6968             mon_cmd($vmid, 'block-stream',
6969                 'job-id' => $job_id,
6970                 device => "$ds",
6971             );
6972             $jobs->{$job_id} = {};
6973         }
6974
6975         mon_cmd($vmid, 'cont');
6976         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6977
6978         print "restore-drive jobs finished successfully, removing all tracking block devices"
6979             ." to disconnect from Proxmox Backup Server\n";
6980
6981         for my $ds (sort keys %$restored_disks) {
6982             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6983         }
6984
6985         close($qmeventd_fd);
6986     };
6987
6988     my $err = $@;
6989
6990     if ($err) {
6991         warn "An error occurred during live-restore: $err\n";
6992         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6993         die "live-restore failed\n";
6994     }
6995 }
6996
6997 sub restore_vma_archive {
6998     my ($archive, $vmid, $user, $opts, $comp) = @_;
6999
7000     my $readfrom = $archive;
7001
7002     my $cfg = PVE::Storage::config();
7003     my $commands = [];
7004     my $bwlimit = $opts->{bwlimit};
7005
7006     my $dbg_cmdstring = '';
7007     my $add_pipe = sub {
7008         my ($cmd) = @_;
7009         push @$commands, $cmd;
7010         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7011         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7012         $readfrom = '-';
7013     };
7014
7015     my $input = undef;
7016     if ($archive eq '-') {
7017         $input = '<&STDIN';
7018     } else {
7019         # If we use a backup from a PVE defined storage we also consider that
7020         # storage's rate limit:
7021         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7022         if (defined($volid)) {
7023             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7024             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7025             if ($readlimit) {
7026                 print STDERR "applying read rate limit: $readlimit\n";
7027                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7028                 $add_pipe->($cstream);
7029             }
7030         }
7031     }
7032
7033     if ($comp) {
7034         my $info = PVE::Storage::decompressor_info('vma', $comp);
7035         my $cmd = $info->{decompressor};
7036         push @$cmd, $readfrom;
7037         $add_pipe->($cmd);
7038     }
7039
7040     my $tmpdir = "/var/tmp/vzdumptmp$$";
7041     rmtree $tmpdir;
7042
7043     # disable interrupts (always do cleanups)
7044     local $SIG{INT} =
7045         local $SIG{TERM} =
7046         local $SIG{QUIT} =
7047         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7048
7049     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7050     POSIX::mkfifo($mapfifo, 0600);
7051     my $fifofh;
7052     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7053
7054     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7055
7056     my $oldtimeout;
7057     my $timeout = 5;
7058
7059     my $devinfo = {}; # info about drives included in backup
7060     my $virtdev_hash = {}; # info about allocated drives
7061
7062     my $rpcenv = PVE::RPCEnvironment::get();
7063
7064     my $conffile = PVE::QemuConfig->config_file($vmid);
7065
7066     # Note: $oldconf is undef if VM does not exist
7067     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7068     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7069     my $new_conf_raw = '';
7070
7071     my %storage_limits;
7072
7073     my $print_devmap = sub {
7074         my $cfgfn = "$tmpdir/qemu-server.conf";
7075
7076         # we can read the config - that is already extracted
7077         my $fh = IO::File->new($cfgfn, "r") ||
7078             die "unable to read qemu-server.conf - $!\n";
7079
7080         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7081         if (-f $fwcfgfn) {
7082             my $pve_firewall_dir = '/etc/pve/firewall';
7083             mkdir $pve_firewall_dir; # make sure the dir exists
7084             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7085         }
7086
7087         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7088
7089         foreach my $info (values %{$virtdev_hash}) {
7090             my $storeid = $info->{storeid};
7091             next if defined($storage_limits{$storeid});
7092
7093             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7094             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7095             $storage_limits{$storeid} = $limit * 1024;
7096         }
7097
7098         foreach my $devname (keys %$devinfo) {
7099             die "found no device mapping information for device '$devname'\n"
7100                 if !$devinfo->{$devname}->{virtdev};
7101         }
7102
7103         # create empty/temp config
7104         if ($oldconf) {
7105             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7106             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7107         }
7108
7109         # allocate volumes
7110         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7111
7112         # print restore information to $fifofh
7113         foreach my $virtdev (sort keys %$virtdev_hash) {
7114             my $d = $virtdev_hash->{$virtdev};
7115             next if $d->{is_cloudinit}; # no need to restore cloudinit
7116
7117             my $storeid = $d->{storeid};
7118             my $volid = $d->{volid};
7119
7120             my $map_opts = '';
7121             if (my $limit = $storage_limits{$storeid}) {
7122                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7123             }
7124
7125             my $write_zeros = 1;
7126             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7127                 $write_zeros = 0;
7128             }
7129
7130             my $path = PVE::Storage::path($cfg, $volid);
7131
7132             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7133
7134             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7135         }
7136
7137         $fh->seek(0, 0) || die "seek failed - $!\n";
7138
7139         my $cookie = { netcount => 0 };
7140         while (defined(my $line = <$fh>)) {
7141             $new_conf_raw .= restore_update_config_line(
7142                 $cookie,
7143                 $map,
7144                 $line,
7145                 $opts->{unique},
7146             );
7147         }
7148
7149         $fh->close();
7150     };
7151
7152     eval {
7153         # enable interrupts
7154         local $SIG{INT} =
7155             local $SIG{TERM} =
7156             local $SIG{QUIT} =
7157             local $SIG{HUP} =
7158             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7159         local $SIG{ALRM} = sub { die "got timeout\n"; };
7160
7161         $oldtimeout = alarm($timeout);
7162
7163         my $parser = sub {
7164             my $line = shift;
7165
7166             print "$line\n";
7167
7168             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7169                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7170                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7171             } elsif ($line =~ m/^CTIME: /) {
7172                 # we correctly received the vma config, so we can disable
7173                 # the timeout now for disk allocation (set to 10 minutes, so
7174                 # that we always timeout if something goes wrong)
7175                 alarm(600);
7176                 &$print_devmap();
7177                 print $fifofh "done\n";
7178                 my $tmp = $oldtimeout || 0;
7179                 $oldtimeout = undef;
7180                 alarm($tmp);
7181                 close($fifofh);
7182                 $fifofh = undef;
7183             }
7184         };
7185
7186         print "restore vma archive: $dbg_cmdstring\n";
7187         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7188     };
7189     my $err = $@;
7190
7191     alarm($oldtimeout) if $oldtimeout;
7192
7193     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7194
7195     close($fifofh) if $fifofh;
7196     unlink $mapfifo;
7197     rmtree $tmpdir;
7198
7199     if ($err) {
7200         $restore_destroy_volumes->($cfg, $virtdev_hash);
7201         die $err;
7202     }
7203
7204     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7205     PVE::QemuConfig->write_config($vmid, $new_conf);
7206
7207     eval { rescan($vmid, 1); };
7208     warn $@ if $@;
7209
7210     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7211 }
7212
7213 sub restore_tar_archive {
7214     my ($archive, $vmid, $user, $opts) = @_;
7215
7216     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7217         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7218         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7219     }
7220
7221     if ($archive ne '-') {
7222         my $firstfile = tar_archive_read_firstfile($archive);
7223         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7224             if $firstfile ne 'qemu-server.conf';
7225     }
7226
7227     my $storecfg = PVE::Storage::config();
7228
7229     # avoid zombie disks when restoring over an existing VM -> cleanup first
7230     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7231     # skiplock=1 because qmrestore has set the 'create' lock itself already
7232     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7233     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7234
7235     my $tocmd = "/usr/lib/qemu-server/qmextract";
7236
7237     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7238     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7239     $tocmd .= ' --prealloc' if $opts->{prealloc};
7240     $tocmd .= ' --info' if $opts->{info};
7241
7242     # tar option "xf" does not autodetect compression when read from STDIN,
7243     # so we pipe to zcat
7244     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7245         PVE::Tools::shellquote("--to-command=$tocmd");
7246
7247     my $tmpdir = "/var/tmp/vzdumptmp$$";
7248     mkpath $tmpdir;
7249
7250     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7251     local $ENV{VZDUMP_VMID} = $vmid;
7252     local $ENV{VZDUMP_USER} = $user;
7253
7254     my $conffile = PVE::QemuConfig->config_file($vmid);
7255     my $new_conf_raw = '';
7256
7257     # disable interrupts (always do cleanups)
7258     local $SIG{INT} =
7259         local $SIG{TERM} =
7260         local $SIG{QUIT} =
7261         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7262
7263     eval {
7264         # enable interrupts
7265         local $SIG{INT} =
7266             local $SIG{TERM} =
7267             local $SIG{QUIT} =
7268             local $SIG{HUP} =
7269             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7270
7271         if ($archive eq '-') {
7272             print "extracting archive from STDIN\n";
7273             run_command($cmd, input => "<&STDIN");
7274         } else {
7275             print "extracting archive '$archive'\n";
7276             run_command($cmd);
7277         }
7278
7279         return if $opts->{info};
7280
7281         # read new mapping
7282         my $map = {};
7283         my $statfile = "$tmpdir/qmrestore.stat";
7284         if (my $fd = IO::File->new($statfile, "r")) {
7285             while (defined (my $line = <$fd>)) {
7286                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7287                     $map->{$1} = $2 if $1;
7288                 } else {
7289                     print STDERR "unable to parse line in statfile - $line\n";
7290                 }
7291             }
7292             $fd->close();
7293         }
7294
7295         my $confsrc = "$tmpdir/qemu-server.conf";
7296
7297         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7298
7299         my $cookie = { netcount => 0 };
7300         while (defined (my $line = <$srcfd>)) {
7301             $new_conf_raw .= restore_update_config_line(
7302                 $cookie,
7303                 $map,
7304                 $line,
7305                 $opts->{unique},
7306             );
7307         }
7308
7309         $srcfd->close();
7310     };
7311     if (my $err = $@) {
7312         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7313         die $err;
7314     }
7315
7316     rmtree $tmpdir;
7317
7318     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7319
7320     PVE::Cluster::cfs_update(); # make sure we read new file
7321
7322     eval { rescan($vmid, 1); };
7323     warn $@ if $@;
7324 };
7325
7326 sub foreach_storage_used_by_vm {
7327     my ($conf, $func) = @_;
7328
7329     my $sidhash = {};
7330
7331     PVE::QemuConfig->foreach_volume($conf, sub {
7332         my ($ds, $drive) = @_;
7333         return if drive_is_cdrom($drive);
7334
7335         my $volid = $drive->{file};
7336
7337         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7338         $sidhash->{$sid} = $sid if $sid;
7339     });
7340
7341     foreach my $sid (sort keys %$sidhash) {
7342         &$func($sid);
7343     }
7344 }
7345
7346 my $qemu_snap_storage = {
7347     rbd => 1,
7348 };
7349 sub do_snapshots_with_qemu {
7350     my ($storecfg, $volid, $deviceid) = @_;
7351
7352     return if $deviceid =~ m/tpmstate0/;
7353
7354     my $storage_name = PVE::Storage::parse_volume_id($volid);
7355     my $scfg = $storecfg->{ids}->{$storage_name};
7356     die "could not find storage '$storage_name'\n" if !defined($scfg);
7357
7358     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7359         return 1;
7360     }
7361
7362     if ($volid =~ m/\.(qcow2|qed)$/){
7363         return 1;
7364     }
7365
7366     return;
7367 }
7368
7369 sub qga_check_running {
7370     my ($vmid, $nowarn) = @_;
7371
7372     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7373     if ($@) {
7374         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7375         return 0;
7376     }
7377     return 1;
7378 }
7379
7380 sub template_create {
7381     my ($vmid, $conf, $disk) = @_;
7382
7383     my $storecfg = PVE::Storage::config();
7384
7385     PVE::QemuConfig->foreach_volume($conf, sub {
7386         my ($ds, $drive) = @_;
7387
7388         return if drive_is_cdrom($drive);
7389         return if $disk && $ds ne $disk;
7390
7391         my $volid = $drive->{file};
7392         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7393
7394         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7395         $drive->{file} = $voliddst;
7396         $conf->{$ds} = print_drive($drive);
7397         PVE::QemuConfig->write_config($vmid, $conf);
7398     });
7399 }
7400
7401 sub convert_iscsi_path {
7402     my ($path) = @_;
7403
7404     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7405         my $portal = $1;
7406         my $target = $2;
7407         my $lun = $3;
7408
7409         my $initiator_name = get_initiator_name();
7410
7411         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7412                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7413     }
7414
7415     die "cannot convert iscsi path '$path', unkown format\n";
7416 }
7417
7418 sub qemu_img_convert {
7419     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7420
7421     my $storecfg = PVE::Storage::config();
7422     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7423     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7424
7425     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7426
7427     my $cachemode;
7428     my $src_path;
7429     my $src_is_iscsi = 0;
7430     my $src_format;
7431
7432     if ($src_storeid) {
7433         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7434         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7435         $src_format = qemu_img_format($src_scfg, $src_volname);
7436         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7437         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7438         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7439     } elsif (-f $src_volid || -b $src_volid) {
7440         $src_path = $src_volid;
7441         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7442             $src_format = $1;
7443         }
7444     }
7445
7446     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7447
7448     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7449     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7450     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7451     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7452
7453     my $cmd = [];
7454     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7455     push @$cmd, '-l', "snapshot.name=$snapname"
7456         if $snapname && $src_format && $src_format eq "qcow2";
7457     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7458     push @$cmd, '-T', $cachemode if defined($cachemode);
7459
7460     if ($src_is_iscsi) {
7461         push @$cmd, '--image-opts';
7462         $src_path = convert_iscsi_path($src_path);
7463     } elsif ($src_format) {
7464         push @$cmd, '-f', $src_format;
7465     }
7466
7467     if ($dst_is_iscsi) {
7468         push @$cmd, '--target-image-opts';
7469         $dst_path = convert_iscsi_path($dst_path);
7470     } else {
7471         push @$cmd, '-O', $dst_format;
7472     }
7473
7474     push @$cmd, $src_path;
7475
7476     if (!$dst_is_iscsi && $is_zero_initialized) {
7477         push @$cmd, "zeroinit:$dst_path";
7478     } else {
7479         push @$cmd, $dst_path;
7480     }
7481
7482     my $parser = sub {
7483         my $line = shift;
7484         if($line =~ m/\((\S+)\/100\%\)/){
7485             my $percent = $1;
7486             my $transferred = int($size * $percent / 100);
7487             my $total_h = render_bytes($size, 1);
7488             my $transferred_h = render_bytes($transferred, 1);
7489
7490             print "transferred $transferred_h of $total_h ($percent%)\n";
7491         }
7492
7493     };
7494
7495     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7496     my $err = $@;
7497     die "copy failed: $err" if $err;
7498 }
7499
7500 sub qemu_img_format {
7501     my ($scfg, $volname) = @_;
7502
7503     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7504         return $1;
7505     } else {
7506         return "raw";
7507     }
7508 }
7509
7510 sub qemu_drive_mirror {
7511     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7512
7513     $jobs = {} if !$jobs;
7514
7515     my $qemu_target;
7516     my $format;
7517     $jobs->{"drive-$drive"} = {};
7518
7519     if ($dst_volid =~ /^nbd:/) {
7520         $qemu_target = $dst_volid;
7521         $format = "nbd";
7522     } else {
7523         my $storecfg = PVE::Storage::config();
7524         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7525
7526         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7527
7528         $format = qemu_img_format($dst_scfg, $dst_volname);
7529
7530         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7531
7532         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7533     }
7534
7535     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7536     $opts->{format} = $format if $format;
7537
7538     if (defined($src_bitmap)) {
7539         $opts->{sync} = 'incremental';
7540         $opts->{bitmap} = $src_bitmap;
7541         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7542     }
7543
7544     if (defined($bwlimit)) {
7545         $opts->{speed} = $bwlimit * 1024;
7546         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7547     } else {
7548         print "drive mirror is starting for drive-$drive\n";
7549     }
7550
7551     # if a job already runs for this device we get an error, catch it for cleanup
7552     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7553     if (my $err = $@) {
7554         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7555         warn "$@\n" if $@;
7556         die "mirroring error: $err\n";
7557     }
7558
7559     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7560 }
7561
7562 # $completion can be either
7563 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7564 # 'cancel': wait until all jobs are ready, block-job-cancel them
7565 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7566 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7567 sub qemu_drive_mirror_monitor {
7568     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7569
7570     $completion //= 'complete';
7571     $op //= "mirror";
7572
7573     eval {
7574         my $err_complete = 0;
7575
7576         my $starttime = time ();
7577         while (1) {
7578             die "block job ('$op') timed out\n" if $err_complete > 300;
7579
7580             my $stats = mon_cmd($vmid, "query-block-jobs");
7581             my $ctime = time();
7582
7583             my $running_jobs = {};
7584             for my $stat (@$stats) {
7585                 next if $stat->{type} ne $op;
7586                 $running_jobs->{$stat->{device}} = $stat;
7587             }
7588
7589             my $readycounter = 0;
7590
7591             for my $job_id (sort keys %$jobs) {
7592                 my $job = $running_jobs->{$job_id};
7593
7594                 my $vanished = !defined($job);
7595                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7596                 if($complete || ($vanished && $completion eq 'auto')) {
7597                     print "$job_id: $op-job finished\n";
7598                     delete $jobs->{$job_id};
7599                     next;
7600                 }
7601
7602                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7603
7604                 my $busy = $job->{busy};
7605                 my $ready = $job->{ready};
7606                 if (my $total = $job->{len}) {
7607                     my $transferred = $job->{offset} || 0;
7608                     my $remaining = $total - $transferred;
7609                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7610
7611                     my $duration = $ctime - $starttime;
7612                     my $total_h = render_bytes($total, 1);
7613                     my $transferred_h = render_bytes($transferred, 1);
7614
7615                     my $status = sprintf(
7616                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7617                         render_duration($duration),
7618                     );
7619
7620                     if ($ready) {
7621                         if ($busy) {
7622                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7623                         } else {
7624                             $status .= ", ready";
7625                         }
7626                     }
7627                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7628                     $jobs->{$job_id}->{ready} = $ready;
7629                 }
7630
7631                 $readycounter++ if $job->{ready};
7632             }
7633
7634             last if scalar(keys %$jobs) == 0;
7635
7636             if ($readycounter == scalar(keys %$jobs)) {
7637                 print "all '$op' jobs are ready\n";
7638
7639                 # do the complete later (or has already been done)
7640                 last if $completion eq 'skip' || $completion eq 'auto';
7641
7642                 if ($vmiddst && $vmiddst != $vmid) {
7643                     my $agent_running = $qga && qga_check_running($vmid);
7644                     if ($agent_running) {
7645                         print "freeze filesystem\n";
7646                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7647                         warn $@ if $@;
7648                     } else {
7649                         print "suspend vm\n";
7650                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7651                         warn $@ if $@;
7652                     }
7653
7654                     # if we clone a disk for a new target vm, we don't switch the disk
7655                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7656
7657                     if ($agent_running) {
7658                         print "unfreeze filesystem\n";
7659                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7660                         warn $@ if $@;
7661                     } else {
7662                         print "resume vm\n";
7663                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7664                         warn $@ if $@;
7665                     }
7666
7667                     last;
7668                 } else {
7669
7670                     for my $job_id (sort keys %$jobs) {
7671                         # try to switch the disk if source and destination are on the same guest
7672                         print "$job_id: Completing block job_id...\n";
7673
7674                         my $op;
7675                         if ($completion eq 'complete') {
7676                             $op = 'block-job-complete';
7677                         } elsif ($completion eq 'cancel') {
7678                             $op = 'block-job-cancel';
7679                         } else {
7680                             die "invalid completion value: $completion\n";
7681                         }
7682                         eval { mon_cmd($vmid, $op, device => $job_id) };
7683                         if ($@ =~ m/cannot be completed/) {
7684                             print "$job_id: block job cannot be completed, trying again.\n";
7685                             $err_complete++;
7686                         }else {
7687                             print "$job_id: Completed successfully.\n";
7688                             $jobs->{$job_id}->{complete} = 1;
7689                         }
7690                     }
7691                 }
7692             }
7693             sleep 1;
7694         }
7695     };
7696     my $err = $@;
7697
7698     if ($err) {
7699         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7700         die "block job ($op) error: $err";
7701     }
7702 }
7703
7704 sub qemu_blockjobs_cancel {
7705     my ($vmid, $jobs) = @_;
7706
7707     foreach my $job (keys %$jobs) {
7708         print "$job: Cancelling block job\n";
7709         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7710         $jobs->{$job}->{cancel} = 1;
7711     }
7712
7713     while (1) {
7714         my $stats = mon_cmd($vmid, "query-block-jobs");
7715
7716         my $running_jobs = {};
7717         foreach my $stat (@$stats) {
7718             $running_jobs->{$stat->{device}} = $stat;
7719         }
7720
7721         foreach my $job (keys %$jobs) {
7722
7723             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7724                 print "$job: Done.\n";
7725                 delete $jobs->{$job};
7726             }
7727         }
7728
7729         last if scalar(keys %$jobs) == 0;
7730
7731         sleep 1;
7732     }
7733 }
7734
7735 sub clone_disk {
7736     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7737
7738     my ($vmid, $running) = $source->@{qw(vmid running)};
7739     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7740
7741     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7742     my ($storage, $format) = $dest->@{qw(storage format)};
7743
7744     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7745
7746     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7747         die "cloning from/to EFI disk requires EFI disk\n"
7748             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7749         die "cloning from/to TPM state requires TPM state\n"
7750             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7751
7752         # This would lead to two device nodes in QEMU pointing to the same backing image!
7753         die "cannot change drive name when cloning disk from/to the same VM\n"
7754             if $use_drive_mirror && $vmid == $newvmid;
7755     }
7756
7757     die "cannot move TPM state while VM is running\n"
7758         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7759
7760     my $newvolid;
7761
7762     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7763     print "$src_drivename " if $src_drivename;
7764     print "($drive->{file})\n";
7765
7766     if (!$full) {
7767         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7768         push @$newvollist, $newvolid;
7769     } else {
7770
7771         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7772         $storeid = $storage if $storage;
7773
7774         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7775
7776         my $name = undef;
7777         my $size = undef;
7778         if (drive_is_cloudinit($drive)) {
7779             $name = "vm-$newvmid-cloudinit";
7780             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7781             if ($scfg->{path}) {
7782                 $name .= ".$dst_format";
7783             }
7784             $snapname = undef;
7785             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7786         } elsif ($dst_drivename eq 'efidisk0') {
7787             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7788         } elsif ($dst_drivename eq 'tpmstate0') {
7789             $dst_format = 'raw';
7790             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7791         } else {
7792             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7793         }
7794         $newvolid = PVE::Storage::vdisk_alloc(
7795             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7796         );
7797         push @$newvollist, $newvolid;
7798
7799         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7800
7801         if (drive_is_cloudinit($drive)) {
7802             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7803             # if this is the case, we have to complete any block-jobs still there from
7804             # previous drive-mirrors
7805             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7806                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7807             }
7808             goto no_data_clone;
7809         }
7810
7811         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7812         if ($use_drive_mirror) {
7813             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7814                 $completion, $qga, $bwlimit);
7815         } else {
7816             # TODO: handle bwlimits
7817             if ($dst_drivename eq 'efidisk0') {
7818                 # the relevant data on the efidisk may be smaller than the source
7819                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7820                 # that is given by the OVMF_VARS.fd
7821                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7822                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7823
7824                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7825
7826                 # better for Ceph if block size is not too small, see bug #3324
7827                 my $bs = 1024*1024;
7828
7829                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7830
7831                 if ($src_format eq 'qcow2' && $snapname) {
7832                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7833                         if !min_version(kvm_user_version(), 6, 2);
7834                     push $cmd->@*, '-l', $snapname;
7835                 }
7836                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7837                 run_command($cmd);
7838             } else {
7839                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7840             }
7841         }
7842     }
7843
7844 no_data_clone:
7845     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7846
7847     my $disk = dclone($drive);
7848     delete $disk->{format};
7849     $disk->{file} = $newvolid;
7850     $disk->{size} = $size if defined($size);
7851
7852     return $disk;
7853 }
7854
7855 sub get_running_qemu_version {
7856     my ($vmid) = @_;
7857     my $res = mon_cmd($vmid, "query-version");
7858     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7859 }
7860
7861 sub qemu_use_old_bios_files {
7862     my ($machine_type) = @_;
7863
7864     return if !$machine_type;
7865
7866     my $use_old_bios_files = undef;
7867
7868     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7869         $machine_type = $1;
7870         $use_old_bios_files = 1;
7871     } else {
7872         my $version = extract_version($machine_type, kvm_user_version());
7873         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7874         # load new efi bios files on migration. So this hack is required to allow
7875         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7876         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7877         $use_old_bios_files = !min_version($version, 2, 4);
7878     }
7879
7880     return ($use_old_bios_files, $machine_type);
7881 }
7882
7883 sub get_efivars_size {
7884     my ($conf, $efidisk) = @_;
7885
7886     my $arch = get_vm_arch($conf);
7887     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7888     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7889     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7890     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7891     return -s $ovmf_vars;
7892 }
7893
7894 sub update_efidisk_size {
7895     my ($conf) = @_;
7896
7897     return if !defined($conf->{efidisk0});
7898
7899     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7900     $disk->{size} = get_efivars_size($conf);
7901     $conf->{efidisk0} = print_drive($disk);
7902
7903     return;
7904 }
7905
7906 sub update_tpmstate_size {
7907     my ($conf) = @_;
7908
7909     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7910     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7911     $conf->{tpmstate0} = print_drive($disk);
7912 }
7913
7914 sub create_efidisk($$$$$$$) {
7915     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7916
7917     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7918     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7919
7920     my $vars_size_b = -s $ovmf_vars;
7921     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7922     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7923     PVE::Storage::activate_volumes($storecfg, [$volid]);
7924
7925     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7926     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7927
7928     return ($volid, $size/1024);
7929 }
7930
7931 sub vm_iothreads_list {
7932     my ($vmid) = @_;
7933
7934     my $res = mon_cmd($vmid, 'query-iothreads');
7935
7936     my $iothreads = {};
7937     foreach my $iothread (@$res) {
7938         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7939     }
7940
7941     return $iothreads;
7942 }
7943
7944 sub scsihw_infos {
7945     my ($conf, $drive) = @_;
7946
7947     my $maxdev = 0;
7948
7949     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7950         $maxdev = 7;
7951     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7952         $maxdev = 1;
7953     } else {
7954         $maxdev = 256;
7955     }
7956
7957     my $controller = int($drive->{index} / $maxdev);
7958     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7959         ? "virtioscsi"
7960         : "scsihw";
7961
7962     return ($maxdev, $controller, $controller_prefix);
7963 }
7964
7965 sub resolve_dst_disk_format {
7966         my ($storecfg, $storeid, $src_volname, $format) = @_;
7967         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7968
7969         if (!$format) {
7970             # if no target format is specified, use the source disk format as hint
7971             if ($src_volname) {
7972                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7973                 $format = qemu_img_format($scfg, $src_volname);
7974             } else {
7975                 return $defFormat;
7976             }
7977         }
7978
7979         # test if requested format is supported - else use default
7980         my $supported = grep { $_ eq $format } @$validFormats;
7981         $format = $defFormat if !$supported;
7982         return $format;
7983 }
7984
7985 # NOTE: if this logic changes, please update docs & possibly gui logic
7986 sub find_vmstate_storage {
7987     my ($conf, $storecfg) = @_;
7988
7989     # first, return storage from conf if set
7990     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7991
7992     my ($target, $shared, $local);
7993
7994     foreach_storage_used_by_vm($conf, sub {
7995         my ($sid) = @_;
7996         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7997         my $dst = $scfg->{shared} ? \$shared : \$local;
7998         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7999     });
8000
8001     # second, use shared storage where VM has at least one disk
8002     # third, use local storage where VM has at least one disk
8003     # fall back to local storage
8004     $target = $shared // $local // 'local';
8005
8006     return $target;
8007 }
8008
8009 sub generate_uuid {
8010     my ($uuid, $uuid_str);
8011     UUID::generate($uuid);
8012     UUID::unparse($uuid, $uuid_str);
8013     return $uuid_str;
8014 }
8015
8016 sub generate_smbios1_uuid {
8017     return "uuid=".generate_uuid();
8018 }
8019
8020 sub nbd_stop {
8021     my ($vmid) = @_;
8022
8023     mon_cmd($vmid, 'nbd-server-stop');
8024 }
8025
8026 sub create_reboot_request {
8027     my ($vmid) = @_;
8028     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8029         or die "failed to create reboot trigger file: $!\n";
8030     close($fh);
8031 }
8032
8033 sub clear_reboot_request {
8034     my ($vmid) = @_;
8035     my $path = "/run/qemu-server/$vmid.reboot";
8036     my $res = 0;
8037
8038     $res = unlink($path);
8039     die "could not remove reboot request for $vmid: $!"
8040         if !$res && $! != POSIX::ENOENT;
8041
8042     return $res;
8043 }
8044
8045 sub bootorder_from_legacy {
8046     my ($conf, $bootcfg) = @_;
8047
8048     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8049     my $bootindex_hash = {};
8050     my $i = 1;
8051     foreach my $o (split(//, $boot)) {
8052         $bootindex_hash->{$o} = $i*100;
8053         $i++;
8054     }
8055
8056     my $bootorder = {};
8057
8058     PVE::QemuConfig->foreach_volume($conf, sub {
8059         my ($ds, $drive) = @_;
8060
8061         if (drive_is_cdrom ($drive, 1)) {
8062             if ($bootindex_hash->{d}) {
8063                 $bootorder->{$ds} = $bootindex_hash->{d};
8064                 $bootindex_hash->{d} += 1;
8065             }
8066         } elsif ($bootindex_hash->{c}) {
8067             $bootorder->{$ds} = $bootindex_hash->{c}
8068                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8069             $bootindex_hash->{c} += 1;
8070         }
8071     });
8072
8073     if ($bootindex_hash->{n}) {
8074         for (my $i = 0; $i < $MAX_NETS; $i++) {
8075             my $netname = "net$i";
8076             next if !$conf->{$netname};
8077             $bootorder->{$netname} = $bootindex_hash->{n};
8078             $bootindex_hash->{n} += 1;
8079         }
8080     }
8081
8082     return $bootorder;
8083 }
8084
8085 # Generate default device list for 'boot: order=' property. Matches legacy
8086 # default boot order, but with explicit device names. This is important, since
8087 # the fallback for when neither 'order' nor the old format is specified relies
8088 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8089 sub get_default_bootdevices {
8090     my ($conf) = @_;
8091
8092     my @ret = ();
8093
8094     # harddisk
8095     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8096     push @ret, $first if $first;
8097
8098     # cdrom
8099     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8100     push @ret, $first if $first;
8101
8102     # network
8103     for (my $i = 0; $i < $MAX_NETS; $i++) {
8104         my $netname = "net$i";
8105         next if !$conf->{$netname};
8106         push @ret, $netname;
8107         last;
8108     }
8109
8110     return \@ret;
8111 }
8112
8113 sub device_bootorder {
8114     my ($conf) = @_;
8115
8116     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8117
8118     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8119
8120     my $bootorder = {};
8121     if (!defined($boot) || $boot->{legacy}) {
8122         $bootorder = bootorder_from_legacy($conf, $boot);
8123     } elsif ($boot->{order}) {
8124         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8125         for my $dev (PVE::Tools::split_list($boot->{order})) {
8126             $bootorder->{$dev} = $i++;
8127         }
8128     }
8129
8130     return $bootorder;
8131 }
8132
8133 sub register_qmeventd_handle {
8134     my ($vmid) = @_;
8135
8136     my $fh;
8137     my $peer = "/var/run/qmeventd.sock";
8138     my $count = 0;
8139
8140     for (;;) {
8141         $count++;
8142         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8143         last if $fh;
8144         if ($! != EINTR && $! != EAGAIN) {
8145             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8146         }
8147         if ($count > 4) {
8148             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8149               . "after $count retries\n";
8150         }
8151         usleep(25000);
8152     }
8153
8154     # send handshake to mark VM as backing up
8155     print $fh to_json({vzdump => {vmid => "$vmid"}});
8156
8157     # return handle to be closed later when inhibit is no longer required
8158     return $fh;
8159 }
8160
8161 # bash completion helper
8162
8163 sub complete_backup_archives {
8164     my ($cmdname, $pname, $cvalue) = @_;
8165
8166     my $cfg = PVE::Storage::config();
8167
8168     my $storeid;
8169
8170     if ($cvalue =~ m/^([^:]+):/) {
8171         $storeid = $1;
8172     }
8173
8174     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8175
8176     my $res = [];
8177     foreach my $id (keys %$data) {
8178         foreach my $item (@{$data->{$id}}) {
8179             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8180             push @$res, $item->{volid} if defined($item->{volid});
8181         }
8182     }
8183
8184     return $res;
8185 }
8186
8187 my $complete_vmid_full = sub {
8188     my ($running) = @_;
8189
8190     my $idlist = vmstatus();
8191
8192     my $res = [];
8193
8194     foreach my $id (keys %$idlist) {
8195         my $d = $idlist->{$id};
8196         if (defined($running)) {
8197             next if $d->{template};
8198             next if $running && $d->{status} ne 'running';
8199             next if !$running && $d->{status} eq 'running';
8200         }
8201         push @$res, $id;
8202
8203     }
8204     return $res;
8205 };
8206
8207 sub complete_vmid {
8208     return &$complete_vmid_full();
8209 }
8210
8211 sub complete_vmid_stopped {
8212     return &$complete_vmid_full(0);
8213 }
8214
8215 sub complete_vmid_running {
8216     return &$complete_vmid_full(1);
8217 }
8218
8219 sub complete_storage {
8220
8221     my $cfg = PVE::Storage::config();
8222     my $ids = $cfg->{ids};
8223
8224     my $res = [];
8225     foreach my $sid (keys %$ids) {
8226         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8227         next if !$ids->{$sid}->{content}->{images};
8228         push @$res, $sid;
8229     }
8230
8231     return $res;
8232 }
8233
8234 sub complete_migration_storage {
8235     my ($cmd, $param, $current_value, $all_args) = @_;
8236
8237     my $targetnode = @$all_args[1];
8238
8239     my $cfg = PVE::Storage::config();
8240     my $ids = $cfg->{ids};
8241
8242     my $res = [];
8243     foreach my $sid (keys %$ids) {
8244         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8245         next if !$ids->{$sid}->{content}->{images};
8246         push @$res, $sid;
8247     }
8248
8249     return $res;
8250 }
8251
8252 sub vm_is_paused {
8253     my ($vmid) = @_;
8254     my $qmpstatus = eval {
8255         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8256         mon_cmd($vmid, "query-status");
8257     };
8258     warn "$@\n" if $@;
8259     return $qmpstatus && $qmpstatus->{status} eq "paused";
8260 }
8261
8262 sub check_volume_storage_type {
8263     my ($storecfg, $vol) = @_;
8264
8265     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8266     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8267     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8268
8269     die "storage '$storeid' does not support content-type '$vtype'\n"
8270         if !$scfg->{content}->{$vtype};
8271
8272     return 1;
8273 }
8274
8275 1;