PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RESTEnvironment qw(log_warn);
  40 use PVE::RPCEnvironment;
  41 use PVE::Storage;
  42 use PVE::SysFSTools;
  43 use PVE::Systemd;
  44 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  45
  46 use PVE::QMPClient;
  47 use PVE::QemuConfig;
  48 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  49 use PVE::QemuServer::Cloudinit;
  50 use PVE::QemuServer::CGroup;
  51 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  52 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  53 use PVE::QemuServer::Machine;
  54 use PVE::QemuServer::Memory;
  55 use PVE::QemuServer::Monitor qw(mon_cmd);
  56 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  57 use PVE::QemuServer::USB qw(parse_usb_device);
  58
  59 my $have_sdn;
  60 eval {
  61     require PVE::Network::SDN::Zones;
  62     $have_sdn = 1;
  63 };
  64
  65 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  66 my $OVMF = {
  67     x86_64 => {
  68         '4m-no-smm' => [
  69             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  70             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  71         ],
  72         '4m-no-smm-ms' => [
  73             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  74             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  75         ],
  76         '4m' => [
  77             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  78             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  79         ],
  80         '4m-ms' => [
  81             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  82             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  83         ],
  84         default => [
  85             "$EDK2_FW_BASE/OVMF_CODE.fd",
  86             "$EDK2_FW_BASE/OVMF_VARS.fd",
  87         ],
  88     },
  89     aarch64 => {
  90         default => [
  91             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  92             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  93         ],
  94     },
  95 };
  96
  97 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  98
  99 # Note about locking: we use flock on the config file protect against concurent actions.
 100 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 101 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 102 # But you can ignore this kind of lock with the --skiplock flag.
 103
 104 cfs_register_file('/qemu-server/',
 105                   \&parse_vm_config,
 106                   \&write_vm_config);
 107
 108 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 109     description => "Some command save/restore state from this location.",
 110     type => 'string',
 111     maxLength => 128,
 112     optional => 1,
 113 });
 114
 115 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 116         description => "Specifies the Qemu machine type.",
 117         type => 'string',
 118         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 119         maxLength => 40,
 120         optional => 1,
 121 });
 122
 123 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 124     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 125     type => 'string',
 126     format => 'storage-pair-list',
 127     optional => 1,
 128 });
 129
 130 #no warnings 'redefine';
 131
 132 my $nodename_cache;
 133 sub nodename {
 134     $nodename_cache //= PVE::INotify::nodename();
 135     return $nodename_cache;
 136 }
 137
 138 my $watchdog_fmt = {
 139     model => {
 140         default_key => 1,
 141         type => 'string',
 142         enum => [qw(i6300esb ib700)],
 143         description => "Watchdog type to emulate.",
 144         default => 'i6300esb',
 145         optional => 1,
 146     },
 147     action => {
 148         type => 'string',
 149         enum => [qw(reset shutdown poweroff pause debug none)],
 150         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 151         optional => 1,
 152     },
 153 };
 154 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 155
 156 my $agent_fmt = {
 157     enabled => {
 158         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 159         type => 'boolean',
 160         default => 0,
 161         default_key => 1,
 162     },
 163     fstrim_cloned_disks => {
 164         description => "Run fstrim after moving a disk or migrating the VM.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     type => {
 170         description => "Select the agent type",
 171         type => 'string',
 172         default => 'virtio',
 173         optional => 1,
 174         enum => [qw(virtio isa)],
 175     },
 176 };
 177
 178 my $vga_fmt = {
 179     type => {
 180         description => "Select the VGA type.",
 181         type => 'string',
 182         default => 'std',
 183         optional => 1,
 184         default_key => 1,
 185         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 186     },
 187     memory => {
 188         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 189         type => 'integer',
 190         optional => 1,
 191         minimum => 4,
 192         maximum => 512,
 193     },
 194 };
 195
 196 my $ivshmem_fmt = {
 197     size => {
 198         type => 'integer',
 199         minimum => 1,
 200         description => "The size of the file in MB.",
 201     },
 202     name => {
 203         type => 'string',
 204         pattern => '[a-zA-Z0-9\-]+',
 205         optional => 1,
 206         format_description => 'string',
 207         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 208     },
 209 };
 210
 211 my $audio_fmt = {
 212     device => {
 213         type => 'string',
 214         enum => [qw(ich9-intel-hda intel-hda AC97)],
 215         description =>  "Configure an audio device."
 216     },
 217     driver =>  {
 218         type => 'string',
 219         enum => ['spice', 'none'],
 220         default => 'spice',
 221         optional => 1,
 222         description => "Driver backend for the audio device."
 223     },
 224 };
 225
 226 my $spice_enhancements_fmt = {
 227     foldersharing => {
 228         type => 'boolean',
 229         optional => 1,
 230         default => '0',
 231         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 232     },
 233     videostreaming =>  {
 234         type => 'string',
 235         enum => ['off', 'all', 'filter'],
 236         default => 'off',
 237         optional => 1,
 238         description => "Enable video streaming. Uses compression for detected video streams."
 239     },
 240 };
 241
 242 my $rng_fmt = {
 243     source => {
 244         type => 'string',
 245         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 246         default_key => 1,
 247         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 248             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 249             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 250             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 251             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 252             ." a hardware RNG from the host.",
 253     },
 254     max_bytes => {
 255         type => 'integer',
 256         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 257             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 258             ." `0` to disable limiting (potentially dangerous!).",
 259         optional => 1,
 260
 261         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 262         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 263         # reading from /dev/urandom
 264         default => 1024,
 265     },
 266     period => {
 267         type => 'integer',
 268         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 269             ." the guest to retrieve another 'max_bytes' of entropy.",
 270         optional => 1,
 271         default => 1000,
 272     },
 273 };
 274
 275 my $meta_info_fmt = {
 276     'ctime' => {
 277         type => 'integer',
 278         description => "The guest creation timestamp as UNIX epoch time",
 279         minimum => 0,
 280         optional => 1,
 281     },
 282     'creation-qemu' => {
 283         type => 'string',
 284         description => "The QEMU (machine) version from the time this VM was created.",
 285         pattern => '\d+(\.\d+)+',
 286         optional => 1,
 287     },
 288 };
 289
 290 my $confdesc = {
 291     onboot => {
 292         optional => 1,
 293         type => 'boolean',
 294         description => "Specifies whether a VM will be started during system bootup.",
 295         default => 0,
 296     },
 297     autostart => {
 298         optional => 1,
 299         type => 'boolean',
 300         description => "Automatic restart after crash (currently ignored).",
 301         default => 0,
 302     },
 303     hotplug => {
 304         optional => 1,
 305         type => 'string', format => 'pve-hotplug-features',
 306         description => "Selectively enable hotplug features. This is a comma separated list of"
 307             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 308             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 309         default => 'network,disk,usb',
 310     },
 311     reboot => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 315         default => 1,
 316     },
 317     lock => {
 318         optional => 1,
 319         type => 'string',
 320         description => "Lock/unlock the VM.",
 321         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 322     },
 323     cpulimit => {
 324         optional => 1,
 325         type => 'number',
 326         description => "Limit of CPU usage.",
 327         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 328             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 329         minimum => 0,
 330         maximum => 128,
 331         default => 0,
 332     },
 333     cpuunits => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 337         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 338             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 339             ." weights of all the other running VMs.",
 340         minimum => 1,
 341         maximum => 262144,
 342         default => 'cgroup v1: 1024, cgroup v2: 100',
 343     },
 344     memory => {
 345         optional => 1,
 346         type => 'integer',
 347         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 348             ." you use the balloon device.",
 349         minimum => 16,
 350         default => 512,
 351     },
 352     balloon => {
 353         optional => 1,
 354         type => 'integer',
 355         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 356         minimum => 0,
 357     },
 358     shares => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 362             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 363             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 364         minimum => 0,
 365         maximum => 50000,
 366         default => 1000,
 367     },
 368     keyboard => {
 369         optional => 1,
 370         type => 'string',
 371         description => "Keyboard layout for VNC server. This option is generally not required and"
 372         ." is often better handled from within the guest OS.",
 373         enum => PVE::Tools::kvmkeymaplist(),
 374         default => undef,
 375     },
 376     name => {
 377         optional => 1,
 378         type => 'string', format => 'dns-name',
 379         description => "Set a name for the VM. Only used on the configuration web interface.",
 380     },
 381     scsihw => {
 382         optional => 1,
 383         type => 'string',
 384         description => "SCSI controller model",
 385         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 386         default => 'lsi',
 387     },
 388     description => {
 389         optional => 1,
 390         type => 'string',
 391         description => "Description for the VM. Shown in the web-interface VM's summary."
 392             ." This is saved as comment inside the configuration file.",
 393         maxLength => 1024 * 8,
 394     },
 395     ostype => {
 396         optional => 1,
 397         type => 'string',
 398         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 399         description => "Specify guest operating system.",
 400         verbose_description => <<EODESC,
 401 Specify guest operating system. This is used to enable special
 402 optimization/features for specific operating systems:
 403
 404 [horizontal]
 405 other;; unspecified OS
 406 wxp;; Microsoft Windows XP
 407 w2k;; Microsoft Windows 2000
 408 w2k3;; Microsoft Windows 2003
 409 w2k8;; Microsoft Windows 2008
 410 wvista;; Microsoft Windows Vista
 411 win7;; Microsoft Windows 7
 412 win8;; Microsoft Windows 8/2012/2012r2
 413 win10;; Microsoft Windows 10/2016/2019
 414 win11;; Microsoft Windows 11/2022
 415 l24;; Linux 2.4 Kernel
 416 l26;; Linux 2.6 - 5.X Kernel
 417 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 418 EODESC
 419     },
 420     boot => {
 421         optional => 1,
 422         type => 'string', format => 'pve-qm-boot',
 423         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 424             ." key or 'legacy=' is deprecated.",
 425     },
 426     bootdisk => {
 427         optional => 1,
 428         type => 'string', format => 'pve-qm-bootdisk',
 429         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 430         pattern => '(ide|sata|scsi|virtio)\d+',
 431     },
 432     smp => {
 433         optional => 1,
 434         type => 'integer',
 435         description => "The number of CPUs. Please use option -sockets instead.",
 436         minimum => 1,
 437         default => 1,
 438     },
 439     sockets => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "The number of CPU sockets.",
 443         minimum => 1,
 444         default => 1,
 445     },
 446     cores => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "The number of cores per socket.",
 450         minimum => 1,
 451         default => 1,
 452     },
 453     numa => {
 454         optional => 1,
 455         type => 'boolean',
 456         description => "Enable/disable NUMA.",
 457         default => 0,
 458     },
 459     hugepages => {
 460         optional => 1,
 461         type => 'string',
 462         description => "Enable/disable hugepages memory.",
 463         enum => [qw(any 2 1024)],
 464     },
 465     keephugepages => {
 466         optional => 1,
 467         type => 'boolean',
 468         default => 0,
 469         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 470             ." after VM shutdown and can be used for subsequent starts.",
 471     },
 472     vcpus => {
 473         optional => 1,
 474         type => 'integer',
 475         description => "Number of hotplugged vcpus.",
 476         minimum => 1,
 477         default => 0,
 478     },
 479     acpi => {
 480         optional => 1,
 481         type => 'boolean',
 482         description => "Enable/disable ACPI.",
 483         default => 1,
 484     },
 485     agent => {
 486         optional => 1,
 487         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 488         type => 'string',
 489         format => $agent_fmt,
 490     },
 491     kvm => {
 492         optional => 1,
 493         type => 'boolean',
 494         description => "Enable/disable KVM hardware virtualization.",
 495         default => 1,
 496     },
 497     tdf => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Enable/disable time drift fix.",
 501         default => 0,
 502     },
 503     localtime => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 507             ." the `ostype` indicates a Microsoft Windows OS.",
 508     },
 509     freeze => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 513     },
 514     vga => {
 515         optional => 1,
 516         type => 'string', format => $vga_fmt,
 517         description => "Configure the VGA hardware.",
 518         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 519             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 520             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 521             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 522             ." display server. For win* OS you can select how many independent displays you want,"
 523             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 524             ." using a serial device as terminal.",
 525     },
 526     watchdog => {
 527         optional => 1,
 528         type => 'string', format => 'pve-qm-watchdog',
 529         description => "Create a virtual hardware watchdog device.",
 530         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 531             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 532             ." else the watchdog will reset the guest (or execute the respective action specified)",
 533     },
 534     startdate => {
 535         optional => 1,
 536         type => 'string',
 537         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 538         description => "Set the initial date of the real time clock. Valid format for date are:"
 539             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 540         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 541         default => 'now',
 542     },
 543     startup =>  get_standard_option('pve-startup-order'),
 544     template => {
 545         optional => 1,
 546         type => 'boolean',
 547         description => "Enable/disable Template.",
 548         default => 0,
 549     },
 550     args => {
 551         optional => 1,
 552         type => 'string',
 553         description => "Arbitrary arguments passed to kvm.",
 554         verbose_description => <<EODESCR,
 555 Arbitrary arguments passed to kvm, for example:
 556
 557 args: -no-reboot -no-hpet
 558
 559 NOTE: this option is for experts only.
 560 EODESCR
 561     },
 562     tablet => {
 563         optional => 1,
 564         type => 'boolean',
 565         default => 1,
 566         description => "Enable/disable the USB tablet device.",
 567         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 568             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 569             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 570             ." may consider disabling this to save some context switches. This is turned off by"
 571             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 572     },
 573     migrate_speed => {
 574         optional => 1,
 575         type => 'integer',
 576         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 577         minimum => 0,
 578         default => 0,
 579     },
 580     migrate_downtime => {
 581         optional => 1,
 582         type => 'number',
 583         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 584         minimum => 0,
 585         default => 0.1,
 586     },
 587     cdrom => {
 588         optional => 1,
 589         type => 'string', format => 'pve-qm-ide',
 590         typetext => '<volume>',
 591         description => "This is an alias for option -ide2",
 592     },
 593     cpu => {
 594         optional => 1,
 595         description => "Emulated CPU type.",
 596         type => 'string',
 597         format => 'pve-vm-cpu-conf',
 598     },
 599     parent => get_standard_option('pve-snapshot-name', {
 600         optional => 1,
 601         description => "Parent snapshot name. This is used internally, and should not be modified.",
 602     }),
 603     snaptime => {
 604         optional => 1,
 605         description => "Timestamp for snapshots.",
 606         type => 'integer',
 607         minimum => 0,
 608     },
 609     vmstate => {
 610         optional => 1,
 611         type => 'string', format => 'pve-volume-id',
 612         description => "Reference to a volume which stores the VM state. This is used internally"
 613             ." for snapshots.",
 614     },
 615     vmstatestorage => get_standard_option('pve-storage-id', {
 616         description => "Default storage for VM state volumes/files.",
 617         optional => 1,
 618     }),
 619     runningmachine => get_standard_option('pve-qemu-machine', {
 620         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 621             ." for snapshots.",
 622     }),
 623     runningcpu => {
 624         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 625             ." internally for snapshots.",
 626         optional => 1,
 627         type => 'string',
 628         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 629         format_description => 'QEMU -cpu parameter'
 630     },
 631     machine => get_standard_option('pve-qemu-machine'),
 632     arch => {
 633         description => "Virtual processor architecture. Defaults to the host.",
 634         optional => 1,
 635         type => 'string',
 636         enum => [qw(x86_64 aarch64)],
 637     },
 638     smbios1 => {
 639         description => "Specify SMBIOS type 1 fields.",
 640         type => 'string', format => 'pve-qm-smbios1',
 641         maxLength => 512,
 642         optional => 1,
 643     },
 644     protection => {
 645         optional => 1,
 646         type => 'boolean',
 647         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 648             ." remove disk operations.",
 649         default => 0,
 650     },
 651     bios => {
 652         optional => 1,
 653         type => 'string',
 654         enum => [ qw(seabios ovmf) ],
 655         description => "Select BIOS implementation.",
 656         default => 'seabios',
 657     },
 658     vmgenid => {
 659         type => 'string',
 660         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 661         format_description => 'UUID',
 662         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 663             ." to disable explicitly.",
 664         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 665             ." value identifier to the guest OS. This allows to notify the guest operating system"
 666             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 667             ." execution or creation from a template). The guest operating system notices the"
 668             ." change, and is then able to react as appropriate by marking its copies of"
 669             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 670             ."Note that auto-creation only works when done through API/CLI create or update methods"
 671             .", but not when manually editing the config file.",
 672         default => "1 (autogenerated)",
 673         optional => 1,
 674     },
 675     hookscript => {
 676         type => 'string',
 677         format => 'pve-volume-id',
 678         optional => 1,
 679         description => "Script that will be executed during various steps in the vms lifetime.",
 680     },
 681     ivshmem => {
 682         type => 'string',
 683         format => $ivshmem_fmt,
 684         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 685             ." the host.",
 686         optional => 1,
 687     },
 688     audio0 => {
 689         type => 'string',
 690         format => $audio_fmt,
 691         description => "Configure a audio device, useful in combination with QXL/Spice.",
 692         optional => 1
 693     },
 694     spice_enhancements => {
 695         type => 'string',
 696         format => $spice_enhancements_fmt,
 697         description => "Configure additional enhancements for SPICE.",
 698         optional => 1
 699     },
 700     tags => {
 701         type => 'string', format => 'pve-tag-list',
 702         description => 'Tags of the VM. This is only meta information.',
 703         optional => 1,
 704     },
 705     rng0 => {
 706         type => 'string',
 707         format => $rng_fmt,
 708         description => "Configure a VirtIO-based Random Number Generator.",
 709         optional => 1,
 710     },
 711     meta => {
 712         type => 'string',
 713         format => $meta_info_fmt,
 714         description => "Some (read-only) meta-information about this guest.",
 715         optional => 1,
 716     },
 717 };
 718
 719 my $cicustom_fmt = {
 720     meta => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all meta data passed to the VM via"
 724             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728     network => {
 729         type => 'string',
 730         optional => 1,
 731         description => 'Specify a custom file containing all network data passed to the VM via'
 732             .' cloud-init.',
 733         format => 'pve-volume-id',
 734         format_description => 'volume',
 735     },
 736     user => {
 737         type => 'string',
 738         optional => 1,
 739         description => 'Specify a custom file containing all user data passed to the VM via'
 740             .' cloud-init.',
 741         format => 'pve-volume-id',
 742         format_description => 'volume',
 743     },
 744     vendor => {
 745     type => 'string',
 746     optional => 1,
 747     description => 'Specify a custom file containing all vendor data passed to the VM via'
 748      .' cloud-init.',
 749     format => 'pve-volume-id',
 750     format_description => 'volume',
 751     },
 752 };
 753 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 754
 755 my $confdesc_cloudinit = {
 756     citype => {
 757         optional => 1,
 758         type => 'string',
 759         description => 'Specifies the cloud-init configuration format. The default depends on the'
 760             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 761             .' and `configdrive2` for windows.',
 762         enum => ['configdrive2', 'nocloud', 'opennebula'],
 763     },
 764     ciuser => {
 765         optional => 1,
 766         type => 'string',
 767         description => "cloud-init: User name to change ssh keys and password for instead of the"
 768             ." image's configured default user.",
 769     },
 770     cipassword => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'cloud-init: Password to assign the user. Using this is generally not'
 774             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 775             .' support hashed passwords.',
 776     },
 777     cicustom => {
 778         optional => 1,
 779         type => 'string',
 780         description => 'cloud-init: Specify custom files to replace the automatically generated'
 781             .' ones at start.',
 782         format => 'pve-qm-cicustom',
 783     },
 784     searchdomain => {
 785         optional => 1,
 786         type => 'string',
 787         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 788             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 789             .' are set.',
 790     },
 791     nameserver => {
 792         optional => 1,
 793         type => 'string', format => 'address-list',
 794         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 795             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 796             .' are set.',
 797     },
 798     sshkeys => {
 799         optional => 1,
 800         type => 'string',
 801         format => 'urlencoded',
 802         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 803     },
 804 };
 805
 806 # what about other qemu settings ?
 807 #cpu => 'string',
 808 #machine => 'string',
 809 #fda => 'file',
 810 #fdb => 'file',
 811 #mtdblock => 'file',
 812 #sd => 'file',
 813 #pflash => 'file',
 814 #snapshot => 'bool',
 815 #bootp => 'file',
 816 ##tftp => 'dir',
 817 ##smb => 'dir',
 818 #kernel => 'file',
 819 #append => 'string',
 820 #initrd => 'file',
 821 ##soundhw => 'string',
 822
 823 while (my ($k, $v) = each %$confdesc) {
 824     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 825 }
 826
 827 my $MAX_USB_DEVICES = 5;
 828 my $MAX_NETS = 32;
 829 my $MAX_SERIAL_PORTS = 4;
 830 my $MAX_PARALLEL_PORTS = 3;
 831 my $MAX_NUMA = 8;
 832
 833 my $numa_fmt = {
 834     cpus => {
 835         type => "string",
 836         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 837         description => "CPUs accessing this NUMA node.",
 838         format_description => "id[-id];...",
 839     },
 840     memory => {
 841         type => "number",
 842         description => "Amount of memory this NUMA node provides.",
 843         optional => 1,
 844     },
 845     hostnodes => {
 846         type => "string",
 847         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 848         description => "Host NUMA nodes to use.",
 849         format_description => "id[-id];...",
 850         optional => 1,
 851     },
 852     policy => {
 853         type => 'string',
 854         enum => [qw(preferred bind interleave)],
 855         description => "NUMA allocation policy.",
 856         optional => 1,
 857     },
 858 };
 859 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 860 my $numadesc = {
 861     optional => 1,
 862     type => 'string', format => $numa_fmt,
 863     description => "NUMA topology.",
 864 };
 865 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 866
 867 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 868     $confdesc->{"numa$i"} = $numadesc;
 869 }
 870
 871 my $nic_model_list = [
 872     'e1000',
 873     'e1000-82540em',
 874     'e1000-82544gc',
 875     'e1000-82545em',
 876     'e1000e',
 877     'i82551',
 878     'i82557b',
 879     'i82559er',
 880     'ne2k_isa',
 881     'ne2k_pci',
 882     'pcnet',
 883     'rtl8139',
 884     'virtio',
 885     'vmxnet3',
 886 ];
 887 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 888
 889 my $net_fmt_bridge_descr = <<__EOD__;
 890 Bridge to attach the network device to. The Proxmox VE standard bridge
 891 is called 'vmbr0'.
 892
 893 If you do not specify a bridge, we create a kvm user (NATed) network
 894 device, which provides DHCP and DNS services. The following addresses
 895 are used:
 896
 897  10.0.2.2   Gateway
 898  10.0.2.3   DNS Server
 899  10.0.2.4   SMB Server
 900
 901 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 902 __EOD__
 903
 904 my $net_fmt = {
 905     macaddr  => get_standard_option('mac-addr', {
 906         description => "MAC address. That address must be unique withing your network. This is"
 907             ." automatically generated if not specified.",
 908     }),
 909     model => {
 910         type => 'string',
 911         description => "Network Card Model. The 'virtio' model provides the best performance with"
 912             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 913             ." best to use 'e1000'.",
 914         enum => $nic_model_list,
 915         default_key => 1,
 916     },
 917     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 918     bridge => get_standard_option('pve-bridge-id', {
 919         description => $net_fmt_bridge_descr,
 920         optional => 1,
 921     }),
 922     queues => {
 923         type => 'integer',
 924         minimum => 0, maximum => 16,
 925         description => 'Number of packet queues to be used on the device.',
 926         optional => 1,
 927     },
 928     rate => {
 929         type => 'number',
 930         minimum => 0,
 931         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 932         optional => 1,
 933     },
 934     tag => {
 935         type => 'integer',
 936         minimum => 1, maximum => 4094,
 937         description => 'VLAN tag to apply to packets on this interface.',
 938         optional => 1,
 939     },
 940     trunks => {
 941         type => 'string',
 942         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 943         description => 'VLAN trunks to pass through this interface.',
 944         format_description => 'vlanid[;vlanid...]',
 945         optional => 1,
 946     },
 947     firewall => {
 948         type => 'boolean',
 949         description => 'Whether this interface should be protected by the firewall.',
 950         optional => 1,
 951     },
 952     link_down => {
 953         type => 'boolean',
 954         description => 'Whether this interface should be disconnected (like pulling the plug).',
 955         optional => 1,
 956     },
 957     mtu => {
 958         type => 'integer',
 959         minimum => 1, maximum => 65520,
 960         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 961         optional => 1,
 962     },
 963 };
 964
 965 my $netdesc = {
 966     optional => 1,
 967     type => 'string', format => $net_fmt,
 968     description => "Specify network devices.",
 969 };
 970
 971 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 972
 973 my $ipconfig_fmt = {
 974     ip => {
 975         type => 'string',
 976         format => 'pve-ipv4-config',
 977         format_description => 'IPv4Format/CIDR',
 978         description => 'IPv4 address in CIDR format.',
 979         optional => 1,
 980         default => 'dhcp',
 981     },
 982     gw => {
 983         type => 'string',
 984         format => 'ipv4',
 985         format_description => 'GatewayIPv4',
 986         description => 'Default gateway for IPv4 traffic.',
 987         optional => 1,
 988         requires => 'ip',
 989     },
 990     ip6 => {
 991         type => 'string',
 992         format => 'pve-ipv6-config',
 993         format_description => 'IPv6Format/CIDR',
 994         description => 'IPv6 address in CIDR format.',
 995         optional => 1,
 996         default => 'dhcp',
 997     },
 998     gw6 => {
 999         type => 'string',
1000         format => 'ipv6',
1001         format_description => 'GatewayIPv6',
1002         description => 'Default gateway for IPv6 traffic.',
1003         optional => 1,
1004         requires => 'ip6',
1005     },
1006 };
1007 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1008 my $ipconfigdesc = {
1009     optional => 1,
1010     type => 'string', format => 'pve-qm-ipconfig',
1011     description => <<'EODESCR',
1012 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1013
1014 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1015
1016 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1017 gateway should be provided.
1018 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1019 cloud-init 19.4 or newer.
1020
1021 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1022 dhcp on IPv4.
1023 EODESCR
1024 };
1025 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1026
1027 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1028     $confdesc->{"net$i"} = $netdesc;
1029     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1030 }
1031
1032 foreach my $key (keys %$confdesc_cloudinit) {
1033     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1034 }
1035
1036 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1037 sub verify_volume_id_or_qm_path {
1038     my ($volid, $noerr) = @_;
1039
1040     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1041
1042     return verify_volume_id_or_absolute_path($volid, $noerr);
1043 }
1044
1045 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1046 sub verify_volume_id_or_absolute_path {
1047     my ($volid, $noerr) = @_;
1048
1049     return $volid if $volid =~ m|^/|;
1050
1051     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1052     if ($@) {
1053         return if $noerr;
1054         die $@;
1055     }
1056     return $volid;
1057 }
1058
1059 my $usb_fmt = {
1060     host => {
1061         default_key => 1,
1062         type => 'string', format => 'pve-qm-usb-device',
1063         format_description => 'HOSTUSBDEVICE|spice',
1064         description => <<EODESCR,
1065 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1066
1067  'bus-port(.port)*' (decimal numbers) or
1068  'vendor_id:product_id' (hexadeciaml numbers) or
1069  'spice'
1070
1071 You can use the 'lsusb -t' command to list existing usb devices.
1072
1073 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1074 machines - use with special care.
1075
1076 The value 'spice' can be used to add a usb redirection devices for spice.
1077 EODESCR
1078     },
1079     usb3 => {
1080         optional => 1,
1081         type => 'boolean',
1082         description => "Specifies whether if given host option is a USB3 device or port.",
1083         default => 0,
1084     },
1085 };
1086
1087 my $usbdesc = {
1088     optional => 1,
1089     type => 'string', format => $usb_fmt,
1090     description => "Configure an USB device (n is 0 to 4).",
1091 };
1092 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1093
1094 my $serialdesc = {
1095         optional => 1,
1096         type => 'string',
1097         pattern => '(/dev/.+|socket)',
1098         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1099         verbose_description =>  <<EODESCR,
1100 Create a serial device inside the VM (n is 0 to 3), and pass through a
1101 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1102 host side (use 'qm terminal' to open a terminal connection).
1103
1104 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1105 use with special care.
1106
1107 CAUTION: Experimental! User reported problems with this option.
1108 EODESCR
1109 };
1110
1111 my $paralleldesc= {
1112         optional => 1,
1113         type => 'string',
1114         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1115         description =>  "Map host parallel devices (n is 0 to 2).",
1116         verbose_description =>  <<EODESCR,
1117 Map host parallel devices (n is 0 to 2).
1118
1119 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1120 machines - use with special care.
1121
1122 CAUTION: Experimental! User reported problems with this option.
1123 EODESCR
1124 };
1125
1126 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1127     $confdesc->{"parallel$i"} = $paralleldesc;
1128 }
1129
1130 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1131     $confdesc->{"serial$i"} = $serialdesc;
1132 }
1133
1134 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1135     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1136 }
1137
1138 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1139     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1140 }
1141
1142 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1143     $confdesc->{"usb$i"} = $usbdesc;
1144 }
1145
1146 my $boot_fmt = {
1147     legacy => {
1148         optional => 1,
1149         default_key => 1,
1150         type => 'string',
1151         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1152                      . " Deprecated, use 'order=' instead.",
1153         pattern => '[acdn]{1,4}',
1154         format_description => "[acdn]{1,4}",
1155
1156         # note: this is also the fallback if boot: is not given at all
1157         default => 'cdn',
1158     },
1159     order => {
1160         optional => 1,
1161         type => 'string',
1162         format => 'pve-qm-bootdev-list',
1163         format_description => "device[;device...]",
1164         description => <<EODESC,
1165 The guest will attempt to boot from devices in the order they appear here.
1166
1167 Disks, optical drives and passed-through storage USB devices will be directly
1168 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1169 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1170
1171 Note that only devices in this list will be marked as bootable and thus loaded
1172 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1173 (e.g. software-raid), you need to specify all of them here.
1174
1175 Overrides the deprecated 'legacy=[acdn]*' value when given.
1176 EODESC
1177     },
1178 };
1179 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1180
1181 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1182 sub verify_bootdev {
1183     my ($dev, $noerr) = @_;
1184
1185     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1186     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1187
1188     my $check = sub {
1189         my ($base) = @_;
1190         return 0 if $dev !~ m/^$base\d+$/;
1191         return 0 if !$confdesc->{$dev};
1192         return 1;
1193     };
1194
1195     return $dev if $check->("net");
1196     return $dev if $check->("usb");
1197     return $dev if $check->("hostpci");
1198
1199     return if $noerr;
1200     die "invalid boot device '$dev'\n";
1201 }
1202
1203 sub print_bootorder {
1204     my ($devs) = @_;
1205     return "" if !@$devs;
1206     my $data = { order => join(';', @$devs) };
1207     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1208 }
1209
1210 my $kvm_api_version = 0;
1211
1212 sub kvm_version {
1213     return $kvm_api_version if $kvm_api_version;
1214
1215     open my $fh, '<', '/dev/kvm' or return;
1216
1217     # 0xae00 => KVM_GET_API_VERSION
1218     $kvm_api_version = ioctl($fh, 0xae00, 0);
1219     close($fh);
1220
1221     return $kvm_api_version;
1222 }
1223
1224 my $kvm_user_version = {};
1225 my $kvm_mtime = {};
1226
1227 sub kvm_user_version {
1228     my ($binary) = @_;
1229
1230     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1231     my $st = stat($binary);
1232
1233     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1234     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1235         $cachedmtime == $st->mtime;
1236
1237     $kvm_user_version->{$binary} = 'unknown';
1238     $kvm_mtime->{$binary} = $st->mtime;
1239
1240     my $code = sub {
1241         my $line = shift;
1242         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1243             $kvm_user_version->{$binary} = $2;
1244         }
1245     };
1246
1247     eval { run_command([$binary, '--version'], outfunc => $code); };
1248     warn $@ if $@;
1249
1250     return $kvm_user_version->{$binary};
1251
1252 }
1253 my sub extract_version {
1254     my ($machine_type, $version) = @_;
1255     $version = kvm_user_version() if !defined($version);
1256     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1257 }
1258
1259 sub kernel_has_vhost_net {
1260     return -c '/dev/vhost-net';
1261 }
1262
1263 sub option_exists {
1264     my $key = shift;
1265     return defined($confdesc->{$key});
1266 }
1267
1268 my $cdrom_path;
1269 sub get_cdrom_path {
1270
1271     return  $cdrom_path if $cdrom_path;
1272
1273     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1274     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1275     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1276 }
1277
1278 sub get_iso_path {
1279     my ($storecfg, $vmid, $cdrom) = @_;
1280
1281     if ($cdrom eq 'cdrom') {
1282         return get_cdrom_path();
1283     } elsif ($cdrom eq 'none') {
1284         return '';
1285     } elsif ($cdrom =~ m|^/|) {
1286         return $cdrom;
1287     } else {
1288         return PVE::Storage::path($storecfg, $cdrom);
1289     }
1290 }
1291
1292 # try to convert old style file names to volume IDs
1293 sub filename_to_volume_id {
1294     my ($vmid, $file, $media) = @_;
1295
1296      if (!($file eq 'none' || $file eq 'cdrom' ||
1297           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1298
1299         return if $file =~ m|/|;
1300
1301         if ($media && $media eq 'cdrom') {
1302             $file = "local:iso/$file";
1303         } else {
1304             $file = "local:$vmid/$file";
1305         }
1306     }
1307
1308     return $file;
1309 }
1310
1311 sub verify_media_type {
1312     my ($opt, $vtype, $media) = @_;
1313
1314     return if !$media;
1315
1316     my $etype;
1317     if ($media eq 'disk') {
1318         $etype = 'images';
1319     } elsif ($media eq 'cdrom') {
1320         $etype = 'iso';
1321     } else {
1322         die "internal error";
1323     }
1324
1325     return if ($vtype eq $etype);
1326
1327     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1328 }
1329
1330 sub cleanup_drive_path {
1331     my ($opt, $storecfg, $drive) = @_;
1332
1333     # try to convert filesystem paths to volume IDs
1334
1335     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1336         ($drive->{file} !~ m|^/dev/.+|) &&
1337         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1338         ($drive->{file} !~ m/^\d+$/)) {
1339         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1340         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1341             if !$vtype;
1342         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1343         verify_media_type($opt, $vtype, $drive->{media});
1344         $drive->{file} = $volid;
1345     }
1346
1347     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1348 }
1349
1350 sub parse_hotplug_features {
1351     my ($data) = @_;
1352
1353     my $res = {};
1354
1355     return $res if $data eq '0';
1356
1357     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1358
1359     foreach my $feature (PVE::Tools::split_list($data)) {
1360         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1361             $res->{$1} = 1;
1362         } else {
1363             die "invalid hotplug feature '$feature'\n";
1364         }
1365     }
1366     return $res;
1367 }
1368
1369 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1370 sub pve_verify_hotplug_features {
1371     my ($value, $noerr) = @_;
1372
1373     return $value if parse_hotplug_features($value);
1374
1375     return if $noerr;
1376
1377     die "unable to parse hotplug option\n";
1378 }
1379
1380 sub scsi_inquiry {
1381     my($fh, $noerr) = @_;
1382
1383     my $SG_IO = 0x2285;
1384     my $SG_GET_VERSION_NUM = 0x2282;
1385
1386     my $versionbuf = "\x00" x 8;
1387     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1388     if (!$ret) {
1389         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1390         return;
1391     }
1392     my $version = unpack("I", $versionbuf);
1393     if ($version < 30000) {
1394         die "scsi generic interface too old\n"  if !$noerr;
1395         return;
1396     }
1397
1398     my $buf = "\x00" x 36;
1399     my $sensebuf = "\x00" x 8;
1400     my $cmd = pack("C x3 C x1", 0x12, 36);
1401
1402     # see /usr/include/scsi/sg.h
1403     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1404
1405     my $packet = pack(
1406         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1407     );
1408
1409     $ret = ioctl($fh, $SG_IO, $packet);
1410     if (!$ret) {
1411         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1412         return;
1413     }
1414
1415     my @res = unpack($sg_io_hdr_t, $packet);
1416     if ($res[17] || $res[18]) {
1417         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1418         return;
1419     }
1420
1421     my $res = {};
1422     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1423
1424     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1425     $res->{type} &= 0x1F;
1426
1427     return $res;
1428 }
1429
1430 sub path_is_scsi {
1431     my ($path) = @_;
1432
1433     my $fh = IO::File->new("+<$path") || return;
1434     my $res = scsi_inquiry($fh, 1);
1435     close($fh);
1436
1437     return $res;
1438 }
1439
1440 sub print_tabletdevice_full {
1441     my ($conf, $arch) = @_;
1442
1443     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1444
1445     # we use uhci for old VMs because tablet driver was buggy in older qemu
1446     my $usbbus;
1447     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1448         $usbbus = 'ehci';
1449     } else {
1450         $usbbus = 'uhci';
1451     }
1452
1453     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1454 }
1455
1456 sub print_keyboarddevice_full {
1457     my ($conf, $arch) = @_;
1458
1459     return if $arch ne 'aarch64';
1460
1461     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1462 }
1463
1464 my sub get_drive_id {
1465     my ($drive) = @_;
1466     return "$drive->{interface}$drive->{index}";
1467 }
1468
1469 sub print_drivedevice_full {
1470     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1471
1472     my $device = '';
1473     my $maxdev = 0;
1474
1475     my $drive_id = get_drive_id($drive);
1476     if ($drive->{interface} eq 'virtio') {
1477         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1478         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1479         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1480     } elsif ($drive->{interface} eq 'scsi') {
1481
1482         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1483         my $unit = $drive->{index} % $maxdev;
1484         my $devicetype = 'hd';
1485         my $path = '';
1486         if (drive_is_cdrom($drive)) {
1487             $devicetype = 'cd';
1488         } else {
1489             if ($drive->{file} =~ m|^/|) {
1490                 $path = $drive->{file};
1491                 if (my $info = path_is_scsi($path)) {
1492                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1493                         $devicetype = 'block';
1494                     } elsif ($info->{type} == 1) { # tape
1495                         $devicetype = 'generic';
1496                     }
1497                 }
1498             } else {
1499                  $path = PVE::Storage::path($storecfg, $drive->{file});
1500             }
1501
1502             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1503             my $version = extract_version($machine_type, kvm_user_version());
1504             if ($path =~ m/^iscsi\:\/\// &&
1505                !min_version($version, 4, 1)) {
1506                 $devicetype = 'generic';
1507             }
1508         }
1509
1510         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1511             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1512         } else {
1513             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1514                 .",lun=$drive->{index}";
1515         }
1516         $device .= ",drive=drive-$drive_id,id=$drive_id";
1517
1518         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1519             $device .= ",rotation_rate=1";
1520         }
1521         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1522
1523     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1524         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1525         my $controller = int($drive->{index} / $maxdev);
1526         my $unit = $drive->{index} % $maxdev;
1527         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1528
1529         $device = "ide-$devicetype";
1530         if ($drive->{interface} eq 'ide') {
1531             $device .= ",bus=ide.$controller,unit=$unit";
1532         } else {
1533             $device .= ",bus=ahci$controller.$unit";
1534         }
1535         $device .= ",drive=drive-$drive_id,id=$drive_id";
1536
1537         if ($devicetype eq 'hd') {
1538             if (my $model = $drive->{model}) {
1539                 $model = URI::Escape::uri_unescape($model);
1540                 $device .= ",model=$model";
1541             }
1542             if ($drive->{ssd}) {
1543                 $device .= ",rotation_rate=1";
1544             }
1545         }
1546         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1547     } elsif ($drive->{interface} eq 'usb') {
1548         die "implement me";
1549         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1550     } else {
1551         die "unsupported interface type";
1552     }
1553
1554     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1555
1556     if (my $serial = $drive->{serial}) {
1557         $serial = URI::Escape::uri_unescape($serial);
1558         $device .= ",serial=$serial";
1559     }
1560
1561
1562     return $device;
1563 }
1564
1565 sub get_initiator_name {
1566     my $initiator;
1567
1568     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1569     while (defined(my $line = <$fh>)) {
1570         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1571         $initiator = $1;
1572         last;
1573     }
1574     $fh->close();
1575
1576     return $initiator;
1577 }
1578
1579 sub print_drive_commandline_full {
1580     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1581
1582     my $path;
1583     my $volid = $drive->{file};
1584     my $format = $drive->{format};
1585     my $drive_id = get_drive_id($drive);
1586
1587     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1588     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1589
1590     if (drive_is_cdrom($drive)) {
1591         $path = get_iso_path($storecfg, $vmid, $volid);
1592         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1593     } else {
1594         if ($storeid) {
1595             $path = PVE::Storage::path($storecfg, $volid);
1596             $format //= qemu_img_format($scfg, $volname);
1597         } else {
1598             $path = $volid;
1599             $format //= "raw";
1600         }
1601    }
1602
1603    my $is_rbd = $path =~ m/^rbd:/;
1604
1605     my $opts = '';
1606     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1607     foreach my $o (@qemu_drive_options) {
1608         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1609     }
1610
1611     # snapshot only accepts on|off
1612     if (defined($drive->{snapshot})) {
1613         my $v = $drive->{snapshot} ? 'on' : 'off';
1614         $opts .= ",snapshot=$v";
1615     }
1616
1617     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1618         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1619     }
1620
1621     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1622         my ($dir, $qmpname) = @$type;
1623         if (my $v = $drive->{"mbps$dir"}) {
1624             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1625         }
1626         if (my $v = $drive->{"mbps${dir}_max"}) {
1627             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1628         }
1629         if (my $v = $drive->{"bps${dir}_max_length"}) {
1630             $opts .= ",throttling.bps$qmpname-max-length=$v";
1631         }
1632         if (my $v = $drive->{"iops${dir}"}) {
1633             $opts .= ",throttling.iops$qmpname=$v";
1634         }
1635         if (my $v = $drive->{"iops${dir}_max"}) {
1636             $opts .= ",throttling.iops$qmpname-max=$v";
1637         }
1638         if (my $v = $drive->{"iops${dir}_max_length"}) {
1639             $opts .= ",throttling.iops$qmpname-max-length=$v";
1640         }
1641     }
1642
1643     if ($pbs_name) {
1644         $format = "rbd" if $is_rbd;
1645         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1646             if !$format;
1647         $opts .= ",format=alloc-track,file.driver=$format";
1648     } elsif ($format) {
1649         $opts .= ",format=$format";
1650     }
1651
1652     my $cache_direct = 0;
1653
1654     if (my $cache = $drive->{cache}) {
1655         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1656     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1657         $opts .= ",cache=none";
1658         $cache_direct = 1;
1659     }
1660
1661     # io_uring with cache mode writeback or writethrough on krbd will hang...
1662     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1663
1664     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1665     # sometimes, just plain disable...
1666     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1667
1668     if (!$drive->{aio}) {
1669         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1670             # io_uring supports all cache modes
1671             $opts .= ",aio=io_uring";
1672         } else {
1673             # aio native works only with O_DIRECT
1674             if($cache_direct) {
1675                 $opts .= ",aio=native";
1676             } else {
1677                 $opts .= ",aio=threads";
1678             }
1679         }
1680     }
1681
1682     if (!drive_is_cdrom($drive)) {
1683         my $detectzeroes;
1684         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1685             $detectzeroes = 'off';
1686         } elsif ($drive->{discard}) {
1687             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1688         } else {
1689             # This used to be our default with discard not being specified:
1690             $detectzeroes = 'on';
1691         }
1692
1693         # note: 'detect-zeroes' works per blockdev and we want it to persist
1694         # after the alloc-track is removed, so put it on 'file' directly
1695         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1696         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1697     }
1698
1699     if ($pbs_name) {
1700         $opts .= ",backing=$pbs_name";
1701         $opts .= ",auto-remove=on";
1702     }
1703
1704     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1705     my $file_param = "file";
1706     if ($pbs_name) {
1707         # non-rbd drivers require the underlying file to be a seperate block
1708         # node, so add a second .file indirection
1709         $file_param .= ".file" if !$is_rbd;
1710         $file_param .= ".filename";
1711     }
1712     my $pathinfo = $path ? "$file_param=$path," : '';
1713
1714     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1715 }
1716
1717 sub print_pbs_blockdev {
1718     my ($pbs_conf, $pbs_name) = @_;
1719     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1720     $blockdev .= ",repository=$pbs_conf->{repository}";
1721     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1722     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1723     $blockdev .= ",archive=$pbs_conf->{archive}";
1724     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1725     return $blockdev;
1726 }
1727
1728 sub print_netdevice_full {
1729     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1730
1731     my $device = $net->{model};
1732     if ($net->{model} eq 'virtio') {
1733          $device = 'virtio-net-pci';
1734      };
1735
1736     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1737     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1738     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1739         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1740         # and out of each queue plus one config interrupt and control vector queue
1741         my $vectors = $net->{queues} * 2 + 2;
1742         $tmpstr .= ",vectors=$vectors,mq=on";
1743     }
1744     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1745
1746     if (my $mtu = $net->{mtu}) {
1747         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1748             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1749             if ($mtu == 1) {
1750                  $mtu = $bridge_mtu;
1751             } elsif ($mtu < 576) {
1752                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1753             } elsif ($mtu > $bridge_mtu) {
1754                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1755             }
1756             $tmpstr .= ",host_mtu=$mtu";
1757         } else {
1758             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1759         }
1760     }
1761
1762     if ($use_old_bios_files) {
1763         my $romfile;
1764         if ($device eq 'virtio-net-pci') {
1765             $romfile = 'pxe-virtio.rom';
1766         } elsif ($device eq 'e1000') {
1767             $romfile = 'pxe-e1000.rom';
1768         } elsif ($device eq 'e1000e') {
1769             $romfile = 'pxe-e1000e.rom';
1770         } elsif ($device eq 'ne2k') {
1771             $romfile = 'pxe-ne2k_pci.rom';
1772         } elsif ($device eq 'pcnet') {
1773             $romfile = 'pxe-pcnet.rom';
1774         } elsif ($device eq 'rtl8139') {
1775             $romfile = 'pxe-rtl8139.rom';
1776         }
1777         $tmpstr .= ",romfile=$romfile" if $romfile;
1778     }
1779
1780     return $tmpstr;
1781 }
1782
1783 sub print_netdev_full {
1784     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1785
1786     my $i = '';
1787     if ($netid =~ m/^net(\d+)$/) {
1788         $i = int($1);
1789     }
1790
1791     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1792
1793     my $ifname = "tap${vmid}i$i";
1794
1795     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1796     die "interface name '$ifname' is too long (max 15 character)\n"
1797         if length($ifname) >= 16;
1798
1799     my $vhostparam = '';
1800     if (is_native($arch)) {
1801         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1802     }
1803
1804     my $vmname = $conf->{name} || "vm$vmid";
1805
1806     my $netdev = "";
1807     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1808
1809     if ($net->{bridge}) {
1810         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1811             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1812     } else {
1813         $netdev = "type=user,id=$netid,hostname=$vmname";
1814     }
1815
1816     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1817
1818     return $netdev;
1819 }
1820
1821 my $vga_map = {
1822     'cirrus' => 'cirrus-vga',
1823     'std' => 'VGA',
1824     'vmware' => 'vmware-svga',
1825     'virtio' => 'virtio-vga',
1826     'virtio-gl' => 'virtio-vga-gl',
1827 };
1828
1829 sub print_vga_device {
1830     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1831
1832     my $type = $vga_map->{$vga->{type}};
1833     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1834         $type = 'virtio-gpu';
1835     }
1836     my $vgamem_mb = $vga->{memory};
1837
1838     my $max_outputs = '';
1839     if ($qxlnum) {
1840         $type = $id ? 'qxl' : 'qxl-vga';
1841
1842         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1843             # set max outputs so linux can have up to 4 qxl displays with one device
1844             if (min_version($machine_version, 4, 1)) {
1845                 $max_outputs = ",max_outputs=4";
1846             }
1847         }
1848     }
1849
1850     die "no devicetype for $vga->{type}\n" if !$type;
1851
1852     my $memory = "";
1853     if ($vgamem_mb) {
1854         if ($vga->{type} =~ /^virtio/) {
1855             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1856             $memory = ",max_hostmem=$bytes";
1857         } elsif ($qxlnum) {
1858             # from https://www.spice-space.org/multiple-monitors.html
1859             $memory = ",vgamem_mb=$vga->{memory}";
1860             my $ram = $vgamem_mb * 4;
1861             my $vram = $vgamem_mb * 2;
1862             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1863         } else {
1864             $memory = ",vgamem_mb=$vga->{memory}";
1865         }
1866     } elsif ($qxlnum && $id) {
1867         $memory = ",ram_size=67108864,vram_size=33554432";
1868     }
1869
1870     my $edidoff = "";
1871     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1872         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1873     }
1874
1875     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1876     my $vgaid = "vga" . ($id // '');
1877     my $pciaddr;
1878     if ($q35 && $vgaid eq 'vga') {
1879         # the first display uses pcie.0 bus on q35 machines
1880         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1881     } else {
1882         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1883     }
1884
1885     if ($vga->{type} eq 'virtio-gl') {
1886         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1887         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1888             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1889
1890         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1891             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1892     }
1893
1894     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1895 }
1896
1897 sub parse_number_sets {
1898     my ($set) = @_;
1899     my $res = [];
1900     foreach my $part (split(/;/, $set)) {
1901         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1902             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1903             push @$res, [ $1, $2 ];
1904         } else {
1905             die "invalid range: $part\n";
1906         }
1907     }
1908     return $res;
1909 }
1910
1911 sub parse_numa {
1912     my ($data) = @_;
1913
1914     my $res = parse_property_string($numa_fmt, $data);
1915     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1916     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1917     return $res;
1918 }
1919
1920 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1921 sub parse_net {
1922     my ($data) = @_;
1923
1924     my $res = eval { parse_property_string($net_fmt, $data) };
1925     if ($@) {
1926         warn $@;
1927         return;
1928     }
1929     if (!defined($res->{macaddr})) {
1930         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1931         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1932     }
1933     return $res;
1934 }
1935
1936 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1937 sub parse_ipconfig {
1938     my ($data) = @_;
1939
1940     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1941     if ($@) {
1942         warn $@;
1943         return;
1944     }
1945
1946     if ($res->{gw} && !$res->{ip}) {
1947         warn 'gateway specified without specifying an IP address';
1948         return;
1949     }
1950     if ($res->{gw6} && !$res->{ip6}) {
1951         warn 'IPv6 gateway specified without specifying an IPv6 address';
1952         return;
1953     }
1954     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1955         warn 'gateway specified together with DHCP';
1956         return;
1957     }
1958     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1959         # gw6 + auto/dhcp
1960         warn "IPv6 gateway specified together with $res->{ip6} address";
1961         return;
1962     }
1963
1964     if (!$res->{ip} && !$res->{ip6}) {
1965         return { ip => 'dhcp', ip6 => 'dhcp' };
1966     }
1967
1968     return $res;
1969 }
1970
1971 sub print_net {
1972     my $net = shift;
1973
1974     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1975 }
1976
1977 sub add_random_macs {
1978     my ($settings) = @_;
1979
1980     foreach my $opt (keys %$settings) {
1981         next if $opt !~ m/^net(\d+)$/;
1982         my $net = parse_net($settings->{$opt});
1983         next if !$net;
1984         $settings->{$opt} = print_net($net);
1985     }
1986 }
1987
1988 sub vm_is_volid_owner {
1989     my ($storecfg, $vmid, $volid) = @_;
1990
1991     if ($volid !~  m|^/|) {
1992         my ($path, $owner);
1993         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1994         if ($owner && ($owner == $vmid)) {
1995             return 1;
1996         }
1997     }
1998
1999     return;
2000 }
2001
2002 sub vmconfig_register_unused_drive {
2003     my ($storecfg, $vmid, $conf, $drive) = @_;
2004
2005     if (drive_is_cloudinit($drive)) {
2006         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2007         warn $@ if $@;
2008     } elsif (!drive_is_cdrom($drive)) {
2009         my $volid = $drive->{file};
2010         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2011             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2012         }
2013     }
2014 }
2015
2016 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2017 my $smbios1_fmt = {
2018     uuid => {
2019         type => 'string',
2020         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2021         format_description => 'UUID',
2022         description => "Set SMBIOS1 UUID.",
2023         optional => 1,
2024     },
2025     version => {
2026         type => 'string',
2027         pattern => '[A-Za-z0-9+\/]+={0,2}',
2028         format_description => 'Base64 encoded string',
2029         description => "Set SMBIOS1 version.",
2030         optional => 1,
2031     },
2032     serial => {
2033         type => 'string',
2034         pattern => '[A-Za-z0-9+\/]+={0,2}',
2035         format_description => 'Base64 encoded string',
2036         description => "Set SMBIOS1 serial number.",
2037         optional => 1,
2038     },
2039     manufacturer => {
2040         type => 'string',
2041         pattern => '[A-Za-z0-9+\/]+={0,2}',
2042         format_description => 'Base64 encoded string',
2043         description => "Set SMBIOS1 manufacturer.",
2044         optional => 1,
2045     },
2046     product => {
2047         type => 'string',
2048         pattern => '[A-Za-z0-9+\/]+={0,2}',
2049         format_description => 'Base64 encoded string',
2050         description => "Set SMBIOS1 product ID.",
2051         optional => 1,
2052     },
2053     sku => {
2054         type => 'string',
2055         pattern => '[A-Za-z0-9+\/]+={0,2}',
2056         format_description => 'Base64 encoded string',
2057         description => "Set SMBIOS1 SKU string.",
2058         optional => 1,
2059     },
2060     family => {
2061         type => 'string',
2062         pattern => '[A-Za-z0-9+\/]+={0,2}',
2063         format_description => 'Base64 encoded string',
2064         description => "Set SMBIOS1 family string.",
2065         optional => 1,
2066     },
2067     base64 => {
2068         type => 'boolean',
2069         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2070         optional => 1,
2071     },
2072 };
2073
2074 sub parse_smbios1 {
2075     my ($data) = @_;
2076
2077     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2078     warn $@ if $@;
2079     return $res;
2080 }
2081
2082 sub print_smbios1 {
2083     my ($smbios1) = @_;
2084     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2085 }
2086
2087 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2088
2089 sub parse_watchdog {
2090     my ($value) = @_;
2091
2092     return if !$value;
2093
2094     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2095     warn $@ if $@;
2096     return $res;
2097 }
2098
2099 sub parse_guest_agent {
2100     my ($conf) = @_;
2101
2102     return {} if !defined($conf->{agent});
2103
2104     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2105     warn $@ if $@;
2106
2107     # if the agent is disabled ignore the other potentially set properties
2108     return {} if !$res->{enabled};
2109     return $res;
2110 }
2111
2112 sub get_qga_key {
2113     my ($conf, $key) = @_;
2114     return undef if !defined($conf->{agent});
2115
2116     my $agent = parse_guest_agent($conf);
2117     return $agent->{$key};
2118 }
2119
2120 sub parse_vga {
2121     my ($value) = @_;
2122
2123     return {} if !$value;
2124     my $res = eval { parse_property_string($vga_fmt, $value) };
2125     warn $@ if $@;
2126     return $res;
2127 }
2128
2129 sub parse_rng {
2130     my ($value) = @_;
2131
2132     return if !$value;
2133
2134     my $res = eval { parse_property_string($rng_fmt, $value) };
2135     warn $@ if $@;
2136     return $res;
2137 }
2138
2139 sub parse_meta_info {
2140     my ($value) = @_;
2141
2142     return if !$value;
2143
2144     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2145     warn $@ if $@;
2146     return $res;
2147 }
2148
2149 sub new_meta_info_string {
2150     my () = @_; # for now do not allow to override any value
2151
2152     return PVE::JSONSchema::print_property_string(
2153         {
2154             'creation-qemu' => kvm_user_version(),
2155             ctime => "". int(time()),
2156         },
2157         $meta_info_fmt
2158     );
2159 }
2160
2161 sub qemu_created_version_fixups {
2162     my ($conf, $forcemachine, $kvmver) = @_;
2163
2164     my $meta = parse_meta_info($conf->{meta}) // {};
2165     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2166
2167     # check if we need to apply some handling for VMs that always use the latest machine version but
2168     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2169     # would be required (we do not want to pin machine version for non-windows OS type)
2170     if (
2171         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2172         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2173         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2174         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2175     ) {
2176         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2177         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2178             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2179             # and thus with the predictable interface naming of systemd
2180             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2181         }
2182     }
2183     return;
2184 }
2185
2186 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2187 sub verify_usb_device {
2188     my ($value, $noerr) = @_;
2189
2190     return $value if parse_usb_device($value);
2191
2192     return if $noerr;
2193
2194     die "unable to parse usb device\n";
2195 }
2196
2197 # add JSON properties for create and set function
2198 sub json_config_properties {
2199     my ($prop, $with_disk_alloc) = @_;
2200
2201     my $skip_json_config_opts = {
2202         parent => 1,
2203         snaptime => 1,
2204         vmstate => 1,
2205         runningmachine => 1,
2206         runningcpu => 1,
2207         meta => 1,
2208     };
2209
2210     foreach my $opt (keys %$confdesc) {
2211         next if $skip_json_config_opts->{$opt};
2212
2213         if ($with_disk_alloc && is_valid_drivename($opt)) {
2214             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2215         } else {
2216             $prop->{$opt} = $confdesc->{$opt};
2217         }
2218     }
2219
2220     return $prop;
2221 }
2222
2223 # Properties that we can read from an OVF file
2224 sub json_ovf_properties {
2225     my $prop = {};
2226
2227     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2228         $prop->{$device} = {
2229             type => 'string',
2230             format => 'pve-volume-id-or-absolute-path',
2231             description => "Disk image that gets imported to $device",
2232             optional => 1,
2233         };
2234     }
2235
2236     $prop->{cores} = {
2237         type => 'integer',
2238         description => "The number of CPU cores.",
2239         optional => 1,
2240     };
2241     $prop->{memory} = {
2242         type => 'integer',
2243         description => "Amount of RAM for the VM in MB.",
2244         optional => 1,
2245     };
2246     $prop->{name} = {
2247         type => 'string',
2248         description => "Name of the VM.",
2249         optional => 1,
2250     };
2251
2252     return $prop;
2253 }
2254
2255 # return copy of $confdesc_cloudinit to generate documentation
2256 sub cloudinit_config_properties {
2257
2258     return dclone($confdesc_cloudinit);
2259 }
2260
2261 sub check_type {
2262     my ($key, $value) = @_;
2263
2264     die "unknown setting '$key'\n" if !$confdesc->{$key};
2265
2266     my $type = $confdesc->{$key}->{type};
2267
2268     if (!defined($value)) {
2269         die "got undefined value\n";
2270     }
2271
2272     if ($value =~ m/[\n\r]/) {
2273         die "property contains a line feed\n";
2274     }
2275
2276     if ($type eq 'boolean') {
2277         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2278         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2279         die "type check ('boolean') failed - got '$value'\n";
2280     } elsif ($type eq 'integer') {
2281         return int($1) if $value =~ m/^(\d+)$/;
2282         die "type check ('integer') failed - got '$value'\n";
2283     } elsif ($type eq 'number') {
2284         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2285         die "type check ('number') failed - got '$value'\n";
2286     } elsif ($type eq 'string') {
2287         if (my $fmt = $confdesc->{$key}->{format}) {
2288             PVE::JSONSchema::check_format($fmt, $value);
2289             return $value;
2290         }
2291         $value =~ s/^\"(.*)\"$/$1/;
2292         return $value;
2293     } else {
2294         die "internal error"
2295     }
2296 }
2297
2298 sub destroy_vm {
2299     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2300
2301     my $conf = PVE::QemuConfig->load_config($vmid);
2302
2303     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2304
2305     if ($conf->{template}) {
2306         # check if any base image is still used by a linked clone
2307         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2308                 my ($ds, $drive) = @_;
2309                 return if drive_is_cdrom($drive);
2310
2311                 my $volid = $drive->{file};
2312                 return if !$volid || $volid =~ m|^/|;
2313
2314                 die "base volume '$volid' is still in use by linked cloned\n"
2315                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2316
2317         });
2318     }
2319
2320     my $volids = {};
2321     my $remove_owned_drive = sub {
2322         my ($ds, $drive) = @_;
2323         return if drive_is_cdrom($drive, 1);
2324
2325         my $volid = $drive->{file};
2326         return if !$volid || $volid =~ m|^/|;
2327         return if $volids->{$volid};
2328
2329         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2330         return if !$path || !$owner || ($owner != $vmid);
2331
2332         $volids->{$volid} = 1;
2333         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2334         warn "Could not remove disk '$volid', check manually: $@" if $@;
2335     };
2336
2337     # only remove disks owned by this VM (referenced in the config)
2338     my $include_opts = {
2339         include_unused => 1,
2340         extra_keys => ['vmstate'],
2341     };
2342     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2343
2344     for my $snap (values %{$conf->{snapshots}}) {
2345         next if !defined($snap->{vmstate});
2346         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2347         next if !defined($drive);
2348         $remove_owned_drive->('vmstate', $drive);
2349     }
2350
2351     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2352
2353     if ($purge_unreferenced) { # also remove unreferenced disk
2354         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2355         PVE::Storage::foreach_volid($vmdisks, sub {
2356             my ($volid, $sid, $volname, $d) = @_;
2357             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2358             warn $@ if $@;
2359         });
2360     }
2361
2362     if (defined $replacement_conf) {
2363         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2364     } else {
2365         PVE::QemuConfig->destroy_config($vmid);
2366     }
2367 }
2368
2369 sub parse_vm_config {
2370     my ($filename, $raw, $strict) = @_;
2371
2372     return if !defined($raw);
2373
2374     my $res = {
2375         digest => Digest::SHA::sha1_hex($raw),
2376         snapshots => {},
2377         pending => {},
2378     };
2379
2380     my $handle_error = sub {
2381         my ($msg) = @_;
2382
2383         if ($strict) {
2384             die $msg;
2385         } else {
2386             warn $msg;
2387         }
2388     };
2389
2390     $filename =~ m|/qemu-server/(\d+)\.conf$|
2391         || die "got strange filename '$filename'";
2392
2393     my $vmid = $1;
2394
2395     my $conf = $res;
2396     my $descr;
2397     my $section = '';
2398
2399     my @lines = split(/\n/, $raw);
2400     foreach my $line (@lines) {
2401         next if $line =~ m/^\s*$/;
2402
2403         if ($line =~ m/^\[PENDING\]\s*$/i) {
2404             $section = 'pending';
2405             if (defined($descr)) {
2406                 $descr =~ s/\s+$//;
2407                 $conf->{description} = $descr;
2408             }
2409             $descr = undef;
2410             $conf = $res->{$section} = {};
2411             next;
2412
2413         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2414             $section = $1;
2415             if (defined($descr)) {
2416                 $descr =~ s/\s+$//;
2417                 $conf->{description} = $descr;
2418             }
2419             $descr = undef;
2420             $conf = $res->{snapshots}->{$section} = {};
2421             next;
2422         }
2423
2424         if ($line =~ m/^\#(.*)$/) {
2425             $descr = '' if !defined($descr);
2426             $descr .= PVE::Tools::decode_text($1) . "\n";
2427             next;
2428         }
2429
2430         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2431             $descr = '' if !defined($descr);
2432             $descr .= PVE::Tools::decode_text($2);
2433         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2434             $conf->{snapstate} = $1;
2435         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2436             my $key = $1;
2437             my $value = $2;
2438             $conf->{$key} = $value;
2439         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2440             my $value = $1;
2441             if ($section eq 'pending') {
2442                 $conf->{delete} = $value; # we parse this later
2443             } else {
2444                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2445             }
2446         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2447             my $key = $1;
2448             my $value = $2;
2449             eval { $value = check_type($key, $value); };
2450             if ($@) {
2451                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2452             } else {
2453                 $key = 'ide2' if $key eq 'cdrom';
2454                 my $fmt = $confdesc->{$key}->{format};
2455                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2456                     my $v = parse_drive($key, $value);
2457                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2458                         $v->{file} = $volid;
2459                         $value = print_drive($v);
2460                     } else {
2461                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2462                         next;
2463                     }
2464                 }
2465
2466                 $conf->{$key} = $value;
2467             }
2468         } else {
2469             $handle_error->("vm $vmid - unable to parse config: $line\n");
2470         }
2471     }
2472
2473     if (defined($descr)) {
2474         $descr =~ s/\s+$//;
2475         $conf->{description} = $descr;
2476     }
2477     delete $res->{snapstate}; # just to be sure
2478
2479     return $res;
2480 }
2481
2482 sub write_vm_config {
2483     my ($filename, $conf) = @_;
2484
2485     delete $conf->{snapstate}; # just to be sure
2486
2487     if ($conf->{cdrom}) {
2488         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2489         $conf->{ide2} = $conf->{cdrom};
2490         delete $conf->{cdrom};
2491     }
2492
2493     # we do not use 'smp' any longer
2494     if ($conf->{sockets}) {
2495         delete $conf->{smp};
2496     } elsif ($conf->{smp}) {
2497         $conf->{sockets} = $conf->{smp};
2498         delete $conf->{cores};
2499         delete $conf->{smp};
2500     }
2501
2502     my $used_volids = {};
2503
2504     my $cleanup_config = sub {
2505         my ($cref, $pending, $snapname) = @_;
2506
2507         foreach my $key (keys %$cref) {
2508             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2509                 $key eq 'snapstate' || $key eq 'pending';
2510             my $value = $cref->{$key};
2511             if ($key eq 'delete') {
2512                 die "propertry 'delete' is only allowed in [PENDING]\n"
2513                     if !$pending;
2514                 # fixme: check syntax?
2515                 next;
2516             }
2517             eval { $value = check_type($key, $value); };
2518             die "unable to parse value of '$key' - $@" if $@;
2519
2520             $cref->{$key} = $value;
2521
2522             if (!$snapname && is_valid_drivename($key)) {
2523                 my $drive = parse_drive($key, $value);
2524                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2525             }
2526         }
2527     };
2528
2529     &$cleanup_config($conf);
2530
2531     &$cleanup_config($conf->{pending}, 1);
2532
2533     foreach my $snapname (keys %{$conf->{snapshots}}) {
2534         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2535         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2536     }
2537
2538     # remove 'unusedX' settings if we re-add a volume
2539     foreach my $key (keys %$conf) {
2540         my $value = $conf->{$key};
2541         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2542             delete $conf->{$key};
2543         }
2544     }
2545
2546     my $generate_raw_config = sub {
2547         my ($conf, $pending) = @_;
2548
2549         my $raw = '';
2550
2551         # add description as comment to top of file
2552         if (defined(my $descr = $conf->{description})) {
2553             if ($descr) {
2554                 foreach my $cl (split(/\n/, $descr)) {
2555                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2556                 }
2557             } else {
2558                 $raw .= "#\n" if $pending;
2559             }
2560         }
2561
2562         foreach my $key (sort keys %$conf) {
2563             next if $key =~ /^(digest|description|pending|snapshots)$/;
2564             $raw .= "$key: $conf->{$key}\n";
2565         }
2566         return $raw;
2567     };
2568
2569     my $raw = &$generate_raw_config($conf);
2570
2571     if (scalar(keys %{$conf->{pending}})){
2572         $raw .= "\n[PENDING]\n";
2573         $raw .= &$generate_raw_config($conf->{pending}, 1);
2574     }
2575
2576     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2577         $raw .= "\n[$snapname]\n";
2578         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2579     }
2580
2581     return $raw;
2582 }
2583
2584 sub load_defaults {
2585
2586     my $res = {};
2587
2588     # we use static defaults from our JSON schema configuration
2589     foreach my $key (keys %$confdesc) {
2590         if (defined(my $default = $confdesc->{$key}->{default})) {
2591             $res->{$key} = $default;
2592         }
2593     }
2594
2595     return $res;
2596 }
2597
2598 sub config_list {
2599     my $vmlist = PVE::Cluster::get_vmlist();
2600     my $res = {};
2601     return $res if !$vmlist || !$vmlist->{ids};
2602     my $ids = $vmlist->{ids};
2603     my $nodename = nodename();
2604
2605     foreach my $vmid (keys %$ids) {
2606         my $d = $ids->{$vmid};
2607         next if !$d->{node} || $d->{node} ne $nodename;
2608         next if !$d->{type} || $d->{type} ne 'qemu';
2609         $res->{$vmid}->{exists} = 1;
2610     }
2611     return $res;
2612 }
2613
2614 # test if VM uses local resources (to prevent migration)
2615 sub check_local_resources {
2616     my ($conf, $noerr) = @_;
2617
2618     my @loc_res = ();
2619
2620     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2621     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2622
2623     push @loc_res, "ivshmem" if $conf->{ivshmem};
2624
2625     foreach my $k (keys %$conf) {
2626         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2627         # sockets are safe: they will recreated be on the target side post-migrate
2628         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2629         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2630     }
2631
2632     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2633
2634     return \@loc_res;
2635 }
2636
2637 # check if used storages are available on all nodes (use by migrate)
2638 sub check_storage_availability {
2639     my ($storecfg, $conf, $node) = @_;
2640
2641     PVE::QemuConfig->foreach_volume($conf, sub {
2642         my ($ds, $drive) = @_;
2643
2644         my $volid = $drive->{file};
2645         return if !$volid;
2646
2647         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2648         return if !$sid;
2649
2650         # check if storage is available on both nodes
2651         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2652         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2653
2654         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2655
2656         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2657             if !$scfg->{content}->{$vtype};
2658    });
2659 }
2660
2661 # list nodes where all VM images are available (used by has_feature API)
2662 sub shared_nodes {
2663     my ($conf, $storecfg) = @_;
2664
2665     my $nodelist = PVE::Cluster::get_nodelist();
2666     my $nodehash = { map { $_ => 1 } @$nodelist };
2667     my $nodename = nodename();
2668
2669     PVE::QemuConfig->foreach_volume($conf, sub {
2670         my ($ds, $drive) = @_;
2671
2672         my $volid = $drive->{file};
2673         return if !$volid;
2674
2675         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2676         if ($storeid) {
2677             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2678             if ($scfg->{disable}) {
2679                 $nodehash = {};
2680             } elsif (my $avail = $scfg->{nodes}) {
2681                 foreach my $node (keys %$nodehash) {
2682                     delete $nodehash->{$node} if !$avail->{$node};
2683                 }
2684             } elsif (!$scfg->{shared}) {
2685                 foreach my $node (keys %$nodehash) {
2686                     delete $nodehash->{$node} if $node ne $nodename
2687                 }
2688             }
2689         }
2690     });
2691
2692     return $nodehash
2693 }
2694
2695 sub check_local_storage_availability {
2696     my ($conf, $storecfg) = @_;
2697
2698     my $nodelist = PVE::Cluster::get_nodelist();
2699     my $nodehash = { map { $_ => {} } @$nodelist };
2700
2701     PVE::QemuConfig->foreach_volume($conf, sub {
2702         my ($ds, $drive) = @_;
2703
2704         my $volid = $drive->{file};
2705         return if !$volid;
2706
2707         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2708         if ($storeid) {
2709             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2710
2711             if ($scfg->{disable}) {
2712                 foreach my $node (keys %$nodehash) {
2713                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2714                 }
2715             } elsif (my $avail = $scfg->{nodes}) {
2716                 foreach my $node (keys %$nodehash) {
2717                     if (!$avail->{$node}) {
2718                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2719                     }
2720                 }
2721             }
2722         }
2723     });
2724
2725     foreach my $node (values %$nodehash) {
2726         if (my $unavail = $node->{unavailable_storages}) {
2727             $node->{unavailable_storages} = [ sort keys %$unavail ];
2728         }
2729     }
2730
2731     return $nodehash
2732 }
2733
2734 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2735 sub check_running {
2736     my ($vmid, $nocheck, $node) = @_;
2737
2738     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2739     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2740 }
2741
2742 sub vzlist {
2743
2744     my $vzlist = config_list();
2745
2746     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2747
2748     while (defined(my $de = $fd->read)) {
2749         next if $de !~ m/^(\d+)\.pid$/;
2750         my $vmid = $1;
2751         next if !defined($vzlist->{$vmid});
2752         if (my $pid = check_running($vmid)) {
2753             $vzlist->{$vmid}->{pid} = $pid;
2754         }
2755     }
2756
2757     return $vzlist;
2758 }
2759
2760 our $vmstatus_return_properties = {
2761     vmid => get_standard_option('pve-vmid'),
2762     status => {
2763         description => "Qemu process status.",
2764         type => 'string',
2765         enum => ['stopped', 'running'],
2766     },
2767     maxmem => {
2768         description => "Maximum memory in bytes.",
2769         type => 'integer',
2770         optional => 1,
2771         renderer => 'bytes',
2772     },
2773     maxdisk => {
2774         description => "Root disk size in bytes.",
2775         type => 'integer',
2776         optional => 1,
2777         renderer => 'bytes',
2778     },
2779     name => {
2780         description => "VM name.",
2781         type => 'string',
2782         optional => 1,
2783     },
2784     qmpstatus => {
2785         description => "Qemu QMP agent status.",
2786         type => 'string',
2787         optional => 1,
2788     },
2789     pid => {
2790         description => "PID of running qemu process.",
2791         type => 'integer',
2792         optional => 1,
2793     },
2794     uptime => {
2795         description => "Uptime.",
2796         type => 'integer',
2797         optional => 1,
2798         renderer => 'duration',
2799     },
2800     cpus => {
2801         description => "Maximum usable CPUs.",
2802         type => 'number',
2803         optional => 1,
2804     },
2805     lock => {
2806         description => "The current config lock, if any.",
2807         type => 'string',
2808         optional => 1,
2809     },
2810     tags => {
2811         description  => "The current configured tags, if any",
2812         type => 'string',
2813         optional => 1,
2814     },
2815     'running-machine' => {
2816         description => "The currently running machine type (if running).",
2817         type => 'string',
2818         optional => 1,
2819     },
2820     'running-qemu' => {
2821         description => "The currently running QEMU version (if running).",
2822         type => 'string',
2823         optional => 1,
2824     },
2825 };
2826
2827 my $last_proc_pid_stat;
2828
2829 # get VM status information
2830 # This must be fast and should not block ($full == false)
2831 # We only query KVM using QMP if $full == true (this can be slow)
2832 sub vmstatus {
2833     my ($opt_vmid, $full) = @_;
2834
2835     my $res = {};
2836
2837     my $storecfg = PVE::Storage::config();
2838
2839     my $list = vzlist();
2840     my $defaults = load_defaults();
2841
2842     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2843
2844     my $cpucount = $cpuinfo->{cpus} || 1;
2845
2846     foreach my $vmid (keys %$list) {
2847         next if $opt_vmid && ($vmid ne $opt_vmid);
2848
2849         my $conf = PVE::QemuConfig->load_config($vmid);
2850
2851         my $d = { vmid => int($vmid) };
2852         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2853
2854         # fixme: better status?
2855         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2856
2857         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2858         if (defined($size)) {
2859             $d->{disk} = 0; # no info available
2860             $d->{maxdisk} = $size;
2861         } else {
2862             $d->{disk} = 0;
2863             $d->{maxdisk} = 0;
2864         }
2865
2866         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2867             * ($conf->{cores} || $defaults->{cores});
2868         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2869         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2870
2871         $d->{name} = $conf->{name} || "VM $vmid";
2872         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2873             : $defaults->{memory}*(1024*1024);
2874
2875         if ($conf->{balloon}) {
2876             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2877             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2878                 : $defaults->{shares};
2879         }
2880
2881         $d->{uptime} = 0;
2882         $d->{cpu} = 0;
2883         $d->{mem} = 0;
2884
2885         $d->{netout} = 0;
2886         $d->{netin} = 0;
2887
2888         $d->{diskread} = 0;
2889         $d->{diskwrite} = 0;
2890
2891         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2892
2893         $d->{serial} = 1 if conf_has_serial($conf);
2894         $d->{lock} = $conf->{lock} if $conf->{lock};
2895         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2896
2897         $res->{$vmid} = $d;
2898     }
2899
2900     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2901     foreach my $dev (keys %$netdev) {
2902         next if $dev !~ m/^tap([1-9]\d*)i/;
2903         my $vmid = $1;
2904         my $d = $res->{$vmid};
2905         next if !$d;
2906
2907         $d->{netout} += $netdev->{$dev}->{receive};
2908         $d->{netin} += $netdev->{$dev}->{transmit};
2909
2910         if ($full) {
2911             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2912             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2913         }
2914
2915     }
2916
2917     my $ctime = gettimeofday;
2918
2919     foreach my $vmid (keys %$list) {
2920
2921         my $d = $res->{$vmid};
2922         my $pid = $d->{pid};
2923         next if !$pid;
2924
2925         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2926         next if !$pstat; # not running
2927
2928         my $used = $pstat->{utime} + $pstat->{stime};
2929
2930         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2931
2932         if ($pstat->{vsize}) {
2933             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2934         }
2935
2936         my $old = $last_proc_pid_stat->{$pid};
2937         if (!$old) {
2938             $last_proc_pid_stat->{$pid} = {
2939                 time => $ctime,
2940                 used => $used,
2941                 cpu => 0,
2942             };
2943             next;
2944         }
2945
2946         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2947
2948         if ($dtime > 1000) {
2949             my $dutime = $used -  $old->{used};
2950
2951             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2952             $last_proc_pid_stat->{$pid} = {
2953                 time => $ctime,
2954                 used => $used,
2955                 cpu => $d->{cpu},
2956             };
2957         } else {
2958             $d->{cpu} = $old->{cpu};
2959         }
2960     }
2961
2962     return $res if !$full;
2963
2964     my $qmpclient = PVE::QMPClient->new();
2965
2966     my $ballooncb = sub {
2967         my ($vmid, $resp) = @_;
2968
2969         my $info = $resp->{'return'};
2970         return if !$info->{max_mem};
2971
2972         my $d = $res->{$vmid};
2973
2974         # use memory assigned to VM
2975         $d->{maxmem} = $info->{max_mem};
2976         $d->{balloon} = $info->{actual};
2977
2978         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2979             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2980             $d->{freemem} = $info->{free_mem};
2981         }
2982
2983         $d->{ballooninfo} = $info;
2984     };
2985
2986     my $blockstatscb = sub {
2987         my ($vmid, $resp) = @_;
2988         my $data = $resp->{'return'} || [];
2989         my $totalrdbytes = 0;
2990         my $totalwrbytes = 0;
2991
2992         for my $blockstat (@$data) {
2993             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2994             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2995
2996             $blockstat->{device} =~ s/drive-//;
2997             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2998         }
2999         $res->{$vmid}->{diskread} = $totalrdbytes;
3000         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3001     };
3002
3003     my $machinecb = sub {
3004         my ($vmid, $resp) = @_;
3005         my $data = $resp->{'return'} || [];
3006
3007         $res->{$vmid}->{'running-machine'} =
3008             PVE::QemuServer::Machine::current_from_query_machines($data);
3009     };
3010
3011     my $versioncb = sub {
3012         my ($vmid, $resp) = @_;
3013         my $data = $resp->{'return'} // {};
3014         my $version = 'unknown';
3015
3016         if (my $v = $data->{qemu}) {
3017             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3018         }
3019
3020         $res->{$vmid}->{'running-qemu'} = $version;
3021     };
3022
3023     my $statuscb = sub {
3024         my ($vmid, $resp) = @_;
3025
3026         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3027         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3028         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3029         # this fails if ballon driver is not loaded, so this must be
3030         # the last commnand (following command are aborted if this fails).
3031         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3032
3033         my $status = 'unknown';
3034         if (!defined($status = $resp->{'return'}->{status})) {
3035             warn "unable to get VM status\n";
3036             return;
3037         }
3038
3039         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3040     };
3041
3042     foreach my $vmid (keys %$list) {
3043         next if $opt_vmid && ($vmid ne $opt_vmid);
3044         next if !$res->{$vmid}->{pid}; # not running
3045         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3046     }
3047
3048     $qmpclient->queue_execute(undef, 2);
3049
3050     foreach my $vmid (keys %$list) {
3051         next if $opt_vmid && ($vmid ne $opt_vmid);
3052         next if !$res->{$vmid}->{pid}; #not running
3053
3054         # we can't use the $qmpclient since it might have already aborted on
3055         # 'query-balloon', but this might also fail for older versions...
3056         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3057         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3058     }
3059
3060     foreach my $vmid (keys %$list) {
3061         next if $opt_vmid && ($vmid ne $opt_vmid);
3062         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3063     }
3064
3065     return $res;
3066 }
3067
3068 sub conf_has_serial {
3069     my ($conf) = @_;
3070
3071     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3072         if ($conf->{"serial$i"}) {
3073             return 1;
3074         }
3075     }
3076
3077     return 0;
3078 }
3079
3080 sub conf_has_audio {
3081     my ($conf, $id) = @_;
3082
3083     $id //= 0;
3084     my $audio = $conf->{"audio$id"};
3085     return if !defined($audio);
3086
3087     my $audioproperties = parse_property_string($audio_fmt, $audio);
3088     my $audiodriver = $audioproperties->{driver} // 'spice';
3089
3090     return {
3091         dev => $audioproperties->{device},
3092         dev_id => "audiodev$id",
3093         backend => $audiodriver,
3094         backend_id => "$audiodriver-backend${id}",
3095     };
3096 }
3097
3098 sub audio_devs {
3099     my ($audio, $audiopciaddr, $machine_version) = @_;
3100
3101     my $devs = [];
3102
3103     my $id = $audio->{dev_id};
3104     my $audiodev = "";
3105     if (min_version($machine_version, 4, 2)) {
3106         $audiodev = ",audiodev=$audio->{backend_id}";
3107     }
3108
3109     if ($audio->{dev} eq 'AC97') {
3110         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3111     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3112         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3113         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3114         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3115     } else {
3116         die "unkown audio device '$audio->{dev}', implement me!";
3117     }
3118
3119     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3120
3121     return $devs;
3122 }
3123
3124 sub get_tpm_paths {
3125     my ($vmid) = @_;
3126     return {
3127         socket => "/var/run/qemu-server/$vmid.swtpm",
3128         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3129     };
3130 }
3131
3132 sub add_tpm_device {
3133     my ($vmid, $devices, $conf) = @_;
3134
3135     return if !$conf->{tpmstate0};
3136
3137     my $paths = get_tpm_paths($vmid);
3138
3139     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3140     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3141     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3142 }
3143
3144 sub start_swtpm {
3145     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3146
3147     return if !$tpmdrive;
3148
3149     my $state;
3150     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3151     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3152     if ($storeid) {
3153         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3154     } else {
3155         $state = $tpm->{file};
3156     }
3157
3158     my $paths = get_tpm_paths($vmid);
3159
3160     # during migration, we will get state from remote
3161     #
3162     if (!$migration) {
3163         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3164         my $setup_cmd = [
3165             "swtpm_setup",
3166             "--tpmstate",
3167             "file://$state",
3168             "--createek",
3169             "--create-ek-cert",
3170             "--create-platform-cert",
3171             "--lock-nvram",
3172             "--config",
3173             "/etc/swtpm_setup.conf", # do not use XDG configs
3174             "--runas",
3175             "0", # force creation as root, error if not possible
3176             "--not-overwrite", # ignore existing state, do not modify
3177         ];
3178
3179         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3180         # TPM 2.0 supports ECC crypto, use if possible
3181         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3182
3183         run_command($setup_cmd, outfunc => sub {
3184             print "swtpm_setup: $1\n";
3185         });
3186     }
3187
3188     my $emulator_cmd = [
3189         "swtpm",
3190         "socket",
3191         "--tpmstate",
3192         "backend-uri=file://$state,mode=0600",
3193         "--ctrl",
3194         "type=unixio,path=$paths->{socket},mode=0600",
3195         "--pid",
3196         "file=$paths->{pid}",
3197         "--terminate", # terminate on QEMU disconnect
3198         "--daemon",
3199     ];
3200     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3201     run_command($emulator_cmd, outfunc => sub { print $1; });
3202
3203     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3204     while (! -e $paths->{pid}) {
3205         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3206         usleep(50_000);
3207     }
3208
3209     # return untainted PID of swtpm daemon so it can be killed on error
3210     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3211     return $1;
3212 }
3213
3214 sub vga_conf_has_spice {
3215     my ($vga) = @_;
3216
3217     my $vgaconf = parse_vga($vga);
3218     my $vgatype = $vgaconf->{type};
3219     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3220
3221     return $1 || 1;
3222 }
3223
3224 sub is_native($) {
3225     my ($arch) = @_;
3226     return get_host_arch() eq $arch;
3227 }
3228
3229 sub get_vm_arch {
3230     my ($conf) = @_;
3231     return $conf->{arch} // get_host_arch();
3232 }
3233
3234 my $default_machines = {
3235     x86_64 => 'pc',
3236     aarch64 => 'virt',
3237 };
3238
3239 sub get_installed_machine_version {
3240     my ($kvmversion) = @_;
3241     $kvmversion = kvm_user_version() if !defined($kvmversion);
3242     $kvmversion =~ m/^(\d+\.\d+)/;
3243     return $1;
3244 }
3245
3246 sub windows_get_pinned_machine_version {
3247     my ($machine, $base_version, $kvmversion) = @_;
3248
3249     my $pin_version = $base_version;
3250     if (!defined($base_version) ||
3251         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3252     ) {
3253         $pin_version = get_installed_machine_version($kvmversion);
3254     }
3255     if (!$machine || $machine eq 'pc') {
3256         $machine = "pc-i440fx-$pin_version";
3257     } elsif ($machine eq 'q35') {
3258         $machine = "pc-q35-$pin_version";
3259     } elsif ($machine eq 'virt') {
3260         $machine = "virt-$pin_version";
3261     } else {
3262         warn "unknown machine type '$machine', not touching that!\n";
3263     }
3264
3265     return $machine;
3266 }
3267
3268 sub get_vm_machine {
3269     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3270
3271     my $machine = $forcemachine || $conf->{machine};
3272
3273     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3274         $kvmversion //= kvm_user_version();
3275         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3276         # layout which confuses windows quite a bit and may result in various regressions..
3277         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3278         if (windows_version($conf->{ostype})) {
3279             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3280         }
3281         $arch //= 'x86_64';
3282         $machine ||= $default_machines->{$arch};
3283         if ($add_pve_version) {
3284             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3285             $machine .= "+pve$pvever";
3286         }
3287     }
3288
3289     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3290         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3291         $machine = $1 if $is_pxe;
3292
3293         # for version-pinned machines that do not include a pve-version (e.g.
3294         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3295         $machine .= '+pve0';
3296
3297         $machine .= '.pxe' if $is_pxe;
3298     }
3299
3300     return $machine;
3301 }
3302
3303 sub get_ovmf_files($$$) {
3304     my ($arch, $efidisk, $smm) = @_;
3305
3306     my $types = $OVMF->{$arch}
3307         or die "no OVMF images known for architecture '$arch'\n";
3308
3309     my $type = 'default';
3310     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3311         $type = $smm ? "4m" : "4m-no-smm";
3312         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3313     }
3314
3315     return $types->{$type}->@*;
3316 }
3317
3318 my $Arch2Qemu = {
3319     aarch64 => '/usr/bin/qemu-system-aarch64',
3320     x86_64 => '/usr/bin/qemu-system-x86_64',
3321 };
3322 sub get_command_for_arch($) {
3323     my ($arch) = @_;
3324     return '/usr/bin/kvm' if is_native($arch);
3325
3326     my $cmd = $Arch2Qemu->{$arch}
3327         or die "don't know how to emulate architecture '$arch'\n";
3328     return $cmd;
3329 }
3330
3331 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3332 # to use in a QEMU command line (-cpu element), first array_intersect the result
3333 # of query_supported_ with query_understood_. This is necessary because:
3334 #
3335 # a) query_understood_ returns flags the host cannot use and
3336 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3337 #    flags, but CPU settings - with most of them being flags. Those settings
3338 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3339 #
3340 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3341 # expensive. If you need the value returned from this, you can get it much
3342 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3343 # $accel being 'kvm' or 'tcg'.
3344 #
3345 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3346 # changes, automatically populating pmxcfs.
3347 #
3348 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3349 # since kvm and tcg machines support different flags
3350 #
3351 sub query_supported_cpu_flags {
3352     my ($arch) = @_;
3353
3354     $arch //= get_host_arch();
3355     my $default_machine = $default_machines->{$arch};
3356
3357     my $flags = {};
3358
3359     # FIXME: Once this is merged, the code below should work for ARM as well:
3360     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3361     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3362         $arch eq "aarch64";
3363
3364     my $kvm_supported = defined(kvm_version());
3365     my $qemu_cmd = get_command_for_arch($arch);
3366     my $fakevmid = -1;
3367     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3368
3369     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3370     my $query_supported_run_qemu = sub {
3371         my ($kvm) = @_;
3372
3373         my $flags = {};
3374         my $cmd = [
3375             $qemu_cmd,
3376             '-machine', $default_machine,
3377             '-display', 'none',
3378             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3379             '-mon', 'chardev=qmp,mode=control',
3380             '-pidfile', $pidfile,
3381             '-S', '-daemonize'
3382         ];
3383
3384         if (!$kvm) {
3385             push @$cmd, '-accel', 'tcg';
3386         }
3387
3388         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3389         die "QEMU flag querying VM exited with code " . $rc if $rc;
3390
3391         eval {
3392             my $cmd_result = mon_cmd(
3393                 $fakevmid,
3394                 'query-cpu-model-expansion',
3395                 type => 'full',
3396                 model => { name => 'host' }
3397             );
3398
3399             my $props = $cmd_result->{model}->{props};
3400             foreach my $prop (keys %$props) {
3401                 next if $props->{$prop} ne '1';
3402                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3403                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3404                 # We only keep those with underscores, to match /proc/cpuinfo
3405                 $prop =~ s/\.|-/_/g;
3406                 $flags->{$prop} = 1;
3407             }
3408         };
3409         my $err = $@;
3410
3411         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3412         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3413
3414         die $err if $err;
3415
3416         return [ sort keys %$flags ];
3417     };
3418
3419     # We need to query QEMU twice, since KVM and TCG have different supported flags
3420     PVE::QemuConfig->lock_config($fakevmid, sub {
3421         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3422         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3423
3424         if ($kvm_supported) {
3425             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3426             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3427         }
3428     });
3429
3430     return $flags;
3431 }
3432
3433 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3434 my $understood_cpu_flag_dir = "/usr/share/kvm";
3435 sub query_understood_cpu_flags {
3436     my $arch = get_host_arch();
3437     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3438
3439     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3440         if ! -e $filepath;
3441
3442     my $raw = file_get_contents($filepath);
3443     $raw =~ s/^\s+|\s+$//g;
3444     my @flags = split(/\s+/, $raw);
3445
3446     return \@flags;
3447 }
3448
3449 my sub get_cpuunits {
3450     my ($conf) = @_;
3451     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3452
3453     my $cpuunits = $conf->{cpuunits};
3454     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3455
3456     if ($is_cgroupv2) {
3457         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3458     } else {
3459         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3460     }
3461     return $cpuunits;
3462 }
3463
3464 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3465 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3466 my sub should_disable_smm {
3467     my ($conf, $vga) = @_;
3468
3469     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3470         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3471 }
3472
3473 sub config_to_command {
3474     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3475         $pbs_backing) = @_;
3476
3477     my $cmd = [];
3478     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3479     my $devices = [];
3480     my $bridges = {};
3481     my $ostype = $conf->{ostype};
3482     my $winversion = windows_version($ostype);
3483     my $kvm = $conf->{kvm};
3484     my $nodename = nodename();
3485
3486     my $arch = get_vm_arch($conf);
3487     my $kvm_binary = get_command_for_arch($arch);
3488     my $kvmver = kvm_user_version($kvm_binary);
3489
3490     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3491         $kvmver //= "undefined";
3492         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3493     }
3494
3495     my $add_pve_version = min_version($kvmver, 4, 1);
3496
3497     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3498     my $machine_version = extract_version($machine_type, $kvmver);
3499     $kvm //= 1 if is_native($arch);
3500
3501     $machine_version =~ m/(\d+)\.(\d+)/;
3502     my ($machine_major, $machine_minor) = ($1, $2);
3503
3504     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3505         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3506     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3507         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3508             ." please upgrade node '$nodename'\n"
3509     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3510         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3511         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3512             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3513             ." node '$nodename'\n";
3514     }
3515
3516     # if a specific +pve version is required for a feature, use $version_guard
3517     # instead of min_version to allow machines to be run with the minimum
3518     # required version
3519     my $required_pve_version = 0;
3520     my $version_guard = sub {
3521         my ($major, $minor, $pve) = @_;
3522         return 0 if !min_version($machine_version, $major, $minor, $pve);
3523         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3524         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3525         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3526         return 1;
3527     };
3528
3529     if ($kvm && !defined kvm_version()) {
3530         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3531             ." or enable in BIOS.\n";
3532     }
3533
3534     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3535     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3536     my $use_old_bios_files = undef;
3537     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3538
3539     push @$cmd, $kvm_binary;
3540
3541     push @$cmd, '-id', $vmid;
3542
3543     my $vmname = $conf->{name} || "vm$vmid";
3544
3545     push @$cmd, '-name', "$vmname,debug-threads=on";
3546
3547     push @$cmd, '-no-shutdown';
3548
3549     my $use_virtio = 0;
3550
3551     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3552     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3553     push @$cmd, '-mon', "chardev=qmp,mode=control";
3554
3555     if (min_version($machine_version, 2, 12)) {
3556         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3557         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3558     }
3559
3560     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3561
3562     push @$cmd, '-daemonize';
3563
3564     if ($conf->{smbios1}) {
3565         my $smbios_conf = parse_smbios1($conf->{smbios1});
3566         if ($smbios_conf->{base64}) {
3567             # Do not pass base64 flag to qemu
3568             delete $smbios_conf->{base64};
3569             my $smbios_string = "";
3570             foreach my $key (keys %$smbios_conf) {
3571                 my $value;
3572                 if ($key eq "uuid") {
3573                     $value = $smbios_conf->{uuid}
3574                 } else {
3575                     $value = decode_base64($smbios_conf->{$key});
3576                 }
3577                 # qemu accepts any binary data, only commas need escaping by double comma
3578                 $value =~ s/,/,,/g;
3579                 $smbios_string .= "," . $key . "=" . $value if $value;
3580             }
3581             push @$cmd, '-smbios', "type=1" . $smbios_string;
3582         } else {
3583             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3584         }
3585     }
3586
3587     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3588         my $d;
3589         if (my $efidisk = $conf->{efidisk0}) {
3590             $d = parse_drive('efidisk0', $efidisk);
3591         }
3592
3593         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3594         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3595
3596         my ($path, $format);
3597         my $read_only_str = '';
3598         if ($d) {
3599             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3600             $format = $d->{format};
3601             if ($storeid) {
3602                 $path = PVE::Storage::path($storecfg, $d->{file});
3603                 if (!defined($format)) {
3604                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3605                     $format = qemu_img_format($scfg, $volname);
3606                 }
3607             } else {
3608                 $path = $d->{file};
3609                 die "efidisk format must be specified\n"
3610                     if !defined($format);
3611             }
3612
3613             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3614         } else {
3615             log_warn("no efidisk configured! Using temporary efivars disk.");
3616             $path = "/tmp/$vmid-ovmf.fd";
3617             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3618             $format = 'raw';
3619         }
3620
3621         my $size_str = "";
3622
3623         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3624             $size_str = ",size=" . (-s $ovmf_vars);
3625         }
3626
3627         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3628         my $cache = "";
3629         if ($path =~ m/^rbd:/) {
3630                 $cache = ',cache=writeback';
3631                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3632         }
3633
3634         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3635         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3636     }
3637
3638     if ($q35) { # tell QEMU to load q35 config early
3639         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3640         if (min_version($machine_version, 4, 0)) {
3641             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3642         } else {
3643             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3644         }
3645     }
3646
3647     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3648         push @$cmd, $fixups->@*;
3649     }
3650
3651     if ($conf->{vmgenid}) {
3652         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3653     }
3654
3655     # add usb controllers
3656     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3657         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3658     push @$devices, @usbcontrollers if @usbcontrollers;
3659     my $vga = parse_vga($conf->{vga});
3660
3661     my $qxlnum = vga_conf_has_spice($conf->{vga});
3662     $vga->{type} = 'qxl' if $qxlnum;
3663
3664     if (!$vga->{type}) {
3665         if ($arch eq 'aarch64') {
3666             $vga->{type} = 'virtio';
3667         } elsif (min_version($machine_version, 2, 9)) {
3668             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3669         } else {
3670             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3671         }
3672     }
3673
3674     # enable absolute mouse coordinates (needed by vnc)
3675     my $tablet = $conf->{tablet};
3676     if (!defined($tablet)) {
3677         $tablet = $defaults->{tablet};
3678         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3679         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3680     }
3681
3682     if ($tablet) {
3683         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3684         my $kbd = print_keyboarddevice_full($conf, $arch);
3685         push @$devices, '-device', $kbd if defined($kbd);
3686     }
3687
3688     my $bootorder = device_bootorder($conf);
3689
3690     # host pci device passthrough
3691     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3692         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3693
3694     # usb devices
3695     my $usb_dev_features = {};
3696     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3697
3698     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3699         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3700     push @$devices, @usbdevices if @usbdevices;
3701
3702     # serial devices
3703     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3704         my $path = $conf->{"serial$i"} or next;
3705         if ($path eq 'socket') {
3706             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3707             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3708             # On aarch64, serial0 is the UART device. Qemu only allows
3709             # connecting UART devices via the '-serial' command line, as
3710             # the device has a fixed slot on the hardware...
3711             if ($arch eq 'aarch64' && $i == 0) {
3712                 push @$devices, '-serial', "chardev:serial$i";
3713             } else {
3714                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3715             }
3716         } else {
3717             die "no such serial device\n" if ! -c $path;
3718             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3719             push @$devices, '-device', "isa-serial,chardev=serial$i";
3720         }
3721     }
3722
3723     # parallel devices
3724     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3725         if (my $path = $conf->{"parallel$i"}) {
3726             die "no such parallel device\n" if ! -c $path;
3727             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3728             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3729             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3730         }
3731     }
3732
3733     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3734         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3735         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3736         push @$devices, @$audio_devs;
3737     }
3738
3739     add_tpm_device($vmid, $devices, $conf);
3740
3741     my $sockets = 1;
3742     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3743     $sockets = $conf->{sockets} if  $conf->{sockets};
3744
3745     my $cores = $conf->{cores} || 1;
3746
3747     my $maxcpus = $sockets * $cores;
3748
3749     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3750
3751     my $allowed_vcpus = $cpuinfo->{cpus};
3752
3753     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3754
3755     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3756         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3757         for (my $i = 2; $i <= $vcpus; $i++)  {
3758             my $cpustr = print_cpu_device($conf,$i);
3759             push @$cmd, '-device', $cpustr;
3760         }
3761
3762     } else {
3763
3764         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3765     }
3766     push @$cmd, '-nodefaults';
3767
3768     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3769
3770     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3771
3772     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3773
3774     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3775         push @$devices, '-device', print_vga_device(
3776             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3777
3778         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3779
3780         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3781         push @$cmd,  '-vnc', "unix:$socket,password=on";
3782     } else {
3783         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3784         push @$cmd, '-nographic';
3785     }
3786
3787     # time drift fix
3788     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3789     my $useLocaltime = $conf->{localtime};
3790
3791     if ($winversion >= 5) { # windows
3792         $useLocaltime = 1 if !defined($conf->{localtime});
3793
3794         # use time drift fix when acpi is enabled
3795         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3796             $tdf = 1 if !defined($conf->{tdf});
3797         }
3798     }
3799
3800     if ($winversion >= 6) {
3801         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3802         push @$cmd, '-no-hpet';
3803     }
3804
3805     push @$rtcFlags, 'driftfix=slew' if $tdf;
3806
3807     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3808         push @$rtcFlags, "base=$conf->{startdate}";
3809     } elsif ($useLocaltime) {
3810         push @$rtcFlags, 'base=localtime';
3811     }
3812
3813     if ($forcecpu) {
3814         push @$cmd, '-cpu', $forcecpu;
3815     } else {
3816         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3817     }
3818
3819     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3820
3821     push @$cmd, '-S' if $conf->{freeze};
3822
3823     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3824
3825     my $guest_agent = parse_guest_agent($conf);
3826
3827     if ($guest_agent->{enabled}) {
3828         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3829         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3830
3831         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3832             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3833             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3834             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3835         } elsif ($guest_agent->{type} eq 'isa') {
3836             push @$devices, '-device', "isa-serial,chardev=qga0";
3837         }
3838     }
3839
3840     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3841     if ($rng && $version_guard->(4, 1, 2)) {
3842         check_rng_source($rng->{source});
3843
3844         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3845         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3846         my $limiter_str = "";
3847         if ($max_bytes) {
3848             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3849         }
3850
3851         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3852         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3853         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3854     }
3855
3856     my $spice_port;
3857
3858     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3859         if ($qxlnum > 1) {
3860             if ($winversion){
3861                 for (my $i = 1; $i < $qxlnum; $i++){
3862                     push @$devices, '-device', print_vga_device(
3863                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3864                 }
3865             } else {
3866                 # assume other OS works like Linux
3867                 my ($ram, $vram) = ("134217728", "67108864");
3868                 if ($vga->{memory}) {
3869                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3870                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3871                 }
3872                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3873                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3874             }
3875         }
3876
3877         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3878
3879         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3880         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3881         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3882
3883         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3884         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3885         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3886
3887         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3888         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3889
3890         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3891         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3892         if ($spice_enhancement->{foldersharing}) {
3893             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3894             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3895         }
3896
3897         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3898         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3899             if $spice_enhancement->{videostreaming};
3900
3901         push @$devices, '-spice', "$spice_opts";
3902     }
3903
3904     # enable balloon by default, unless explicitly disabled
3905     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3906         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3907         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3908         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3909         push @$devices, '-device', $ballooncmd;
3910     }
3911
3912     if ($conf->{watchdog}) {
3913         my $wdopts = parse_watchdog($conf->{watchdog});
3914         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3915         my $watchdog = $wdopts->{model} || 'i6300esb';
3916         push @$devices, '-device', "$watchdog$pciaddr";
3917         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3918     }
3919
3920     my $vollist = [];
3921     my $scsicontroller = {};
3922     my $ahcicontroller = {};
3923     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3924
3925     # Add iscsi initiator name if available
3926     if (my $initiator = get_initiator_name()) {
3927         push @$devices, '-iscsi', "initiator-name=$initiator";
3928     }
3929
3930     PVE::QemuConfig->foreach_volume($conf, sub {
3931         my ($ds, $drive) = @_;
3932
3933         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3934             check_volume_storage_type($storecfg, $drive->{file});
3935             push @$vollist, $drive->{file};
3936         }
3937
3938         # ignore efidisk here, already added in bios/fw handling code above
3939         return if $drive->{interface} eq 'efidisk';
3940         # similar for TPM
3941         return if $drive->{interface} eq 'tpmstate';
3942
3943         $use_virtio = 1 if $ds =~ m/^virtio/;
3944
3945         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3946
3947         if ($drive->{interface} eq 'virtio'){
3948            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3949         }
3950
3951         if ($drive->{interface} eq 'scsi') {
3952
3953             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3954
3955             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3956                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3957
3958             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3959             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3960
3961             my $iothread = '';
3962             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3963                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3964                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3965             } elsif ($drive->{iothread}) {
3966                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3967             }
3968
3969             my $queues = '';
3970             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3971                 $queues = ",num_queues=$drive->{queues}";
3972             }
3973
3974             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3975                 if !$scsicontroller->{$controller};
3976             $scsicontroller->{$controller}=1;
3977         }
3978
3979         if ($drive->{interface} eq 'sata') {
3980             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3981             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3982             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3983                 if !$ahcicontroller->{$controller};
3984             $ahcicontroller->{$controller}=1;
3985         }
3986
3987         my $pbs_conf = $pbs_backing->{$ds};
3988         my $pbs_name = undef;
3989         if ($pbs_conf) {
3990             $pbs_name = "drive-$ds-pbs";
3991             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3992         }
3993
3994         my $drive_cmd = print_drive_commandline_full(
3995             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3996
3997         # extra protection for templates, but SATA and IDE don't support it..
3998         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3999
4000         push @$devices, '-drive',$drive_cmd;
4001         push @$devices, '-device', print_drivedevice_full(
4002             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4003     });
4004
4005     for (my $i = 0; $i < $MAX_NETS; $i++) {
4006         my $netname = "net$i";
4007
4008         next if !$conf->{$netname};
4009         my $d = parse_net($conf->{$netname});
4010         next if !$d;
4011
4012         $use_virtio = 1 if $d->{model} eq 'virtio';
4013
4014         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4015
4016         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4017         push @$devices, '-netdev', $netdevfull;
4018
4019         my $netdevicefull = print_netdevice_full(
4020             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4021
4022         push @$devices, '-device', $netdevicefull;
4023     }
4024
4025     if ($conf->{ivshmem}) {
4026         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4027
4028         my $bus;
4029         if ($q35) {
4030             $bus = print_pcie_addr("ivshmem");
4031         } else {
4032             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4033         }
4034
4035         my $ivshmem_name = $ivshmem->{name} // $vmid;
4036         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4037
4038         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4039         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4040             .",size=$ivshmem->{size}M";
4041     }
4042
4043     # pci.4 is nested in pci.1
4044     $bridges->{1} = 1 if $bridges->{4};
4045
4046     if (!$q35) { # add pci bridges
4047         if (min_version($machine_version, 2, 3)) {
4048            $bridges->{1} = 1;
4049            $bridges->{2} = 1;
4050         }
4051         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4052     }
4053
4054     for my $k (sort {$b cmp $a} keys %$bridges) {
4055         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4056
4057         my $k_name = $k;
4058         if ($k == 2 && $legacy_igd) {
4059             $k_name = "$k-igd";
4060         }
4061         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4062         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4063
4064         if ($q35) { # add after -readconfig pve-q35.cfg
4065             splice @$devices, 2, 0, '-device', $devstr;
4066         } else {
4067             unshift @$devices, '-device', $devstr if $k > 0;
4068         }
4069     }
4070
4071     if (!$kvm) {
4072         push @$machineFlags, 'accel=tcg';
4073     }
4074
4075     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4076
4077     my $machine_type_min = $machine_type;
4078     if ($add_pve_version) {
4079         $machine_type_min =~ s/\+pve\d+$//;
4080         $machine_type_min .= "+pve$required_pve_version";
4081     }
4082     push @$machineFlags, "type=${machine_type_min}";
4083
4084     push @$cmd, @$devices;
4085     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4086     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4087     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4088
4089     if (my $vmstate = $conf->{vmstate}) {
4090         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4091         push @$vollist, $vmstate;
4092         push @$cmd, '-loadstate', $statepath;
4093         print "activating and using '$vmstate' as vmstate\n";
4094     }
4095
4096     if (PVE::QemuConfig->is_template($conf)) {
4097         # needed to workaround base volumes being read-only
4098         push @$cmd, '-snapshot';
4099     }
4100
4101     # add custom args
4102     if ($conf->{args}) {
4103         my $aa = PVE::Tools::split_args($conf->{args});
4104         push @$cmd, @$aa;
4105     }
4106
4107     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4108 }
4109
4110 sub check_rng_source {
4111     my ($source) = @_;
4112
4113     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4114     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4115         if ! -e $source;
4116
4117     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4118     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4119         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4120         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4121         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4122             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4123             ." to the host.\n";
4124     }
4125 }
4126
4127 sub spice_port {
4128     my ($vmid) = @_;
4129
4130     my $res = mon_cmd($vmid, 'query-spice');
4131
4132     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4133 }
4134
4135 sub vm_devices_list {
4136     my ($vmid) = @_;
4137
4138     my $res = mon_cmd($vmid, 'query-pci');
4139     my $devices_to_check = [];
4140     my $devices = {};
4141     foreach my $pcibus (@$res) {
4142         push @$devices_to_check, @{$pcibus->{devices}},
4143     }
4144
4145     while (@$devices_to_check) {
4146         my $to_check = [];
4147         for my $d (@$devices_to_check) {
4148             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4149             next if !$d->{'pci_bridge'};
4150
4151             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4152             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4153         }
4154         $devices_to_check = $to_check;
4155     }
4156
4157     my $resblock = mon_cmd($vmid, 'query-block');
4158     foreach my $block (@$resblock) {
4159         if($block->{device} =~ m/^drive-(\S+)/){
4160                 $devices->{$1} = 1;
4161         }
4162     }
4163
4164     my $resmice = mon_cmd($vmid, 'query-mice');
4165     foreach my $mice (@$resmice) {
4166         if ($mice->{name} eq 'QEMU HID Tablet') {
4167             $devices->{tablet} = 1;
4168             last;
4169         }
4170     }
4171
4172     # for usb devices there is no query-usb
4173     # but we can iterate over the entries in
4174     # qom-list path=/machine/peripheral
4175     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4176     foreach my $per (@$resperipheral) {
4177         if ($per->{name} =~ m/^usb\d+$/) {
4178             $devices->{$per->{name}} = 1;
4179         }
4180     }
4181
4182     return $devices;
4183 }
4184
4185 sub vm_deviceplug {
4186     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4187
4188     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4189
4190     my $devices_list = vm_devices_list($vmid);
4191     return 1 if defined($devices_list->{$deviceid});
4192
4193     # add PCI bridge if we need it for the device
4194     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4195
4196     if ($deviceid eq 'tablet') {
4197         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4198     } elsif ($deviceid eq 'keyboard') {
4199         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4200     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4201         die "usb hotplug currently not reliable\n";
4202         # since we can't reliably hot unplug all added usb devices and usb
4203         # passthrough breaks live migration we disable usb hotplugging for now
4204         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4205     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4206         qemu_iothread_add($vmid, $deviceid, $device);
4207
4208         qemu_driveadd($storecfg, $vmid, $device);
4209         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4210
4211         qemu_deviceadd($vmid, $devicefull);
4212         eval { qemu_deviceaddverify($vmid, $deviceid); };
4213         if (my $err = $@) {
4214             eval { qemu_drivedel($vmid, $deviceid); };
4215             warn $@ if $@;
4216             die $err;
4217         }
4218     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4219         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4220         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4221         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4222
4223         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4224
4225         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4226             qemu_iothread_add($vmid, $deviceid, $device);
4227             $devicefull .= ",iothread=iothread-$deviceid";
4228         }
4229
4230         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4231             $devicefull .= ",num_queues=$device->{queues}";
4232         }
4233
4234         qemu_deviceadd($vmid, $devicefull);
4235         qemu_deviceaddverify($vmid, $deviceid);
4236     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4237         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4238         qemu_driveadd($storecfg, $vmid, $device);
4239
4240         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4241         eval { qemu_deviceadd($vmid, $devicefull); };
4242         if (my $err = $@) {
4243             eval { qemu_drivedel($vmid, $deviceid); };
4244             warn $@ if $@;
4245             die $err;
4246         }
4247     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4248         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4249
4250         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4251         my $use_old_bios_files = undef;
4252         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4253
4254         my $netdevicefull = print_netdevice_full(
4255             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4256         qemu_deviceadd($vmid, $netdevicefull);
4257         eval {
4258             qemu_deviceaddverify($vmid, $deviceid);
4259             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4260         };
4261         if (my $err = $@) {
4262             eval { qemu_netdevdel($vmid, $deviceid); };
4263             warn $@ if $@;
4264             die $err;
4265         }
4266     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4267         my $bridgeid = $2;
4268         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4269         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4270
4271         qemu_deviceadd($vmid, $devicefull);
4272         qemu_deviceaddverify($vmid, $deviceid);
4273     } else {
4274         die "can't hotplug device '$deviceid'\n";
4275     }
4276
4277     return 1;
4278 }
4279
4280 # fixme: this should raise exceptions on error!
4281 sub vm_deviceunplug {
4282     my ($vmid, $conf, $deviceid) = @_;
4283
4284     my $devices_list = vm_devices_list($vmid);
4285     return 1 if !defined($devices_list->{$deviceid});
4286
4287     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4288     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4289
4290     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4291         qemu_devicedel($vmid, $deviceid);
4292     } elsif ($deviceid =~ m/^usb\d+$/) {
4293         die "usb hotplug currently not reliable\n";
4294         # when unplugging usb devices this way, there may be remaining usb
4295         # controllers/hubs so we disable it for now
4296         #qemu_devicedel($vmid, $deviceid);
4297         #qemu_devicedelverify($vmid, $deviceid);
4298     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4299         my $device = parse_drive($deviceid, $conf->{$deviceid});
4300
4301         qemu_devicedel($vmid, $deviceid);
4302         qemu_devicedelverify($vmid, $deviceid);
4303         qemu_drivedel($vmid, $deviceid);
4304         qemu_iothread_del($vmid, $deviceid, $device);
4305     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4306         qemu_devicedel($vmid, $deviceid);
4307         qemu_devicedelverify($vmid, $deviceid);
4308     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4309         my $device = parse_drive($deviceid, $conf->{$deviceid});
4310
4311         qemu_devicedel($vmid, $deviceid);
4312         qemu_devicedelverify($vmid, $deviceid);
4313         qemu_drivedel($vmid, $deviceid);
4314         qemu_deletescsihw($conf, $vmid, $deviceid);
4315
4316         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4317             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4318     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4319         qemu_devicedel($vmid, $deviceid);
4320         qemu_devicedelverify($vmid, $deviceid);
4321         qemu_netdevdel($vmid, $deviceid);
4322     } else {
4323         die "can't unplug device '$deviceid'\n";
4324     }
4325
4326     return 1;
4327 }
4328
4329 sub qemu_deviceadd {
4330     my ($vmid, $devicefull) = @_;
4331
4332     $devicefull = "driver=".$devicefull;
4333     my %options =  split(/[=,]/, $devicefull);
4334
4335     mon_cmd($vmid, "device_add" , %options);
4336 }
4337
4338 sub qemu_devicedel {
4339     my ($vmid, $deviceid) = @_;
4340
4341     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4342 }
4343
4344 sub qemu_iothread_add {
4345     my ($vmid, $deviceid, $device) = @_;
4346
4347     if ($device->{iothread}) {
4348         my $iothreads = vm_iothreads_list($vmid);
4349         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4350     }
4351 }
4352
4353 sub qemu_iothread_del {
4354     my ($vmid, $deviceid, $device) = @_;
4355
4356     if ($device->{iothread}) {
4357         my $iothreads = vm_iothreads_list($vmid);
4358         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4359     }
4360 }
4361
4362 sub qemu_objectadd {
4363     my ($vmid, $objectid, $qomtype) = @_;
4364
4365     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4366
4367     return 1;
4368 }
4369
4370 sub qemu_objectdel {
4371     my ($vmid, $objectid) = @_;
4372
4373     mon_cmd($vmid, "object-del", id => $objectid);
4374
4375     return 1;
4376 }
4377
4378 sub qemu_driveadd {
4379     my ($storecfg, $vmid, $device) = @_;
4380
4381     my $kvmver = get_running_qemu_version($vmid);
4382     my $io_uring = min_version($kvmver, 6, 0);
4383     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4384     $drive =~ s/\\/\\\\/g;
4385     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4386
4387     # If the command succeeds qemu prints: "OK"
4388     return 1 if $ret =~ m/OK/s;
4389
4390     die "adding drive failed: $ret\n";
4391 }
4392
4393 sub qemu_drivedel {
4394     my ($vmid, $deviceid) = @_;
4395
4396     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4397     $ret =~ s/^\s+//;
4398
4399     return 1 if $ret eq "";
4400
4401     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4402     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4403
4404     die "deleting drive $deviceid failed : $ret\n";
4405 }
4406
4407 sub qemu_deviceaddverify {
4408     my ($vmid, $deviceid) = @_;
4409
4410     for (my $i = 0; $i <= 5; $i++) {
4411          my $devices_list = vm_devices_list($vmid);
4412          return 1 if defined($devices_list->{$deviceid});
4413          sleep 1;
4414     }
4415
4416     die "error on hotplug device '$deviceid'\n";
4417 }
4418
4419
4420 sub qemu_devicedelverify {
4421     my ($vmid, $deviceid) = @_;
4422
4423     # need to verify that the device is correctly removed as device_del
4424     # is async and empty return is not reliable
4425
4426     for (my $i = 0; $i <= 5; $i++) {
4427          my $devices_list = vm_devices_list($vmid);
4428          return 1 if !defined($devices_list->{$deviceid});
4429          sleep 1;
4430     }
4431
4432     die "error on hot-unplugging device '$deviceid'\n";
4433 }
4434
4435 sub qemu_findorcreatescsihw {
4436     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4437
4438     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4439
4440     my $scsihwid="$controller_prefix$controller";
4441     my $devices_list = vm_devices_list($vmid);
4442
4443     if (!defined($devices_list->{$scsihwid})) {
4444         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4445     }
4446
4447     return 1;
4448 }
4449
4450 sub qemu_deletescsihw {
4451     my ($conf, $vmid, $opt) = @_;
4452
4453     my $device = parse_drive($opt, $conf->{$opt});
4454
4455     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4456         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4457         return 1;
4458     }
4459
4460     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4461
4462     my $devices_list = vm_devices_list($vmid);
4463     foreach my $opt (keys %{$devices_list}) {
4464         if (is_valid_drivename($opt)) {
4465             my $drive = parse_drive($opt, $conf->{$opt});
4466             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4467                 return 1;
4468             }
4469         }
4470     }
4471
4472     my $scsihwid="scsihw$controller";
4473
4474     vm_deviceunplug($vmid, $conf, $scsihwid);
4475
4476     return 1;
4477 }
4478
4479 sub qemu_add_pci_bridge {
4480     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4481
4482     my $bridges = {};
4483
4484     my $bridgeid;
4485
4486     print_pci_addr($device, $bridges, $arch, $machine_type);
4487
4488     while (my ($k, $v) = each %$bridges) {
4489         $bridgeid = $k;
4490     }
4491     return 1 if !defined($bridgeid) || $bridgeid < 1;
4492
4493     my $bridge = "pci.$bridgeid";
4494     my $devices_list = vm_devices_list($vmid);
4495
4496     if (!defined($devices_list->{$bridge})) {
4497         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4498     }
4499
4500     return 1;
4501 }
4502
4503 sub qemu_set_link_status {
4504     my ($vmid, $device, $up) = @_;
4505
4506     mon_cmd($vmid, "set_link", name => $device,
4507                up => $up ? JSON::true : JSON::false);
4508 }
4509
4510 sub qemu_netdevadd {
4511     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4512
4513     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4514     my %options =  split(/[=,]/, $netdev);
4515
4516     if (defined(my $vhost = $options{vhost})) {
4517         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4518     }
4519
4520     if (defined(my $queues = $options{queues})) {
4521         $options{queues} = $queues + 0;
4522     }
4523
4524     mon_cmd($vmid, "netdev_add",  %options);
4525     return 1;
4526 }
4527
4528 sub qemu_netdevdel {
4529     my ($vmid, $deviceid) = @_;
4530
4531     mon_cmd($vmid, "netdev_del", id => $deviceid);
4532 }
4533
4534 sub qemu_usb_hotplug {
4535     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4536
4537     return if !$device;
4538
4539     # remove the old one first
4540     vm_deviceunplug($vmid, $conf, $deviceid);
4541
4542     # check if xhci controller is necessary and available
4543     if ($device->{usb3}) {
4544
4545         my $devicelist = vm_devices_list($vmid);
4546
4547         if (!$devicelist->{xhci}) {
4548             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4549             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4550         }
4551     }
4552     my $d = parse_usb_device($device->{host});
4553     $d->{usb3} = $device->{usb3};
4554
4555     # add the new one
4556     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4557 }
4558
4559 sub qemu_cpu_hotplug {
4560     my ($vmid, $conf, $vcpus) = @_;
4561
4562     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4563
4564     my $sockets = 1;
4565     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4566     $sockets = $conf->{sockets} if  $conf->{sockets};
4567     my $cores = $conf->{cores} || 1;
4568     my $maxcpus = $sockets * $cores;
4569
4570     $vcpus = $maxcpus if !$vcpus;
4571
4572     die "you can't add more vcpus than maxcpus\n"
4573         if $vcpus > $maxcpus;
4574
4575     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4576
4577     if ($vcpus < $currentvcpus) {
4578
4579         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4580
4581             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4582                 qemu_devicedel($vmid, "cpu$i");
4583                 my $retry = 0;
4584                 my $currentrunningvcpus = undef;
4585                 while (1) {
4586                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4587                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4588                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4589                     $retry++;
4590                     sleep 1;
4591                 }
4592                 #update conf after each succesfull cpu unplug
4593                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4594                 PVE::QemuConfig->write_config($vmid, $conf);
4595             }
4596         } else {
4597             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4598         }
4599
4600         return;
4601     }
4602
4603     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4604     die "vcpus in running vm does not match its configuration\n"
4605         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4606
4607     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4608
4609         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4610             my $cpustr = print_cpu_device($conf, $i);
4611             qemu_deviceadd($vmid, $cpustr);
4612
4613             my $retry = 0;
4614             my $currentrunningvcpus = undef;
4615             while (1) {
4616                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4617                 last if scalar(@{$currentrunningvcpus}) == $i;
4618                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4619                 sleep 1;
4620                 $retry++;
4621             }
4622             #update conf after each succesfull cpu hotplug
4623             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4624             PVE::QemuConfig->write_config($vmid, $conf);
4625         }
4626     } else {
4627
4628         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4629             mon_cmd($vmid, "cpu-add", id => int($i));
4630         }
4631     }
4632 }
4633
4634 sub qemu_block_set_io_throttle {
4635     my ($vmid, $deviceid,
4636         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4637         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4638         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4639         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4640
4641     return if !check_running($vmid) ;
4642
4643     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4644         bps => int($bps),
4645         bps_rd => int($bps_rd),
4646         bps_wr => int($bps_wr),
4647         iops => int($iops),
4648         iops_rd => int($iops_rd),
4649         iops_wr => int($iops_wr),
4650         bps_max => int($bps_max),
4651         bps_rd_max => int($bps_rd_max),
4652         bps_wr_max => int($bps_wr_max),
4653         iops_max => int($iops_max),
4654         iops_rd_max => int($iops_rd_max),
4655         iops_wr_max => int($iops_wr_max),
4656         bps_max_length => int($bps_max_length),
4657         bps_rd_max_length => int($bps_rd_max_length),
4658         bps_wr_max_length => int($bps_wr_max_length),
4659         iops_max_length => int($iops_max_length),
4660         iops_rd_max_length => int($iops_rd_max_length),
4661         iops_wr_max_length => int($iops_wr_max_length),
4662     );
4663
4664 }
4665
4666 sub qemu_block_resize {
4667     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4668
4669     my $running = check_running($vmid);
4670
4671     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4672
4673     return if !$running;
4674
4675     my $padding = (1024 - $size % 1024) % 1024;
4676     $size = $size + $padding;
4677
4678     mon_cmd(
4679         $vmid,
4680         "block_resize",
4681         device => $deviceid,
4682         size => int($size),
4683         timeout => 60,
4684     );
4685 }
4686
4687 sub qemu_volume_snapshot {
4688     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4689
4690     my $running = check_running($vmid);
4691
4692     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4693         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4694     } else {
4695         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4696     }
4697 }
4698
4699 sub qemu_volume_snapshot_delete {
4700     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4701
4702     my $running = check_running($vmid);
4703
4704     if($running) {
4705
4706         $running = undef;
4707         my $conf = PVE::QemuConfig->load_config($vmid);
4708         PVE::QemuConfig->foreach_volume($conf, sub {
4709             my ($ds, $drive) = @_;
4710             $running = 1 if $drive->{file} eq $volid;
4711         });
4712     }
4713
4714     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4715         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4716     } else {
4717         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4718     }
4719 }
4720
4721 sub set_migration_caps {
4722     my ($vmid, $savevm) = @_;
4723
4724     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4725
4726     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4727     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4728
4729     my $cap_ref = [];
4730
4731     my $enabled_cap = {
4732         "auto-converge" => 1,
4733         "xbzrle" => 1,
4734         "x-rdma-pin-all" => 0,
4735         "zero-blocks" => 0,
4736         "compress" => 0,
4737         "dirty-bitmaps" => $dirty_bitmaps,
4738     };
4739
4740     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4741
4742     for my $supported_capability (@$supported_capabilities) {
4743         push @$cap_ref, {
4744             capability => $supported_capability->{capability},
4745             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4746         };
4747     }
4748
4749     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4750 }
4751
4752 sub foreach_volid {
4753     my ($conf, $func, @param) = @_;
4754
4755     my $volhash = {};
4756
4757     my $test_volid = sub {
4758         my ($key, $drive, $snapname) = @_;
4759
4760         my $volid = $drive->{file};
4761         return if !$volid;
4762
4763         $volhash->{$volid}->{cdrom} //= 1;
4764         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4765
4766         my $replicate = $drive->{replicate} // 1;
4767         $volhash->{$volid}->{replicate} //= 0;
4768         $volhash->{$volid}->{replicate} = 1 if $replicate;
4769
4770         $volhash->{$volid}->{shared} //= 0;
4771         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4772
4773         $volhash->{$volid}->{referenced_in_config} //= 0;
4774         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4775
4776         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4777             if defined($snapname);
4778
4779         my $size = $drive->{size};
4780         $volhash->{$volid}->{size} //= $size if $size;
4781
4782         $volhash->{$volid}->{is_vmstate} //= 0;
4783         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4784
4785         $volhash->{$volid}->{is_tpmstate} //= 0;
4786         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4787
4788         $volhash->{$volid}->{is_unused} //= 0;
4789         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4790
4791         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4792     };
4793
4794     my $include_opts = {
4795         extra_keys => ['vmstate'],
4796         include_unused => 1,
4797     };
4798
4799     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4800     foreach my $snapname (keys %{$conf->{snapshots}}) {
4801         my $snap = $conf->{snapshots}->{$snapname};
4802         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4803     }
4804
4805     foreach my $volid (keys %$volhash) {
4806         &$func($volid, $volhash->{$volid}, @param);
4807     }
4808 }
4809
4810 my $fast_plug_option = {
4811     'lock' => 1,
4812     'name' => 1,
4813     'onboot' => 1,
4814     'shares' => 1,
4815     'startup' => 1,
4816     'description' => 1,
4817     'protection' => 1,
4818     'vmstatestorage' => 1,
4819     'hookscript' => 1,
4820     'tags' => 1,
4821 };
4822
4823 # hotplug changes in [PENDING]
4824 # $selection hash can be used to only apply specified options, for
4825 # example: { cores => 1 } (only apply changed 'cores')
4826 # $errors ref is used to return error messages
4827 sub vmconfig_hotplug_pending {
4828     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4829
4830     my $defaults = load_defaults();
4831     my $arch = get_vm_arch($conf);
4832     my $machine_type = get_vm_machine($conf, undef, $arch);
4833
4834     # commit values which do not have any impact on running VM first
4835     # Note: those option cannot raise errors, we we do not care about
4836     # $selection and always apply them.
4837
4838     my $add_error = sub {
4839         my ($opt, $msg) = @_;
4840         $errors->{$opt} = "hotplug problem - $msg";
4841     };
4842
4843     my $changes = 0;
4844     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4845         if ($fast_plug_option->{$opt}) {
4846             $conf->{$opt} = $conf->{pending}->{$opt};
4847             delete $conf->{pending}->{$opt};
4848             $changes = 1;
4849         }
4850     }
4851
4852     if ($changes) {
4853         PVE::QemuConfig->write_config($vmid, $conf);
4854     }
4855
4856     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4857
4858     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4859     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4860     foreach my $opt (sort keys %$pending_delete_hash) {
4861         next if $selection && !$selection->{$opt};
4862         my $force = $pending_delete_hash->{$opt}->{force};
4863         eval {
4864             if ($opt eq 'hotplug') {
4865                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4866             } elsif ($opt eq 'tablet') {
4867                 die "skip\n" if !$hotplug_features->{usb};
4868                 if ($defaults->{tablet}) {
4869                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4870                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4871                         if $arch eq 'aarch64';
4872                 } else {
4873                     vm_deviceunplug($vmid, $conf, 'tablet');
4874                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4875                 }
4876             } elsif ($opt =~ m/^usb\d+/) {
4877                 die "skip\n";
4878                 # since we cannot reliably hot unplug usb devices we are disabling it
4879                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4880                 #vm_deviceunplug($vmid, $conf, $opt);
4881             } elsif ($opt eq 'vcpus') {
4882                 die "skip\n" if !$hotplug_features->{cpu};
4883                 qemu_cpu_hotplug($vmid, $conf, undef);
4884             } elsif ($opt eq 'balloon') {
4885                 # enable balloon device is not hotpluggable
4886                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4887                 # here we reset the ballooning value to memory
4888                 my $balloon = $conf->{memory} || $defaults->{memory};
4889                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4890             } elsif ($fast_plug_option->{$opt}) {
4891                 # do nothing
4892             } elsif ($opt =~ m/^net(\d+)$/) {
4893                 die "skip\n" if !$hotplug_features->{network};
4894                 vm_deviceunplug($vmid, $conf, $opt);
4895             } elsif (is_valid_drivename($opt)) {
4896                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4897                 vm_deviceunplug($vmid, $conf, $opt);
4898                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4899             } elsif ($opt =~ m/^memory$/) {
4900                 die "skip\n" if !$hotplug_features->{memory};
4901                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4902             } elsif ($opt eq 'cpuunits') {
4903                 $cgroup->change_cpu_shares(undef, 1024);
4904             } elsif ($opt eq 'cpulimit') {
4905                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4906             } else {
4907                 die "skip\n";
4908             }
4909         };
4910         if (my $err = $@) {
4911             &$add_error($opt, $err) if $err ne "skip\n";
4912         } else {
4913             delete $conf->{$opt};
4914             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4915         }
4916     }
4917
4918     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4919     $apply_pending_cloudinit = sub {
4920         return if $apply_pending_cloudinit_done; # once is enough
4921         $apply_pending_cloudinit_done = 1; # once is enough
4922
4923         my ($key, $value) = @_;
4924
4925         my @cloudinit_opts = keys %$confdesc_cloudinit;
4926         foreach my $opt (keys %{$conf->{pending}}) {
4927             next if !grep { $_ eq $opt } @cloudinit_opts;
4928             $conf->{$opt} = delete $conf->{pending}->{$opt};
4929         }
4930
4931         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4932         foreach my $opt (sort keys %$pending_delete_hash) {
4933             next if !grep { $_ eq $opt } @cloudinit_opts;
4934             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4935             delete $conf->{$opt};
4936         }
4937
4938         my $new_conf = { %$conf };
4939         $new_conf->{$key} = $value;
4940         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4941     };
4942
4943     foreach my $opt (keys %{$conf->{pending}}) {
4944         next if $selection && !$selection->{$opt};
4945         my $value = $conf->{pending}->{$opt};
4946         eval {
4947             if ($opt eq 'hotplug') {
4948                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4949             } elsif ($opt eq 'tablet') {
4950                 die "skip\n" if !$hotplug_features->{usb};
4951                 if ($value == 1) {
4952                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4953                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4954                         if $arch eq 'aarch64';
4955                 } elsif ($value == 0) {
4956                     vm_deviceunplug($vmid, $conf, 'tablet');
4957                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4958                 }
4959             } elsif ($opt =~ m/^usb\d+$/) {
4960                 die "skip\n";
4961                 # since we cannot reliably hot unplug usb devices we disable it for now
4962                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4963                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4964                 #die "skip\n" if !$d;
4965                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4966             } elsif ($opt eq 'vcpus') {
4967                 die "skip\n" if !$hotplug_features->{cpu};
4968                 qemu_cpu_hotplug($vmid, $conf, $value);
4969             } elsif ($opt eq 'balloon') {
4970                 # enable/disable balloning device is not hotpluggable
4971                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4972                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4973                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4974
4975                 # allow manual ballooning if shares is set to zero
4976                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4977                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4978                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4979                 }
4980             } elsif ($opt =~ m/^net(\d+)$/) {
4981                 # some changes can be done without hotplug
4982                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4983                                     $vmid, $opt, $value, $arch, $machine_type);
4984             } elsif (is_valid_drivename($opt)) {
4985                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4986                 # some changes can be done without hotplug
4987                 my $drive = parse_drive($opt, $value);
4988                 if (drive_is_cloudinit($drive)) {
4989                     &$apply_pending_cloudinit($opt, $value);
4990                 }
4991                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4992                                      $vmid, $opt, $value, $arch, $machine_type);
4993             } elsif ($opt =~ m/^memory$/) { #dimms
4994                 die "skip\n" if !$hotplug_features->{memory};
4995                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4996             } elsif ($opt eq 'cpuunits') {
4997                 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4998                 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4999             } elsif ($opt eq 'cpulimit') {
5000                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5001                 $cgroup->change_cpu_quota($cpulimit, 100000);
5002             } elsif ($opt eq 'agent') {
5003                 vmconfig_update_agent($conf, $opt, $value);
5004             } else {
5005                 die "skip\n";  # skip non-hot-pluggable options
5006             }
5007         };
5008         if (my $err = $@) {
5009             &$add_error($opt, $err) if $err ne "skip\n";
5010         } else {
5011             $conf->{$opt} = $value;
5012             delete $conf->{pending}->{$opt};
5013         }
5014     }
5015
5016     PVE::QemuConfig->write_config($vmid, $conf);
5017 }
5018
5019 sub try_deallocate_drive {
5020     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5021
5022     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5023         my $volid = $drive->{file};
5024         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5025             my $sid = PVE::Storage::parse_volume_id($volid);
5026             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5027
5028             # check if the disk is really unused
5029             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5030                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5031             PVE::Storage::vdisk_free($storecfg, $volid);
5032             return 1;
5033         } else {
5034             # If vm is not owner of this disk remove from config
5035             return 1;
5036         }
5037     }
5038
5039     return;
5040 }
5041
5042 sub vmconfig_delete_or_detach_drive {
5043     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5044
5045     my $drive = parse_drive($opt, $conf->{$opt});
5046
5047     my $rpcenv = PVE::RPCEnvironment::get();
5048     my $authuser = $rpcenv->get_user();
5049
5050     if ($force) {
5051         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5052         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5053     } else {
5054         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5055     }
5056 }
5057
5058
5059
5060 sub vmconfig_apply_pending {
5061     my ($vmid, $conf, $storecfg, $errors) = @_;
5062
5063     return if !scalar(keys %{$conf->{pending}});
5064
5065     my $add_apply_error = sub {
5066         my ($opt, $msg) = @_;
5067         my $err_msg = "unable to apply pending change $opt : $msg";
5068         $errors->{$opt} = $err_msg;
5069         warn $err_msg;
5070     };
5071
5072     # cold plug
5073
5074     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5075     foreach my $opt (sort keys %$pending_delete_hash) {
5076         my $force = $pending_delete_hash->{$opt}->{force};
5077         eval {
5078             if ($opt =~ m/^unused/) {
5079                 die "internal error";
5080             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5081                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5082             }
5083         };
5084         if (my $err = $@) {
5085             $add_apply_error->($opt, $err);
5086         } else {
5087             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5088             delete $conf->{$opt};
5089         }
5090     }
5091
5092     PVE::QemuConfig->cleanup_pending($conf);
5093
5094     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5095         next if $opt eq 'delete'; # just to be sure
5096         eval {
5097             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5098                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5099             }
5100         };
5101         if (my $err = $@) {
5102             $add_apply_error->($opt, $err);
5103         } else {
5104             $conf->{$opt} = delete $conf->{pending}->{$opt};
5105         }
5106     }
5107
5108     # write all changes at once to avoid unnecessary i/o
5109     PVE::QemuConfig->write_config($vmid, $conf);
5110 }
5111
5112 sub vmconfig_update_net {
5113     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5114
5115     my $newnet = parse_net($value);
5116
5117     if ($conf->{$opt}) {
5118         my $oldnet = parse_net($conf->{$opt});
5119
5120         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5121             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5122             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5123             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5124
5125             # for non online change, we try to hot-unplug
5126             die "skip\n" if !$hotplug;
5127             vm_deviceunplug($vmid, $conf, $opt);
5128         } else {
5129
5130             die "internal error" if $opt !~ m/net(\d+)/;
5131             my $iface = "tap${vmid}i$1";
5132
5133             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5134                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5135                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5136                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5137                 PVE::Network::tap_unplug($iface);
5138
5139                 if ($have_sdn) {
5140                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5141                 } else {
5142                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5143                 }
5144             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5145                 # Rate can be applied on its own but any change above needs to
5146                 # include the rate in tap_plug since OVS resets everything.
5147                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5148             }
5149
5150             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5151                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5152             }
5153
5154             return 1;
5155         }
5156     }
5157
5158     if ($hotplug) {
5159         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5160     } else {
5161         die "skip\n";
5162     }
5163 }
5164
5165 sub vmconfig_update_agent {
5166     my ($conf, $opt, $value) = @_;
5167
5168     die "skip\n" if !$conf->{$opt};
5169
5170     my $hotplug_options = { fstrim_cloned_disks => 1 };
5171
5172     my $old_agent = parse_guest_agent($conf);
5173     my $agent = parse_guest_agent({$opt => $value});
5174
5175     for my $option (keys %$agent) { # added/changed options
5176         next if defined($hotplug_options->{$option});
5177         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5178     }
5179
5180     for my $option (keys %$old_agent) { # removed options
5181         next if defined($hotplug_options->{$option});
5182         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5183     }
5184
5185     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5186 }
5187
5188 sub vmconfig_update_disk {
5189     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5190
5191     my $drive = parse_drive($opt, $value);
5192
5193     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5194         my $media = $drive->{media} || 'disk';
5195         my $oldmedia = $old_drive->{media} || 'disk';
5196         die "unable to change media type\n" if $media ne $oldmedia;
5197
5198         if (!drive_is_cdrom($old_drive)) {
5199
5200             if ($drive->{file} ne $old_drive->{file}) {
5201
5202                 die "skip\n" if !$hotplug;
5203
5204                 # unplug and register as unused
5205                 vm_deviceunplug($vmid, $conf, $opt);
5206                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5207
5208             } else {
5209                 # update existing disk
5210
5211                 # skip non hotpluggable value
5212                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5213                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5214                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5215                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5216                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5217                     die "skip\n";
5218                 }
5219
5220                 # apply throttle
5221                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5222                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5223                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5224                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5225                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5226                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5227                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5228                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5229                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5230                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5231                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5232                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5233                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5234                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5235                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5236                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5237                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5238                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5239
5240                     qemu_block_set_io_throttle(
5241                         $vmid,"drive-$opt",
5242                         ($drive->{mbps} || 0)*1024*1024,
5243                         ($drive->{mbps_rd} || 0)*1024*1024,
5244                         ($drive->{mbps_wr} || 0)*1024*1024,
5245                          $drive->{iops} || 0,
5246                          $drive->{iops_rd} || 0,
5247                          $drive->{iops_wr} || 0,
5248                         ($drive->{mbps_max} || 0)*1024*1024,
5249                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5250                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5251                          $drive->{iops_max} || 0,
5252                          $drive->{iops_rd_max} || 0,
5253                          $drive->{iops_wr_max} || 0,
5254                          $drive->{bps_max_length} || 1,
5255                          $drive->{bps_rd_max_length} || 1,
5256                          $drive->{bps_wr_max_length} || 1,
5257                          $drive->{iops_max_length} || 1,
5258                          $drive->{iops_rd_max_length} || 1,
5259                          $drive->{iops_wr_max_length} || 1,
5260                     );
5261
5262                 }
5263
5264                 return 1;
5265             }
5266
5267         } else { # cdrom
5268
5269             if ($drive->{file} eq 'none') {
5270                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5271                 if (drive_is_cloudinit($old_drive)) {
5272                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5273                 }
5274             } else {
5275                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5276
5277                 # force eject if locked
5278                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5279
5280                 if ($path) {
5281                     mon_cmd($vmid, "blockdev-change-medium",
5282                         id => "$opt", filename => "$path");
5283                 }
5284             }
5285
5286             return 1;
5287         }
5288     }
5289
5290     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5291     # hotplug new disks
5292     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5293     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5294 }
5295
5296 # called in locked context by incoming migration
5297 sub vm_migrate_get_nbd_disks {
5298     my ($storecfg, $conf, $replicated_volumes) = @_;
5299
5300     my $local_volumes = {};
5301     PVE::QemuConfig->foreach_volume($conf, sub {
5302         my ($ds, $drive) = @_;
5303
5304         return if drive_is_cdrom($drive);
5305         return if $ds eq 'tpmstate0';
5306
5307         my $volid = $drive->{file};
5308
5309         return if !$volid;
5310
5311         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5312
5313         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5314         return if $scfg->{shared};
5315
5316         # replicated disks re-use existing state via bitmap
5317         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5318         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5319     });
5320     return $local_volumes;
5321 }
5322
5323 # called in locked context by incoming migration
5324 sub vm_migrate_alloc_nbd_disks {
5325     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5326
5327     my $nbd = {};
5328     foreach my $opt (sort keys %$source_volumes) {
5329         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5330
5331         if ($use_existing) {
5332             $nbd->{$opt}->{drivestr} = print_drive($drive);
5333             $nbd->{$opt}->{volid} = $volid;
5334             $nbd->{$opt}->{replicated} = 1;
5335             next;
5336         }
5337
5338         # storage mapping + volname = regular migration
5339         # storage mapping + format = remote migration
5340         # order of precedence, filtered by whether storage supports it:
5341         # 1. explicit requested format
5342         # 2. format of current volume
5343         # 3. default format of storage
5344         if (!$storagemap->{identity}) {
5345             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5346             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5347             if (!$format || !grep { $format eq $_ } @$validFormats) {
5348                 if ($volname) {
5349                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5350                     my $fileFormat = qemu_img_format($scfg, $volname);
5351                     $format = $fileFormat
5352                         if grep { $fileFormat eq $_ } @$validFormats;
5353                 }
5354                 $format //= $defFormat;
5355             }
5356         } else {
5357             # can't happen for remote migration, so $volname is always defined
5358             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5359             $format = qemu_img_format($scfg, $volname);
5360         }
5361
5362         my $size = $drive->{size} / 1024;
5363         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5364         my $newdrive = $drive;
5365         $newdrive->{format} = $format;
5366         $newdrive->{file} = $newvolid;
5367         my $drivestr = print_drive($newdrive);
5368         $nbd->{$opt}->{drivestr} = $drivestr;
5369         $nbd->{$opt}->{volid} = $newvolid;
5370     }
5371
5372     return $nbd;
5373 }
5374
5375 # see vm_start_nolock for parameters, additionally:
5376 # migrate_opts:
5377 #   storagemap = parsed storage map for allocating NBD disks
5378 sub vm_start {
5379     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5380
5381     return PVE::QemuConfig->lock_config($vmid, sub {
5382         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5383
5384         die "you can't start a vm if it's a template\n"
5385             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5386
5387         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5388         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5389
5390         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5391
5392         if ($has_backup_lock && $running) {
5393             # a backup is currently running, attempt to start the guest in the
5394             # existing QEMU instance
5395             return vm_resume($vmid);
5396         }
5397
5398         PVE::QemuConfig->check_lock($conf)
5399             if !($params->{skiplock} || $has_suspended_lock);
5400
5401         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5402
5403         die "VM $vmid already running\n" if $running;
5404
5405         if (my $storagemap = $migrate_opts->{storagemap}) {
5406             my $replicated = $migrate_opts->{replicated_volumes};
5407             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5408             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5409
5410             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5411                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5412             }
5413         }
5414
5415         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5416     });
5417 }
5418
5419
5420 # params:
5421 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5422 #   skiplock => 0/1, skip checking for config lock
5423 #   skiptemplate => 0/1, skip checking whether VM is template
5424 #   forcemachine => to force Qemu machine (rollback/migration)
5425 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5426 #   timeout => in seconds
5427 #   paused => start VM in paused state (backup)
5428 #   resume => resume from hibernation
5429 #   pbs-backing => {
5430 #      sata0 => {
5431 #         repository
5432 #         snapshot
5433 #         keyfile
5434 #         archive
5435 #      },
5436 #      virtio2 => ...
5437 #   }
5438 # migrate_opts:
5439 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5440 #   migratedfrom => source node
5441 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5442 #   network => CIDR of migration network
5443 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5444 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5445 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5446 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5447 #       contained in config
5448 sub vm_start_nolock {
5449     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5450
5451     my $statefile = $params->{statefile};
5452     my $resume = $params->{resume};
5453
5454     my $migratedfrom = $migrate_opts->{migratedfrom};
5455     my $migration_type = $migrate_opts->{type};
5456
5457     my $res = {};
5458
5459     # clean up leftover reboot request files
5460     eval { clear_reboot_request($vmid); };
5461     warn $@ if $@;
5462
5463     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5464         vmconfig_apply_pending($vmid, $conf, $storecfg);
5465         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5466     }
5467
5468     # don't regenerate the ISO if the VM is started as part of a live migration
5469     # this way we can reuse the old ISO with the correct config
5470     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5471
5472     # override offline migrated volumes, conf is out of date still
5473     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5474         for my $key (sort keys $offline_volumes->%*) {
5475             my $parsed = parse_drive($key, $conf->{$key});
5476             $parsed->{file} = $offline_volumes->{$key};
5477             $conf->{$key} = print_drive($parsed);
5478         }
5479     }
5480
5481     my $defaults = load_defaults();
5482
5483     # set environment variable useful inside network script
5484     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5485
5486     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5487
5488     my $forcemachine = $params->{forcemachine};
5489     my $forcecpu = $params->{forcecpu};
5490     if ($resume) {
5491         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5492         $forcemachine = $conf->{runningmachine};
5493         $forcecpu = $conf->{runningcpu};
5494         print "Resuming suspended VM\n";
5495     }
5496
5497     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5498         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5499
5500     my $migration_ip;
5501     my $get_migration_ip = sub {
5502         my ($nodename) = @_;
5503
5504         return $migration_ip if defined($migration_ip);
5505
5506         my $cidr = $migrate_opts->{network};
5507
5508         if (!defined($cidr)) {
5509             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5510             $cidr = $dc_conf->{migration}->{network};
5511         }
5512
5513         if (defined($cidr)) {
5514             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5515
5516             die "could not get IP: no address configured on local " .
5517                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5518
5519             die "could not get IP: multiple addresses configured on local " .
5520                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5521
5522             $migration_ip = @$ips[0];
5523         }
5524
5525         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5526             if !defined($migration_ip);
5527
5528         return $migration_ip;
5529     };
5530
5531     my $migrate_uri;
5532     if ($statefile) {
5533         if ($statefile eq 'tcp') {
5534             my $localip = "localhost";
5535             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5536             my $nodename = nodename();
5537
5538             if (!defined($migration_type)) {
5539                 if (defined($datacenterconf->{migration}->{type})) {
5540                     $migration_type = $datacenterconf->{migration}->{type};
5541                 } else {
5542                     $migration_type = 'secure';
5543                 }
5544             }
5545
5546             if ($migration_type eq 'insecure') {
5547                 $localip = $get_migration_ip->($nodename);
5548                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5549             }
5550
5551             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5552             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5553             $migrate_uri = "tcp:${localip}:${migrate_port}";
5554             push @$cmd, '-incoming', $migrate_uri;
5555             push @$cmd, '-S';
5556
5557         } elsif ($statefile eq 'unix') {
5558             # should be default for secure migrations as a ssh TCP forward
5559             # tunnel is not deterministic reliable ready and fails regurarly
5560             # to set up in time, so use UNIX socket forwards
5561             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5562             unlink $socket_addr;
5563
5564             $migrate_uri = "unix:$socket_addr";
5565
5566             push @$cmd, '-incoming', $migrate_uri;
5567             push @$cmd, '-S';
5568
5569         } elsif (-e $statefile) {
5570             push @$cmd, '-loadstate', $statefile;
5571         } else {
5572             my $statepath = PVE::Storage::path($storecfg, $statefile);
5573             push @$vollist, $statefile;
5574             push @$cmd, '-loadstate', $statepath;
5575         }
5576     } elsif ($params->{paused}) {
5577         push @$cmd, '-S';
5578     }
5579
5580     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5581
5582     my $pci_devices = {}; # host pci devices
5583     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5584         my $dev = $conf->{"hostpci$i"} or next;
5585         $pci_devices->{$i} = parse_hostpci($dev);
5586     }
5587
5588     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5589     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5590
5591     # map to a flat list of pci ids
5592     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5593
5594     # reserve all PCI IDs before actually doing anything with them
5595     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5596
5597     eval {
5598         my $uuid;
5599         for my $id (sort keys %$pci_devices) {
5600             my $d = $pci_devices->{$id};
5601             for my $dev ($d->{pciid}->@*) {
5602                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5603
5604                 # nvidia grid needs the uuid of the mdev as qemu parameter
5605                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5606                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5607                 }
5608             }
5609         }
5610         push @$cmd, '-uuid', $uuid if defined($uuid);
5611     };
5612     if (my $err = $@) {
5613         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5614         warn $@ if $@;
5615         die $err;
5616     }
5617
5618     PVE::Storage::activate_volumes($storecfg, $vollist);
5619
5620     eval {
5621         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5622     };
5623     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5624     # timeout should be more than enough here...
5625     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5626
5627     my $cpuunits = get_cpuunits($conf);
5628
5629     my %run_params = (
5630         timeout => $statefile ? undef : $start_timeout,
5631         umask => 0077,
5632         noerr => 1,
5633     );
5634
5635     # when migrating, prefix QEMU output so other side can pick up any
5636     # errors that might occur and show the user
5637     if ($migratedfrom) {
5638         $run_params{quiet} = 1;
5639         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5640     }
5641
5642     my %systemd_properties = (
5643         Slice => 'qemu.slice',
5644         KillMode => 'process',
5645         SendSIGKILL => 0,
5646         TimeoutStopUSec => ULONG_MAX, # infinity
5647     );
5648
5649     if (PVE::CGroup::cgroup_mode() == 2) {
5650         $systemd_properties{CPUWeight} = $cpuunits;
5651     } else {
5652         $systemd_properties{CPUShares} = $cpuunits;
5653     }
5654
5655     if (my $cpulimit = $conf->{cpulimit}) {
5656         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5657     }
5658     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5659
5660     my $run_qemu = sub {
5661         PVE::Tools::run_fork sub {
5662             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5663
5664             my $tpmpid;
5665             if (my $tpm = $conf->{tpmstate0}) {
5666                 # start the TPM emulator so QEMU can connect on start
5667                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5668             }
5669
5670             my $exitcode = run_command($cmd, %run_params);
5671             if ($exitcode) {
5672                 if ($tpmpid) {
5673                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5674                     kill 'TERM', $tpmpid;
5675                 }
5676                 die "QEMU exited with code $exitcode\n";
5677             }
5678         };
5679     };
5680
5681     if ($conf->{hugepages}) {
5682
5683         my $code = sub {
5684             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5685             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5686
5687             PVE::QemuServer::Memory::hugepages_mount();
5688             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5689
5690             eval { $run_qemu->() };
5691             if (my $err = $@) {
5692                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5693                     if !$conf->{keephugepages};
5694                 die $err;
5695             }
5696
5697             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5698                 if !$conf->{keephugepages};
5699         };
5700         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5701
5702     } else {
5703         eval { $run_qemu->() };
5704     }
5705
5706     if (my $err = $@) {
5707         # deactivate volumes if start fails
5708         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5709         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5710
5711         die "start failed: $err";
5712     }
5713
5714     # re-reserve all PCI IDs now that we can know the actual VM PID
5715     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5716     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5717     warn $@ if $@;
5718
5719     print "migration listens on $migrate_uri\n" if $migrate_uri;
5720     $res->{migrate_uri} = $migrate_uri;
5721
5722     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5723         eval { mon_cmd($vmid, "cont"); };
5724         warn $@ if $@;
5725     }
5726
5727     #start nbd server for storage migration
5728     if (my $nbd = $migrate_opts->{nbd}) {
5729         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5730
5731         my $migrate_storage_uri;
5732         # nbd_protocol_version > 0 for unix socket support
5733         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5734             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5735             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5736             $migrate_storage_uri = "nbd:unix:$socket_path";
5737         } else {
5738             my $nodename = nodename();
5739             my $localip = $get_migration_ip->($nodename);
5740             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5741             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5742
5743             mon_cmd($vmid, "nbd-server-start", addr => {
5744                 type => 'inet',
5745                 data => {
5746                     host => "${localip}",
5747                     port => "${storage_migrate_port}",
5748                 },
5749             });
5750             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5751             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5752         }
5753
5754         $res->{migrate_storage_uri} = $migrate_storage_uri;
5755
5756         foreach my $opt (sort keys %$nbd) {
5757             my $drivestr = $nbd->{$opt}->{drivestr};
5758             my $volid = $nbd->{$opt}->{volid};
5759             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5760             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5761             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5762             print "re-using replicated volume: $opt - $volid\n"
5763                 if $nbd->{$opt}->{replicated};
5764
5765             $res->{drives}->{$opt} = $nbd->{$opt};
5766             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5767         }
5768     }
5769
5770     if ($migratedfrom) {
5771         eval {
5772             set_migration_caps($vmid);
5773         };
5774         warn $@ if $@;
5775
5776         if ($spice_port) {
5777             print "spice listens on port $spice_port\n";
5778             $res->{spice_port} = $spice_port;
5779             if ($migrate_opts->{spice_ticket}) {
5780                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5781                     $migrate_opts->{spice_ticket});
5782                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5783             }
5784         }
5785
5786     } else {
5787         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5788             if !$statefile && $conf->{balloon};
5789
5790         foreach my $opt (keys %$conf) {
5791             next if $opt !~  m/^net\d+$/;
5792             my $nicconf = parse_net($conf->{$opt});
5793             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5794         }
5795     }
5796
5797     mon_cmd($vmid, 'qom-set',
5798                 path => "machine/peripheral/balloon0",
5799                 property => "guest-stats-polling-interval",
5800                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5801
5802     if ($resume) {
5803         print "Resumed VM, removing state\n";
5804         if (my $vmstate = $conf->{vmstate}) {
5805             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5806             PVE::Storage::vdisk_free($storecfg, $vmstate);
5807         }
5808         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5809         PVE::QemuConfig->write_config($vmid, $conf);
5810     }
5811
5812     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5813
5814     return $res;
5815 }
5816
5817 sub vm_commandline {
5818     my ($storecfg, $vmid, $snapname) = @_;
5819
5820     my $conf = PVE::QemuConfig->load_config($vmid);
5821
5822     my ($forcemachine, $forcecpu);
5823     if ($snapname) {
5824         my $snapshot = $conf->{snapshots}->{$snapname};
5825         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5826
5827         # check for machine or CPU overrides in snapshot
5828         $forcemachine = $snapshot->{runningmachine};
5829         $forcecpu = $snapshot->{runningcpu};
5830
5831         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5832
5833         $conf = $snapshot;
5834     }
5835
5836     my $defaults = load_defaults();
5837
5838     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5839
5840     return PVE::Tools::cmd2string($cmd);
5841 }
5842
5843 sub vm_reset {
5844     my ($vmid, $skiplock) = @_;
5845
5846     PVE::QemuConfig->lock_config($vmid, sub {
5847
5848         my $conf = PVE::QemuConfig->load_config($vmid);
5849
5850         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5851
5852         mon_cmd($vmid, "system_reset");
5853     });
5854 }
5855
5856 sub get_vm_volumes {
5857     my ($conf) = @_;
5858
5859     my $vollist = [];
5860     foreach_volid($conf, sub {
5861         my ($volid, $attr) = @_;
5862
5863         return if $volid =~ m|^/|;
5864
5865         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5866         return if !$sid;
5867
5868         push @$vollist, $volid;
5869     });
5870
5871     return $vollist;
5872 }
5873
5874 sub vm_stop_cleanup {
5875     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5876
5877     eval {
5878
5879         if (!$keepActive) {
5880             my $vollist = get_vm_volumes($conf);
5881             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5882
5883             if (my $tpmdrive = $conf->{tpmstate0}) {
5884                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5885                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5886                 if ($storeid) {
5887                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5888                 }
5889             }
5890         }
5891
5892         foreach my $ext (qw(mon qmp pid vnc qga)) {
5893             unlink "/var/run/qemu-server/${vmid}.$ext";
5894         }
5895
5896         if ($conf->{ivshmem}) {
5897             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5898             # just delete it for now, VMs which have this already open do not
5899             # are affected, but new VMs will get a separated one. If this
5900             # becomes an issue we either add some sort of ref-counting or just
5901             # add a "don't delete on stop" flag to the ivshmem format.
5902             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5903         }
5904
5905         my $ids = [];
5906         foreach my $key (keys %$conf) {
5907             next if $key !~ m/^hostpci(\d+)$/;
5908             my $hostpciindex = $1;
5909             my $d = parse_hostpci($conf->{$key});
5910             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5911
5912             foreach my $pci (@{$d->{pciid}}) {
5913                 my $pciid = $pci->{id};
5914                 push @$ids, $pci->{id};
5915                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5916             }
5917         }
5918         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5919
5920         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5921     };
5922     warn $@ if $@; # avoid errors - just warn
5923 }
5924
5925 # call only in locked context
5926 sub _do_vm_stop {
5927     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5928
5929     my $pid = check_running($vmid, $nocheck);
5930     return if !$pid;
5931
5932     my $conf;
5933     if (!$nocheck) {
5934         $conf = PVE::QemuConfig->load_config($vmid);
5935         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5936         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5937             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5938             $timeout = $opts->{down} if $opts->{down};
5939         }
5940         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5941     }
5942
5943     eval {
5944         if ($shutdown) {
5945             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5946                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5947             } else {
5948                 mon_cmd($vmid, "system_powerdown");
5949             }
5950         } else {
5951             mon_cmd($vmid, "quit");
5952         }
5953     };
5954     my $err = $@;
5955
5956     if (!$err) {
5957         $timeout = 60 if !defined($timeout);
5958
5959         my $count = 0;
5960         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5961             $count++;
5962             sleep 1;
5963         }
5964
5965         if ($count >= $timeout) {
5966             if ($force) {
5967                 warn "VM still running - terminating now with SIGTERM\n";
5968                 kill 15, $pid;
5969             } else {
5970                 die "VM quit/powerdown failed - got timeout\n";
5971             }
5972         } else {
5973             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5974             return;
5975         }
5976     } else {
5977        if (!check_running($vmid, $nocheck)) {
5978             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5979             return;
5980        }
5981        if ($force) {
5982             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5983             kill 15, $pid;
5984         } else {
5985             die "VM quit/powerdown failed\n";
5986         }
5987     }
5988
5989     # wait again
5990     $timeout = 10;
5991
5992     my $count = 0;
5993     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5994         $count++;
5995         sleep 1;
5996     }
5997
5998     if ($count >= $timeout) {
5999         warn "VM still running - terminating now with SIGKILL\n";
6000         kill 9, $pid;
6001         sleep 1;
6002     }
6003
6004     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6005 }
6006
6007 # Note: use $nocheck to skip tests if VM configuration file exists.
6008 # We need that when migration VMs to other nodes (files already moved)
6009 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6010 sub vm_stop {
6011     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6012
6013     $force = 1 if !defined($force) && !$shutdown;
6014
6015     if ($migratedfrom){
6016         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6017         kill 15, $pid if $pid;
6018         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6019         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6020         return;
6021     }
6022
6023     PVE::QemuConfig->lock_config($vmid, sub {
6024         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6025    });
6026 }
6027
6028 sub vm_reboot {
6029     my ($vmid, $timeout) = @_;
6030
6031     PVE::QemuConfig->lock_config($vmid, sub {
6032         eval {
6033
6034             # only reboot if running, as qmeventd starts it again on a stop event
6035             return if !check_running($vmid);
6036
6037             create_reboot_request($vmid);
6038
6039             my $storecfg = PVE::Storage::config();
6040             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6041
6042         };
6043         if (my $err = $@) {
6044             # avoid that the next normal shutdown will be confused for a reboot
6045             clear_reboot_request($vmid);
6046             die $err;
6047         }
6048    });
6049 }
6050
6051 # note: if using the statestorage parameter, the caller has to check privileges
6052 sub vm_suspend {
6053     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6054
6055     my $conf;
6056     my $path;
6057     my $storecfg;
6058     my $vmstate;
6059
6060     PVE::QemuConfig->lock_config($vmid, sub {
6061
6062         $conf = PVE::QemuConfig->load_config($vmid);
6063
6064         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6065         PVE::QemuConfig->check_lock($conf)
6066             if !($skiplock || $is_backing_up);
6067
6068         die "cannot suspend to disk during backup\n"
6069             if $is_backing_up && $includestate;
6070
6071         if ($includestate) {
6072             $conf->{lock} = 'suspending';
6073             my $date = strftime("%Y-%m-%d", localtime(time()));
6074             $storecfg = PVE::Storage::config();
6075             if (!$statestorage) {
6076                 $statestorage = find_vmstate_storage($conf, $storecfg);
6077                 # check permissions for the storage
6078                 my $rpcenv = PVE::RPCEnvironment::get();
6079                 if ($rpcenv->{type} ne 'cli') {
6080                     my $authuser = $rpcenv->get_user();
6081                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6082                 }
6083             }
6084
6085
6086             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6087                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6088             $path = PVE::Storage::path($storecfg, $vmstate);
6089             PVE::QemuConfig->write_config($vmid, $conf);
6090         } else {
6091             mon_cmd($vmid, "stop");
6092         }
6093     });
6094
6095     if ($includestate) {
6096         # save vm state
6097         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6098
6099         eval {
6100             set_migration_caps($vmid, 1);
6101             mon_cmd($vmid, "savevm-start", statefile => $path);
6102             for(;;) {
6103                 my $state = mon_cmd($vmid, "query-savevm");
6104                 if (!$state->{status}) {
6105                     die "savevm not active\n";
6106                 } elsif ($state->{status} eq 'active') {
6107                     sleep(1);
6108                     next;
6109                 } elsif ($state->{status} eq 'completed') {
6110                     print "State saved, quitting\n";
6111                     last;
6112                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6113                     die "query-savevm failed with error '$state->{error}'\n"
6114                 } else {
6115                     die "query-savevm returned status '$state->{status}'\n";
6116                 }
6117             }
6118         };
6119         my $err = $@;
6120
6121         PVE::QemuConfig->lock_config($vmid, sub {
6122             $conf = PVE::QemuConfig->load_config($vmid);
6123             if ($err) {
6124                 # cleanup, but leave suspending lock, to indicate something went wrong
6125                 eval {
6126                     mon_cmd($vmid, "savevm-end");
6127                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6128                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6129                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6130                     PVE::QemuConfig->write_config($vmid, $conf);
6131                 };
6132                 warn $@ if $@;
6133                 die $err;
6134             }
6135
6136             die "lock changed unexpectedly\n"
6137                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6138
6139             mon_cmd($vmid, "quit");
6140             $conf->{lock} = 'suspended';
6141             PVE::QemuConfig->write_config($vmid, $conf);
6142         });
6143     }
6144 }
6145
6146 sub vm_resume {
6147     my ($vmid, $skiplock, $nocheck) = @_;
6148
6149     PVE::QemuConfig->lock_config($vmid, sub {
6150         my $res = mon_cmd($vmid, 'query-status');
6151         my $resume_cmd = 'cont';
6152         my $reset = 0;
6153
6154         if ($res->{status}) {
6155             return if $res->{status} eq 'running'; # job done, go home
6156             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6157             $reset = 1 if $res->{status} eq 'shutdown';
6158         }
6159
6160         if (!$nocheck) {
6161
6162             my $conf = PVE::QemuConfig->load_config($vmid);
6163
6164             PVE::QemuConfig->check_lock($conf)
6165                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6166         }
6167
6168         if ($reset) {
6169             # required if a VM shuts down during a backup and we get a resume
6170             # request before the backup finishes for example
6171             mon_cmd($vmid, "system_reset");
6172         }
6173         mon_cmd($vmid, $resume_cmd);
6174     });
6175 }
6176
6177 sub vm_sendkey {
6178     my ($vmid, $skiplock, $key) = @_;
6179
6180     PVE::QemuConfig->lock_config($vmid, sub {
6181
6182         my $conf = PVE::QemuConfig->load_config($vmid);
6183
6184         # there is no qmp command, so we use the human monitor command
6185         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6186         die $res if $res ne '';
6187     });
6188 }
6189
6190 # vzdump restore implementaion
6191
6192 sub tar_archive_read_firstfile {
6193     my $archive = shift;
6194
6195     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6196
6197     # try to detect archive type first
6198     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6199         die "unable to open file '$archive'\n";
6200     my $firstfile = <$fh>;
6201     kill 15, $pid;
6202     close $fh;
6203
6204     die "ERROR: archive contaions no data\n" if !$firstfile;
6205     chomp $firstfile;
6206
6207     return $firstfile;
6208 }
6209
6210 sub tar_restore_cleanup {
6211     my ($storecfg, $statfile) = @_;
6212
6213     print STDERR "starting cleanup\n";
6214
6215     if (my $fd = IO::File->new($statfile, "r")) {
6216         while (defined(my $line = <$fd>)) {
6217             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6218                 my $volid = $2;
6219                 eval {
6220                     if ($volid =~ m|^/|) {
6221                         unlink $volid || die 'unlink failed\n';
6222                     } else {
6223                         PVE::Storage::vdisk_free($storecfg, $volid);
6224                     }
6225                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6226                 };
6227                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6228             } else {
6229                 print STDERR "unable to parse line in statfile - $line";
6230             }
6231         }
6232         $fd->close();
6233     }
6234 }
6235
6236 sub restore_file_archive {
6237     my ($archive, $vmid, $user, $opts) = @_;
6238
6239     return restore_vma_archive($archive, $vmid, $user, $opts)
6240         if $archive eq '-';
6241
6242     my $info = PVE::Storage::archive_info($archive);
6243     my $format = $opts->{format} // $info->{format};
6244     my $comp = $info->{compression};
6245
6246     # try to detect archive format
6247     if ($format eq 'tar') {
6248         return restore_tar_archive($archive, $vmid, $user, $opts);
6249     } else {
6250         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6251     }
6252 }
6253
6254 # hepler to remove disks that will not be used after restore
6255 my $restore_cleanup_oldconf = sub {
6256     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6257
6258     my $kept_disks = {};
6259
6260     PVE::QemuConfig->foreach_volume($oldconf, sub {
6261         my ($ds, $drive) = @_;
6262
6263         return if drive_is_cdrom($drive, 1);
6264
6265         my $volid = $drive->{file};
6266         return if !$volid || $volid =~ m|^/|;
6267
6268         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6269         return if !$path || !$owner || ($owner != $vmid);
6270
6271         # Note: only delete disk we want to restore
6272         # other volumes will become unused
6273         if ($virtdev_hash->{$ds}) {
6274             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6275             if (my $err = $@) {
6276                 warn $err;
6277             }
6278         } else {
6279             $kept_disks->{$volid} = 1;
6280         }
6281     });
6282
6283     # after the restore we have no snapshots anymore
6284     for my $snapname (keys $oldconf->{snapshots}->%*) {
6285         my $snap = $oldconf->{snapshots}->{$snapname};
6286         if ($snap->{vmstate}) {
6287             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6288             if (my $err = $@) {
6289                 warn $err;
6290             }
6291         }
6292
6293         for my $volid (keys $kept_disks->%*) {
6294             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6295             warn $@ if $@;
6296         }
6297     }
6298 };
6299
6300 # Helper to parse vzdump backup device hints
6301 #
6302 # $rpcenv: Environment, used to ckeck storage permissions
6303 # $user: User ID, to check storage permissions
6304 # $storecfg: Storage configuration
6305 # $fh: the file handle for reading the configuration
6306 # $devinfo: should contain device sizes for all backu-up'ed devices
6307 # $options: backup options (pool, default storage)
6308 #
6309 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6310 my $parse_backup_hints = sub {
6311     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6312
6313     my $check_storage = sub { # assert if an image can be allocate
6314         my ($storeid, $scfg) = @_;
6315         die "Content type 'images' is not available on storage '$storeid'\n"
6316             if !$scfg->{content}->{images};
6317         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6318             if $user ne 'root@pam';
6319     };
6320
6321     my $virtdev_hash = {};
6322     while (defined(my $line = <$fh>)) {
6323         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6324             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6325             die "archive does not contain data for drive '$virtdev'\n"
6326                 if !$devinfo->{$devname};
6327
6328             if (defined($options->{storage})) {
6329                 $storeid = $options->{storage} || 'local';
6330             } elsif (!$storeid) {
6331                 $storeid = 'local';
6332             }
6333             $format = 'raw' if !$format;
6334             $devinfo->{$devname}->{devname} = $devname;
6335             $devinfo->{$devname}->{virtdev} = $virtdev;
6336             $devinfo->{$devname}->{format} = $format;
6337             $devinfo->{$devname}->{storeid} = $storeid;
6338
6339             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6340             $check_storage->($storeid, $scfg); # permission and content type check
6341
6342             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6343         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6344             my $virtdev = $1;
6345             my $drive = parse_drive($virtdev, $2);
6346
6347             if (drive_is_cloudinit($drive)) {
6348                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6349                 $storeid = $options->{storage} if defined ($options->{storage});
6350                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6351                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6352
6353                 $check_storage->($storeid, $scfg); # permission and content type check
6354
6355                 $virtdev_hash->{$virtdev} = {
6356                     format => $format,
6357                     storeid => $storeid,
6358                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6359                     is_cloudinit => 1,
6360                 };
6361             }
6362         }
6363     }
6364
6365     return $virtdev_hash;
6366 };
6367
6368 # Helper to allocate and activate all volumes required for a restore
6369 #
6370 # $storecfg: Storage configuration
6371 # $virtdev_hash: as returned by parse_backup_hints()
6372 #
6373 # Returns: { $virtdev => $volid }
6374 my $restore_allocate_devices = sub {
6375     my ($storecfg, $virtdev_hash, $vmid) = @_;
6376
6377     my $map = {};
6378     foreach my $virtdev (sort keys %$virtdev_hash) {
6379         my $d = $virtdev_hash->{$virtdev};
6380         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6381         my $storeid = $d->{storeid};
6382         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6383
6384         # test if requested format is supported
6385         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6386         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6387         $d->{format} = $defFormat if !$supported;
6388
6389         my $name;
6390         if ($d->{is_cloudinit}) {
6391             $name = "vm-$vmid-cloudinit";
6392             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6393             if ($scfg->{path}) {
6394                 $name .= ".$d->{format}";
6395             }
6396         }
6397
6398         my $volid = PVE::Storage::vdisk_alloc(
6399             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6400
6401         print STDERR "new volume ID is '$volid'\n";
6402         $d->{volid} = $volid;
6403
6404         PVE::Storage::activate_volumes($storecfg, [$volid]);
6405
6406         $map->{$virtdev} = $volid;
6407     }
6408
6409     return $map;
6410 };
6411
6412 sub restore_update_config_line {
6413     my ($cookie, $map, $line, $unique) = @_;
6414
6415     return '' if $line =~ m/^\#qmdump\#/;
6416     return '' if $line =~ m/^\#vzdump\#/;
6417     return '' if $line =~ m/^lock:/;
6418     return '' if $line =~ m/^unused\d+:/;
6419     return '' if $line =~ m/^parent:/;
6420
6421     my $res = '';
6422
6423     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6424     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6425         # try to convert old 1.X settings
6426         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6427         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6428             my ($model, $macaddr) = split(/\=/, $devconfig);
6429             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6430             my $net = {
6431                 model => $model,
6432                 bridge => "vmbr$ind",
6433                 macaddr => $macaddr,
6434             };
6435             my $netstr = print_net($net);
6436
6437             $res .= "net$cookie->{netcount}: $netstr\n";
6438             $cookie->{netcount}++;
6439         }
6440     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6441         my ($id, $netstr) = ($1, $2);
6442         my $net = parse_net($netstr);
6443         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6444         $netstr = print_net($net);
6445         $res .= "$id: $netstr\n";
6446     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6447         my $virtdev = $1;
6448         my $value = $3;
6449         my $di = parse_drive($virtdev, $value);
6450         if (defined($di->{backup}) && !$di->{backup}) {
6451             $res .= "#$line";
6452         } elsif ($map->{$virtdev}) {
6453             delete $di->{format}; # format can change on restore
6454             $di->{file} = $map->{$virtdev};
6455             $value = print_drive($di);
6456             $res .= "$virtdev: $value\n";
6457         } else {
6458             $res .= $line;
6459         }
6460     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6461         my $vmgenid = $1;
6462         if ($vmgenid ne '0') {
6463             # always generate a new vmgenid if there was a valid one setup
6464             $vmgenid = generate_uuid();
6465         }
6466         $res .= "vmgenid: $vmgenid\n";
6467     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6468         my ($uuid, $uuid_str);
6469         UUID::generate($uuid);
6470         UUID::unparse($uuid, $uuid_str);
6471         my $smbios1 = parse_smbios1($2);
6472         $smbios1->{uuid} = $uuid_str;
6473         $res .= $1.print_smbios1($smbios1)."\n";
6474     } else {
6475         $res .= $line;
6476     }
6477
6478     return $res;
6479 }
6480
6481 my $restore_deactivate_volumes = sub {
6482     my ($storecfg, $virtdev_hash) = @_;
6483
6484     my $vollist = [];
6485     for my $dev (values $virtdev_hash->%*) {
6486         push $vollist->@*, $dev->{volid} if $dev->{volid};
6487     }
6488
6489     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6490     print STDERR $@ if $@;
6491 };
6492
6493 my $restore_destroy_volumes = sub {
6494     my ($storecfg, $virtdev_hash) = @_;
6495
6496     for my $dev (values $virtdev_hash->%*) {
6497         my $volid = $dev->{volid} or next;
6498         eval {
6499             PVE::Storage::vdisk_free($storecfg, $volid);
6500             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6501         };
6502         print STDERR "unable to cleanup '$volid' - $@" if $@;
6503     }
6504 };
6505
6506 my $restore_merge_config = sub {
6507     my ($filename, $backup_conf_raw, $override_conf) = @_;
6508
6509     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6510     for my $key (keys $override_conf->%*) {
6511         $backup_conf->{$key} = $override_conf->{$key};
6512     }
6513
6514     return $backup_conf;
6515 };
6516
6517 sub scan_volids {
6518     my ($cfg, $vmid) = @_;
6519
6520     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6521
6522     my $volid_hash = {};
6523     foreach my $storeid (keys %$info) {
6524         foreach my $item (@{$info->{$storeid}}) {
6525             next if !($item->{volid} && $item->{size});
6526             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6527             $volid_hash->{$item->{volid}} = $item;
6528         }
6529     }
6530
6531     return $volid_hash;
6532 }
6533
6534 sub update_disk_config {
6535     my ($vmid, $conf, $volid_hash) = @_;
6536
6537     my $changes;
6538     my $prefix = "VM $vmid";
6539
6540     # used and unused disks
6541     my $referenced = {};
6542
6543     # Note: it is allowed to define multiple storages with same path (alias), so
6544     # we need to check both 'volid' and real 'path' (two different volid can point
6545     # to the same path).
6546
6547     my $referencedpath = {};
6548
6549     # update size info
6550     PVE::QemuConfig->foreach_volume($conf, sub {
6551         my ($opt, $drive) = @_;
6552
6553         my $volid = $drive->{file};
6554         return if !$volid;
6555         my $volume = $volid_hash->{$volid};
6556
6557         # mark volid as "in-use" for next step
6558         $referenced->{$volid} = 1;
6559         if ($volume && (my $path = $volume->{path})) {
6560             $referencedpath->{$path} = 1;
6561         }
6562
6563         return if drive_is_cdrom($drive);
6564         return if !$volume;
6565
6566         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6567         if (defined($updated)) {
6568             $changes = 1;
6569             $conf->{$opt} = print_drive($updated);
6570             print "$prefix ($opt): $msg\n";
6571         }
6572     });
6573
6574     # remove 'unusedX' entry if volume is used
6575     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6576         my ($opt, $drive) = @_;
6577
6578         my $volid = $drive->{file};
6579         return if !$volid;
6580
6581         my $path;
6582         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6583         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6584             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6585             $changes = 1;
6586             delete $conf->{$opt};
6587         }
6588
6589         $referenced->{$volid} = 1;
6590         $referencedpath->{$path} = 1 if $path;
6591     });
6592
6593     foreach my $volid (sort keys %$volid_hash) {
6594         next if $volid =~ m/vm-$vmid-state-/;
6595         next if $referenced->{$volid};
6596         my $path = $volid_hash->{$volid}->{path};
6597         next if !$path; # just to be sure
6598         next if $referencedpath->{$path};
6599         $changes = 1;
6600         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6601         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6602         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6603     }
6604
6605     return $changes;
6606 }
6607
6608 sub rescan {
6609     my ($vmid, $nolock, $dryrun) = @_;
6610
6611     my $cfg = PVE::Storage::config();
6612
6613     print "rescan volumes...\n";
6614     my $volid_hash = scan_volids($cfg, $vmid);
6615
6616     my $updatefn =  sub {
6617         my ($vmid) = @_;
6618
6619         my $conf = PVE::QemuConfig->load_config($vmid);
6620
6621         PVE::QemuConfig->check_lock($conf);
6622
6623         my $vm_volids = {};
6624         foreach my $volid (keys %$volid_hash) {
6625             my $info = $volid_hash->{$volid};
6626             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6627         }
6628
6629         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6630
6631         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6632     };
6633
6634     if (defined($vmid)) {
6635         if ($nolock) {
6636             &$updatefn($vmid);
6637         } else {
6638             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6639         }
6640     } else {
6641         my $vmlist = config_list();
6642         foreach my $vmid (keys %$vmlist) {
6643             if ($nolock) {
6644                 &$updatefn($vmid);
6645             } else {
6646                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6647             }
6648         }
6649     }
6650 }
6651
6652 sub restore_proxmox_backup_archive {
6653     my ($archive, $vmid, $user, $options) = @_;
6654
6655     my $storecfg = PVE::Storage::config();
6656
6657     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6658     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6659
6660     my $fingerprint = $scfg->{fingerprint};
6661     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6662
6663     my $repo = PVE::PBSClient::get_repository($scfg);
6664     my $namespace = $scfg->{namespace};
6665
6666     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6667     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6668     local $ENV{PBS_PASSWORD} = $password;
6669     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6670
6671     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6672         PVE::Storage::parse_volname($storecfg, $archive);
6673
6674     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6675
6676     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6677
6678     my $tmpdir = "/var/tmp/vzdumptmp$$";
6679     rmtree $tmpdir;
6680     mkpath $tmpdir;
6681
6682     my $conffile = PVE::QemuConfig->config_file($vmid);
6683      # disable interrupts (always do cleanups)
6684     local $SIG{INT} =
6685         local $SIG{TERM} =
6686         local $SIG{QUIT} =
6687         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6688
6689     # Note: $oldconf is undef if VM does not exists
6690     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6691     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6692     my $new_conf_raw = '';
6693
6694     my $rpcenv = PVE::RPCEnvironment::get();
6695     my $devinfo = {}; # info about drives included in backup
6696     my $virtdev_hash = {}; # info about allocated drives
6697
6698     eval {
6699         # enable interrupts
6700         local $SIG{INT} =
6701             local $SIG{TERM} =
6702             local $SIG{QUIT} =
6703             local $SIG{HUP} =
6704             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6705
6706         my $cfgfn = "$tmpdir/qemu-server.conf";
6707         my $firewall_config_fn = "$tmpdir/fw.conf";
6708         my $index_fn = "$tmpdir/index.json";
6709
6710         my $cmd = "restore";
6711
6712         my $param = [$pbs_backup_name, "index.json", $index_fn];
6713         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6714         my $index = PVE::Tools::file_get_contents($index_fn);
6715         $index = decode_json($index);
6716
6717         foreach my $info (@{$index->{files}}) {
6718             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6719                 my $devname = $1;
6720                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6721                     $devinfo->{$devname}->{size} = $1;
6722                 } else {
6723                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6724                 }
6725             }
6726         }
6727
6728         my $is_qemu_server_backup = scalar(
6729             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6730         );
6731         if (!$is_qemu_server_backup) {
6732             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6733         }
6734         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6735
6736         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6737         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6738
6739         if ($has_firewall_config) {
6740             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6741             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6742
6743             my $pve_firewall_dir = '/etc/pve/firewall';
6744             mkdir $pve_firewall_dir; # make sure the dir exists
6745             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6746         }
6747
6748         my $fh = IO::File->new($cfgfn, "r") ||
6749             die "unable to read qemu-server.conf - $!\n";
6750
6751         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6752
6753         # fixme: rate limit?
6754
6755         # create empty/temp config
6756         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6757
6758         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6759
6760         # allocate volumes
6761         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6762
6763         foreach my $virtdev (sort keys %$virtdev_hash) {
6764             my $d = $virtdev_hash->{$virtdev};
6765             next if $d->{is_cloudinit}; # no need to restore cloudinit
6766
6767             # this fails if storage is unavailable
6768             my $volid = $d->{volid};
6769             my $path = PVE::Storage::path($storecfg, $volid);
6770
6771             # for live-restore we only want to preload the efidisk and TPM state
6772             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6773
6774             my @ns_arg;
6775             if (defined(my $ns = $scfg->{namespace})) {
6776                 @ns_arg = ('--ns', $ns);
6777             }
6778
6779             my $pbs_restore_cmd = [
6780                 '/usr/bin/pbs-restore',
6781                 '--repository', $repo,
6782                 @ns_arg,
6783                 $pbs_backup_name,
6784                 "$d->{devname}.img.fidx",
6785                 $path,
6786                 '--verbose',
6787                 ];
6788
6789             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6790             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6791
6792             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6793                 push @$pbs_restore_cmd, '--skip-zero';
6794             }
6795
6796             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6797             print "restore proxmox backup image: $dbg_cmdstring\n";
6798             run_command($pbs_restore_cmd);
6799         }
6800
6801         $fh->seek(0, 0) || die "seek failed - $!\n";
6802
6803         my $cookie = { netcount => 0 };
6804         while (defined(my $line = <$fh>)) {
6805             $new_conf_raw .= restore_update_config_line(
6806                 $cookie,
6807                 $map,
6808                 $line,
6809                 $options->{unique},
6810             );
6811         }
6812
6813         $fh->close();
6814     };
6815     my $err = $@;
6816
6817     if ($err || !$options->{live}) {
6818         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6819     }
6820
6821     rmtree $tmpdir;
6822
6823     if ($err) {
6824         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6825         die $err;
6826     }
6827
6828     if ($options->{live}) {
6829         # keep lock during live-restore
6830         $new_conf_raw .= "\nlock: create";
6831     }
6832
6833     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6834     PVE::QemuConfig->write_config($vmid, $new_conf);
6835
6836     eval { rescan($vmid, 1); };
6837     warn $@ if $@;
6838
6839     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6840
6841     if ($options->{live}) {
6842         # enable interrupts
6843         local $SIG{INT} =
6844             local $SIG{TERM} =
6845             local $SIG{QUIT} =
6846             local $SIG{HUP} =
6847             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6848
6849         my $conf = PVE::QemuConfig->load_config($vmid);
6850         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6851
6852         # these special drives are already restored before start
6853         delete $devinfo->{'drive-efidisk0'};
6854         delete $devinfo->{'drive-tpmstate0-backup'};
6855
6856         my $pbs_opts = {
6857             repo => $repo,
6858             keyfile => $keyfile,
6859             snapshot => $pbs_backup_name,
6860             namespace => $namespace,
6861         };
6862         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6863
6864         PVE::QemuConfig->remove_lock($vmid, "create");
6865     }
6866 }
6867
6868 sub pbs_live_restore {
6869     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6870
6871     print "starting VM for live-restore\n";
6872     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6873
6874     my $pbs_backing = {};
6875     for my $ds (keys %$restored_disks) {
6876         $ds =~ m/^drive-(.*)$/;
6877         my $confname = $1;
6878         $pbs_backing->{$confname} = {
6879             repository => $opts->{repo},
6880             snapshot => $opts->{snapshot},
6881             archive => "$ds.img.fidx",
6882         };
6883         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6884         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6885
6886         my $drive = parse_drive($confname, $conf->{$confname});
6887         print "restoring '$ds' to '$drive->{file}'\n";
6888     }
6889
6890     my $drives_streamed = 0;
6891     eval {
6892         # make sure HA doesn't interrupt our restore by stopping the VM
6893         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6894             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6895         }
6896
6897         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6898         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6899         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6900
6901         my $qmeventd_fd = register_qmeventd_handle($vmid);
6902
6903         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6904         # this will effectively collapse the backing image chain consisting of
6905         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6906         # removes itself once all backing images vanish with 'auto-remove=on')
6907         my $jobs = {};
6908         for my $ds (sort keys %$restored_disks) {
6909             my $job_id = "restore-$ds";
6910             mon_cmd($vmid, 'block-stream',
6911                 'job-id' => $job_id,
6912                 device => "$ds",
6913             );
6914             $jobs->{$job_id} = {};
6915         }
6916
6917         mon_cmd($vmid, 'cont');
6918         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6919
6920         print "restore-drive jobs finished successfully, removing all tracking block devices"
6921             ." to disconnect from Proxmox Backup Server\n";
6922
6923         for my $ds (sort keys %$restored_disks) {
6924             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6925         }
6926
6927         close($qmeventd_fd);
6928     };
6929
6930     my $err = $@;
6931
6932     if ($err) {
6933         warn "An error occurred during live-restore: $err\n";
6934         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6935         die "live-restore failed\n";
6936     }
6937 }
6938
6939 sub restore_vma_archive {
6940     my ($archive, $vmid, $user, $opts, $comp) = @_;
6941
6942     my $readfrom = $archive;
6943
6944     my $cfg = PVE::Storage::config();
6945     my $commands = [];
6946     my $bwlimit = $opts->{bwlimit};
6947
6948     my $dbg_cmdstring = '';
6949     my $add_pipe = sub {
6950         my ($cmd) = @_;
6951         push @$commands, $cmd;
6952         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6953         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6954         $readfrom = '-';
6955     };
6956
6957     my $input = undef;
6958     if ($archive eq '-') {
6959         $input = '<&STDIN';
6960     } else {
6961         # If we use a backup from a PVE defined storage we also consider that
6962         # storage's rate limit:
6963         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6964         if (defined($volid)) {
6965             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6966             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6967             if ($readlimit) {
6968                 print STDERR "applying read rate limit: $readlimit\n";
6969                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6970                 $add_pipe->($cstream);
6971             }
6972         }
6973     }
6974
6975     if ($comp) {
6976         my $info = PVE::Storage::decompressor_info('vma', $comp);
6977         my $cmd = $info->{decompressor};
6978         push @$cmd, $readfrom;
6979         $add_pipe->($cmd);
6980     }
6981
6982     my $tmpdir = "/var/tmp/vzdumptmp$$";
6983     rmtree $tmpdir;
6984
6985     # disable interrupts (always do cleanups)
6986     local $SIG{INT} =
6987         local $SIG{TERM} =
6988         local $SIG{QUIT} =
6989         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6990
6991     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6992     POSIX::mkfifo($mapfifo, 0600);
6993     my $fifofh;
6994     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6995
6996     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6997
6998     my $oldtimeout;
6999     my $timeout = 5;
7000
7001     my $devinfo = {}; # info about drives included in backup
7002     my $virtdev_hash = {}; # info about allocated drives
7003
7004     my $rpcenv = PVE::RPCEnvironment::get();
7005
7006     my $conffile = PVE::QemuConfig->config_file($vmid);
7007
7008     # Note: $oldconf is undef if VM does not exist
7009     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7010     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7011     my $new_conf_raw = '';
7012
7013     my %storage_limits;
7014
7015     my $print_devmap = sub {
7016         my $cfgfn = "$tmpdir/qemu-server.conf";
7017
7018         # we can read the config - that is already extracted
7019         my $fh = IO::File->new($cfgfn, "r") ||
7020             die "unable to read qemu-server.conf - $!\n";
7021
7022         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7023         if (-f $fwcfgfn) {
7024             my $pve_firewall_dir = '/etc/pve/firewall';
7025             mkdir $pve_firewall_dir; # make sure the dir exists
7026             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7027         }
7028
7029         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7030
7031         foreach my $info (values %{$virtdev_hash}) {
7032             my $storeid = $info->{storeid};
7033             next if defined($storage_limits{$storeid});
7034
7035             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7036             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7037             $storage_limits{$storeid} = $limit * 1024;
7038         }
7039
7040         foreach my $devname (keys %$devinfo) {
7041             die "found no device mapping information for device '$devname'\n"
7042                 if !$devinfo->{$devname}->{virtdev};
7043         }
7044
7045         # create empty/temp config
7046         if ($oldconf) {
7047             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7048             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7049         }
7050
7051         # allocate volumes
7052         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7053
7054         # print restore information to $fifofh
7055         foreach my $virtdev (sort keys %$virtdev_hash) {
7056             my $d = $virtdev_hash->{$virtdev};
7057             next if $d->{is_cloudinit}; # no need to restore cloudinit
7058
7059             my $storeid = $d->{storeid};
7060             my $volid = $d->{volid};
7061
7062             my $map_opts = '';
7063             if (my $limit = $storage_limits{$storeid}) {
7064                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7065             }
7066
7067             my $write_zeros = 1;
7068             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7069                 $write_zeros = 0;
7070             }
7071
7072             my $path = PVE::Storage::path($cfg, $volid);
7073
7074             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7075
7076             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7077         }
7078
7079         $fh->seek(0, 0) || die "seek failed - $!\n";
7080
7081         my $cookie = { netcount => 0 };
7082         while (defined(my $line = <$fh>)) {
7083             $new_conf_raw .= restore_update_config_line(
7084                 $cookie,
7085                 $map,
7086                 $line,
7087                 $opts->{unique},
7088             );
7089         }
7090
7091         $fh->close();
7092     };
7093
7094     eval {
7095         # enable interrupts
7096         local $SIG{INT} =
7097             local $SIG{TERM} =
7098             local $SIG{QUIT} =
7099             local $SIG{HUP} =
7100             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7101         local $SIG{ALRM} = sub { die "got timeout\n"; };
7102
7103         $oldtimeout = alarm($timeout);
7104
7105         my $parser = sub {
7106             my $line = shift;
7107
7108             print "$line\n";
7109
7110             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7111                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7112                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7113             } elsif ($line =~ m/^CTIME: /) {
7114                 # we correctly received the vma config, so we can disable
7115                 # the timeout now for disk allocation (set to 10 minutes, so
7116                 # that we always timeout if something goes wrong)
7117                 alarm(600);
7118                 &$print_devmap();
7119                 print $fifofh "done\n";
7120                 my $tmp = $oldtimeout || 0;
7121                 $oldtimeout = undef;
7122                 alarm($tmp);
7123                 close($fifofh);
7124                 $fifofh = undef;
7125             }
7126         };
7127
7128         print "restore vma archive: $dbg_cmdstring\n";
7129         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7130     };
7131     my $err = $@;
7132
7133     alarm($oldtimeout) if $oldtimeout;
7134
7135     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7136
7137     close($fifofh) if $fifofh;
7138     unlink $mapfifo;
7139     rmtree $tmpdir;
7140
7141     if ($err) {
7142         $restore_destroy_volumes->($cfg, $virtdev_hash);
7143         die $err;
7144     }
7145
7146     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7147     PVE::QemuConfig->write_config($vmid, $new_conf);
7148
7149     eval { rescan($vmid, 1); };
7150     warn $@ if $@;
7151
7152     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7153 }
7154
7155 sub restore_tar_archive {
7156     my ($archive, $vmid, $user, $opts) = @_;
7157
7158     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7159         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7160         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7161     }
7162
7163     if ($archive ne '-') {
7164         my $firstfile = tar_archive_read_firstfile($archive);
7165         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7166             if $firstfile ne 'qemu-server.conf';
7167     }
7168
7169     my $storecfg = PVE::Storage::config();
7170
7171     # avoid zombie disks when restoring over an existing VM -> cleanup first
7172     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7173     # skiplock=1 because qmrestore has set the 'create' lock itself already
7174     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7175     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7176
7177     my $tocmd = "/usr/lib/qemu-server/qmextract";
7178
7179     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7180     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7181     $tocmd .= ' --prealloc' if $opts->{prealloc};
7182     $tocmd .= ' --info' if $opts->{info};
7183
7184     # tar option "xf" does not autodetect compression when read from STDIN,
7185     # so we pipe to zcat
7186     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7187         PVE::Tools::shellquote("--to-command=$tocmd");
7188
7189     my $tmpdir = "/var/tmp/vzdumptmp$$";
7190     mkpath $tmpdir;
7191
7192     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7193     local $ENV{VZDUMP_VMID} = $vmid;
7194     local $ENV{VZDUMP_USER} = $user;
7195
7196     my $conffile = PVE::QemuConfig->config_file($vmid);
7197     my $new_conf_raw = '';
7198
7199     # disable interrupts (always do cleanups)
7200     local $SIG{INT} =
7201         local $SIG{TERM} =
7202         local $SIG{QUIT} =
7203         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7204
7205     eval {
7206         # enable interrupts
7207         local $SIG{INT} =
7208             local $SIG{TERM} =
7209             local $SIG{QUIT} =
7210             local $SIG{HUP} =
7211             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7212
7213         if ($archive eq '-') {
7214             print "extracting archive from STDIN\n";
7215             run_command($cmd, input => "<&STDIN");
7216         } else {
7217             print "extracting archive '$archive'\n";
7218             run_command($cmd);
7219         }
7220
7221         return if $opts->{info};
7222
7223         # read new mapping
7224         my $map = {};
7225         my $statfile = "$tmpdir/qmrestore.stat";
7226         if (my $fd = IO::File->new($statfile, "r")) {
7227             while (defined (my $line = <$fd>)) {
7228                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7229                     $map->{$1} = $2 if $1;
7230                 } else {
7231                     print STDERR "unable to parse line in statfile - $line\n";
7232                 }
7233             }
7234             $fd->close();
7235         }
7236
7237         my $confsrc = "$tmpdir/qemu-server.conf";
7238
7239         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7240
7241         my $cookie = { netcount => 0 };
7242         while (defined (my $line = <$srcfd>)) {
7243             $new_conf_raw .= restore_update_config_line(
7244                 $cookie,
7245                 $map,
7246                 $line,
7247                 $opts->{unique},
7248             );
7249         }
7250
7251         $srcfd->close();
7252     };
7253     if (my $err = $@) {
7254         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7255         die $err;
7256     }
7257
7258     rmtree $tmpdir;
7259
7260     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7261
7262     PVE::Cluster::cfs_update(); # make sure we read new file
7263
7264     eval { rescan($vmid, 1); };
7265     warn $@ if $@;
7266 };
7267
7268 sub foreach_storage_used_by_vm {
7269     my ($conf, $func) = @_;
7270
7271     my $sidhash = {};
7272
7273     PVE::QemuConfig->foreach_volume($conf, sub {
7274         my ($ds, $drive) = @_;
7275         return if drive_is_cdrom($drive);
7276
7277         my $volid = $drive->{file};
7278
7279         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7280         $sidhash->{$sid} = $sid if $sid;
7281     });
7282
7283     foreach my $sid (sort keys %$sidhash) {
7284         &$func($sid);
7285     }
7286 }
7287
7288 my $qemu_snap_storage = {
7289     rbd => 1,
7290 };
7291 sub do_snapshots_with_qemu {
7292     my ($storecfg, $volid, $deviceid) = @_;
7293
7294     return if $deviceid =~ m/tpmstate0/;
7295
7296     my $storage_name = PVE::Storage::parse_volume_id($volid);
7297     my $scfg = $storecfg->{ids}->{$storage_name};
7298     die "could not find storage '$storage_name'\n" if !defined($scfg);
7299
7300     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7301         return 1;
7302     }
7303
7304     if ($volid =~ m/\.(qcow2|qed)$/){
7305         return 1;
7306     }
7307
7308     return;
7309 }
7310
7311 sub qga_check_running {
7312     my ($vmid, $nowarn) = @_;
7313
7314     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7315     if ($@) {
7316         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7317         return 0;
7318     }
7319     return 1;
7320 }
7321
7322 sub template_create {
7323     my ($vmid, $conf, $disk) = @_;
7324
7325     my $storecfg = PVE::Storage::config();
7326
7327     PVE::QemuConfig->foreach_volume($conf, sub {
7328         my ($ds, $drive) = @_;
7329
7330         return if drive_is_cdrom($drive);
7331         return if $disk && $ds ne $disk;
7332
7333         my $volid = $drive->{file};
7334         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7335
7336         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7337         $drive->{file} = $voliddst;
7338         $conf->{$ds} = print_drive($drive);
7339         PVE::QemuConfig->write_config($vmid, $conf);
7340     });
7341 }
7342
7343 sub convert_iscsi_path {
7344     my ($path) = @_;
7345
7346     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7347         my $portal = $1;
7348         my $target = $2;
7349         my $lun = $3;
7350
7351         my $initiator_name = get_initiator_name();
7352
7353         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7354                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7355     }
7356
7357     die "cannot convert iscsi path '$path', unkown format\n";
7358 }
7359
7360 sub qemu_img_convert {
7361     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7362
7363     my $storecfg = PVE::Storage::config();
7364     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7365     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7366
7367     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7368
7369     my $cachemode;
7370     my $src_path;
7371     my $src_is_iscsi = 0;
7372     my $src_format;
7373
7374     if ($src_storeid) {
7375         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7376         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7377         $src_format = qemu_img_format($src_scfg, $src_volname);
7378         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7379         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7380         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7381     } elsif (-f $src_volid || -b $src_volid) {
7382         $src_path = $src_volid;
7383         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7384             $src_format = $1;
7385         }
7386     }
7387
7388     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7389
7390     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7391     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7392     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7393     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7394
7395     my $cmd = [];
7396     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7397     push @$cmd, '-l', "snapshot.name=$snapname"
7398         if $snapname && $src_format && $src_format eq "qcow2";
7399     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7400     push @$cmd, '-T', $cachemode if defined($cachemode);
7401
7402     if ($src_is_iscsi) {
7403         push @$cmd, '--image-opts';
7404         $src_path = convert_iscsi_path($src_path);
7405     } elsif ($src_format) {
7406         push @$cmd, '-f', $src_format;
7407     }
7408
7409     if ($dst_is_iscsi) {
7410         push @$cmd, '--target-image-opts';
7411         $dst_path = convert_iscsi_path($dst_path);
7412     } else {
7413         push @$cmd, '-O', $dst_format;
7414     }
7415
7416     push @$cmd, $src_path;
7417
7418     if (!$dst_is_iscsi && $is_zero_initialized) {
7419         push @$cmd, "zeroinit:$dst_path";
7420     } else {
7421         push @$cmd, $dst_path;
7422     }
7423
7424     my $parser = sub {
7425         my $line = shift;
7426         if($line =~ m/\((\S+)\/100\%\)/){
7427             my $percent = $1;
7428             my $transferred = int($size * $percent / 100);
7429             my $total_h = render_bytes($size, 1);
7430             my $transferred_h = render_bytes($transferred, 1);
7431
7432             print "transferred $transferred_h of $total_h ($percent%)\n";
7433         }
7434
7435     };
7436
7437     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7438     my $err = $@;
7439     die "copy failed: $err" if $err;
7440 }
7441
7442 sub qemu_img_format {
7443     my ($scfg, $volname) = @_;
7444
7445     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7446         return $1;
7447     } else {
7448         return "raw";
7449     }
7450 }
7451
7452 sub qemu_drive_mirror {
7453     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7454
7455     $jobs = {} if !$jobs;
7456
7457     my $qemu_target;
7458     my $format;
7459     $jobs->{"drive-$drive"} = {};
7460
7461     if ($dst_volid =~ /^nbd:/) {
7462         $qemu_target = $dst_volid;
7463         $format = "nbd";
7464     } else {
7465         my $storecfg = PVE::Storage::config();
7466         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7467
7468         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7469
7470         $format = qemu_img_format($dst_scfg, $dst_volname);
7471
7472         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7473
7474         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7475     }
7476
7477     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7478     $opts->{format} = $format if $format;
7479
7480     if (defined($src_bitmap)) {
7481         $opts->{sync} = 'incremental';
7482         $opts->{bitmap} = $src_bitmap;
7483         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7484     }
7485
7486     if (defined($bwlimit)) {
7487         $opts->{speed} = $bwlimit * 1024;
7488         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7489     } else {
7490         print "drive mirror is starting for drive-$drive\n";
7491     }
7492
7493     # if a job already runs for this device we get an error, catch it for cleanup
7494     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7495     if (my $err = $@) {
7496         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7497         warn "$@\n" if $@;
7498         die "mirroring error: $err\n";
7499     }
7500
7501     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7502 }
7503
7504 # $completion can be either
7505 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7506 # 'cancel': wait until all jobs are ready, block-job-cancel them
7507 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7508 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7509 sub qemu_drive_mirror_monitor {
7510     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7511
7512     $completion //= 'complete';
7513     $op //= "mirror";
7514
7515     eval {
7516         my $err_complete = 0;
7517
7518         my $starttime = time ();
7519         while (1) {
7520             die "block job ('$op') timed out\n" if $err_complete > 300;
7521
7522             my $stats = mon_cmd($vmid, "query-block-jobs");
7523             my $ctime = time();
7524
7525             my $running_jobs = {};
7526             for my $stat (@$stats) {
7527                 next if $stat->{type} ne $op;
7528                 $running_jobs->{$stat->{device}} = $stat;
7529             }
7530
7531             my $readycounter = 0;
7532
7533             for my $job_id (sort keys %$jobs) {
7534                 my $job = $running_jobs->{$job_id};
7535
7536                 my $vanished = !defined($job);
7537                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7538                 if($complete || ($vanished && $completion eq 'auto')) {
7539                     print "$job_id: $op-job finished\n";
7540                     delete $jobs->{$job_id};
7541                     next;
7542                 }
7543
7544                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7545
7546                 my $busy = $job->{busy};
7547                 my $ready = $job->{ready};
7548                 if (my $total = $job->{len}) {
7549                     my $transferred = $job->{offset} || 0;
7550                     my $remaining = $total - $transferred;
7551                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7552
7553                     my $duration = $ctime - $starttime;
7554                     my $total_h = render_bytes($total, 1);
7555                     my $transferred_h = render_bytes($transferred, 1);
7556
7557                     my $status = sprintf(
7558                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7559                         render_duration($duration),
7560                     );
7561
7562                     if ($ready) {
7563                         if ($busy) {
7564                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7565                         } else {
7566                             $status .= ", ready";
7567                         }
7568                     }
7569                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7570                     $jobs->{$job_id}->{ready} = $ready;
7571                 }
7572
7573                 $readycounter++ if $job->{ready};
7574             }
7575
7576             last if scalar(keys %$jobs) == 0;
7577
7578             if ($readycounter == scalar(keys %$jobs)) {
7579                 print "all '$op' jobs are ready\n";
7580
7581                 # do the complete later (or has already been done)
7582                 last if $completion eq 'skip' || $completion eq 'auto';
7583
7584                 if ($vmiddst && $vmiddst != $vmid) {
7585                     my $agent_running = $qga && qga_check_running($vmid);
7586                     if ($agent_running) {
7587                         print "freeze filesystem\n";
7588                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7589                         warn $@ if $@;
7590                     } else {
7591                         print "suspend vm\n";
7592                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7593                         warn $@ if $@;
7594                     }
7595
7596                     # if we clone a disk for a new target vm, we don't switch the disk
7597                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7598
7599                     if ($agent_running) {
7600                         print "unfreeze filesystem\n";
7601                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7602                         warn $@ if $@;
7603                     } else {
7604                         print "resume vm\n";
7605                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7606                         warn $@ if $@;
7607                     }
7608
7609                     last;
7610                 } else {
7611
7612                     for my $job_id (sort keys %$jobs) {
7613                         # try to switch the disk if source and destination are on the same guest
7614                         print "$job_id: Completing block job_id...\n";
7615
7616                         my $op;
7617                         if ($completion eq 'complete') {
7618                             $op = 'block-job-complete';
7619                         } elsif ($completion eq 'cancel') {
7620                             $op = 'block-job-cancel';
7621                         } else {
7622                             die "invalid completion value: $completion\n";
7623                         }
7624                         eval { mon_cmd($vmid, $op, device => $job_id) };
7625                         if ($@ =~ m/cannot be completed/) {
7626                             print "$job_id: block job cannot be completed, trying again.\n";
7627                             $err_complete++;
7628                         }else {
7629                             print "$job_id: Completed successfully.\n";
7630                             $jobs->{$job_id}->{complete} = 1;
7631                         }
7632                     }
7633                 }
7634             }
7635             sleep 1;
7636         }
7637     };
7638     my $err = $@;
7639
7640     if ($err) {
7641         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7642         die "block job ($op) error: $err";
7643     }
7644 }
7645
7646 sub qemu_blockjobs_cancel {
7647     my ($vmid, $jobs) = @_;
7648
7649     foreach my $job (keys %$jobs) {
7650         print "$job: Cancelling block job\n";
7651         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7652         $jobs->{$job}->{cancel} = 1;
7653     }
7654
7655     while (1) {
7656         my $stats = mon_cmd($vmid, "query-block-jobs");
7657
7658         my $running_jobs = {};
7659         foreach my $stat (@$stats) {
7660             $running_jobs->{$stat->{device}} = $stat;
7661         }
7662
7663         foreach my $job (keys %$jobs) {
7664
7665             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7666                 print "$job: Done.\n";
7667                 delete $jobs->{$job};
7668             }
7669         }
7670
7671         last if scalar(keys %$jobs) == 0;
7672
7673         sleep 1;
7674     }
7675 }
7676
7677 sub clone_disk {
7678     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7679
7680     my ($vmid, $running) = $source->@{qw(vmid running)};
7681     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7682
7683     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7684     my ($storage, $format) = $dest->@{qw(storage format)};
7685
7686     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7687
7688     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7689         die "cloning from/to EFI disk requires EFI disk\n"
7690             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7691         die "cloning from/to TPM state requires TPM state\n"
7692             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7693
7694         # This would lead to two device nodes in QEMU pointing to the same backing image!
7695         die "cannot change drive name when cloning disk from/to the same VM\n"
7696             if $use_drive_mirror && $vmid == $newvmid;
7697     }
7698
7699     die "cannot move TPM state while VM is running\n"
7700         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7701
7702     my $newvolid;
7703
7704     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7705     print "$src_drivename " if $src_drivename;
7706     print "($drive->{file})\n";
7707
7708     if (!$full) {
7709         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7710         push @$newvollist, $newvolid;
7711     } else {
7712
7713         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7714         $storeid = $storage if $storage;
7715
7716         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7717
7718         my $name = undef;
7719         my $size = undef;
7720         if (drive_is_cloudinit($drive)) {
7721             $name = "vm-$newvmid-cloudinit";
7722             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7723             if ($scfg->{path}) {
7724                 $name .= ".$dst_format";
7725             }
7726             $snapname = undef;
7727             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7728         } elsif ($dst_drivename eq 'efidisk0') {
7729             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7730         } elsif ($dst_drivename eq 'tpmstate0') {
7731             $dst_format = 'raw';
7732             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7733         } else {
7734             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7735         }
7736         $newvolid = PVE::Storage::vdisk_alloc(
7737             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7738         );
7739         push @$newvollist, $newvolid;
7740
7741         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7742
7743         if (drive_is_cloudinit($drive)) {
7744             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7745             # if this is the case, we have to complete any block-jobs still there from
7746             # previous drive-mirrors
7747             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7748                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7749             }
7750             goto no_data_clone;
7751         }
7752
7753         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7754         if ($use_drive_mirror) {
7755             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7756                 $completion, $qga, $bwlimit);
7757         } else {
7758             # TODO: handle bwlimits
7759             if ($dst_drivename eq 'efidisk0') {
7760                 # the relevant data on the efidisk may be smaller than the source
7761                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7762                 # that is given by the OVMF_VARS.fd
7763                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7764                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7765
7766                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7767
7768                 # better for Ceph if block size is not too small, see bug #3324
7769                 my $bs = 1024*1024;
7770
7771                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7772
7773                 if ($src_format eq 'qcow2' && $snapname) {
7774                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7775                         if !min_version(kvm_user_version(), 6, 2);
7776                     push $cmd->@*, '-l', $snapname;
7777                 }
7778                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7779                 run_command($cmd);
7780             } else {
7781                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7782             }
7783         }
7784     }
7785
7786 no_data_clone:
7787     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7788
7789     my $disk = dclone($drive);
7790     delete $disk->{format};
7791     $disk->{file} = $newvolid;
7792     $disk->{size} = $size if defined($size);
7793
7794     return $disk;
7795 }
7796
7797 sub get_running_qemu_version {
7798     my ($vmid) = @_;
7799     my $res = mon_cmd($vmid, "query-version");
7800     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7801 }
7802
7803 sub qemu_use_old_bios_files {
7804     my ($machine_type) = @_;
7805
7806     return if !$machine_type;
7807
7808     my $use_old_bios_files = undef;
7809
7810     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7811         $machine_type = $1;
7812         $use_old_bios_files = 1;
7813     } else {
7814         my $version = extract_version($machine_type, kvm_user_version());
7815         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7816         # load new efi bios files on migration. So this hack is required to allow
7817         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7818         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7819         $use_old_bios_files = !min_version($version, 2, 4);
7820     }
7821
7822     return ($use_old_bios_files, $machine_type);
7823 }
7824
7825 sub get_efivars_size {
7826     my ($conf, $efidisk) = @_;
7827
7828     my $arch = get_vm_arch($conf);
7829     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7830     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7831     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7832     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7833     return -s $ovmf_vars;
7834 }
7835
7836 sub update_efidisk_size {
7837     my ($conf) = @_;
7838
7839     return if !defined($conf->{efidisk0});
7840
7841     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7842     $disk->{size} = get_efivars_size($conf);
7843     $conf->{efidisk0} = print_drive($disk);
7844
7845     return;
7846 }
7847
7848 sub update_tpmstate_size {
7849     my ($conf) = @_;
7850
7851     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7852     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7853     $conf->{tpmstate0} = print_drive($disk);
7854 }
7855
7856 sub create_efidisk($$$$$$$) {
7857     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7858
7859     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7860     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7861
7862     my $vars_size_b = -s $ovmf_vars;
7863     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7864     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7865     PVE::Storage::activate_volumes($storecfg, [$volid]);
7866
7867     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7868     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7869
7870     return ($volid, $size/1024);
7871 }
7872
7873 sub vm_iothreads_list {
7874     my ($vmid) = @_;
7875
7876     my $res = mon_cmd($vmid, 'query-iothreads');
7877
7878     my $iothreads = {};
7879     foreach my $iothread (@$res) {
7880         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7881     }
7882
7883     return $iothreads;
7884 }
7885
7886 sub scsihw_infos {
7887     my ($conf, $drive) = @_;
7888
7889     my $maxdev = 0;
7890
7891     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7892         $maxdev = 7;
7893     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7894         $maxdev = 1;
7895     } else {
7896         $maxdev = 256;
7897     }
7898
7899     my $controller = int($drive->{index} / $maxdev);
7900     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7901         ? "virtioscsi"
7902         : "scsihw";
7903
7904     return ($maxdev, $controller, $controller_prefix);
7905 }
7906
7907 sub windows_version {
7908     my ($ostype) = @_;
7909
7910     return 0 if !$ostype;
7911
7912     my $winversion = 0;
7913
7914     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7915         $winversion = 5;
7916     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7917         $winversion = 6;
7918     } elsif ($ostype =~ m/^win(\d+)$/) {
7919         $winversion = $1;
7920     }
7921
7922     return $winversion;
7923 }
7924
7925 sub resolve_dst_disk_format {
7926         my ($storecfg, $storeid, $src_volname, $format) = @_;
7927         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7928
7929         if (!$format) {
7930             # if no target format is specified, use the source disk format as hint
7931             if ($src_volname) {
7932                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7933                 $format = qemu_img_format($scfg, $src_volname);
7934             } else {
7935                 return $defFormat;
7936             }
7937         }
7938
7939         # test if requested format is supported - else use default
7940         my $supported = grep { $_ eq $format } @$validFormats;
7941         $format = $defFormat if !$supported;
7942         return $format;
7943 }
7944
7945 # NOTE: if this logic changes, please update docs & possibly gui logic
7946 sub find_vmstate_storage {
7947     my ($conf, $storecfg) = @_;
7948
7949     # first, return storage from conf if set
7950     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7951
7952     my ($target, $shared, $local);
7953
7954     foreach_storage_used_by_vm($conf, sub {
7955         my ($sid) = @_;
7956         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7957         my $dst = $scfg->{shared} ? \$shared : \$local;
7958         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7959     });
7960
7961     # second, use shared storage where VM has at least one disk
7962     # third, use local storage where VM has at least one disk
7963     # fall back to local storage
7964     $target = $shared // $local // 'local';
7965
7966     return $target;
7967 }
7968
7969 sub generate_uuid {
7970     my ($uuid, $uuid_str);
7971     UUID::generate($uuid);
7972     UUID::unparse($uuid, $uuid_str);
7973     return $uuid_str;
7974 }
7975
7976 sub generate_smbios1_uuid {
7977     return "uuid=".generate_uuid();
7978 }
7979
7980 sub nbd_stop {
7981     my ($vmid) = @_;
7982
7983     mon_cmd($vmid, 'nbd-server-stop');
7984 }
7985
7986 sub create_reboot_request {
7987     my ($vmid) = @_;
7988     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7989         or die "failed to create reboot trigger file: $!\n";
7990     close($fh);
7991 }
7992
7993 sub clear_reboot_request {
7994     my ($vmid) = @_;
7995     my $path = "/run/qemu-server/$vmid.reboot";
7996     my $res = 0;
7997
7998     $res = unlink($path);
7999     die "could not remove reboot request for $vmid: $!"
8000         if !$res && $! != POSIX::ENOENT;
8001
8002     return $res;
8003 }
8004
8005 sub bootorder_from_legacy {
8006     my ($conf, $bootcfg) = @_;
8007
8008     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8009     my $bootindex_hash = {};
8010     my $i = 1;
8011     foreach my $o (split(//, $boot)) {
8012         $bootindex_hash->{$o} = $i*100;
8013         $i++;
8014     }
8015
8016     my $bootorder = {};
8017
8018     PVE::QemuConfig->foreach_volume($conf, sub {
8019         my ($ds, $drive) = @_;
8020
8021         if (drive_is_cdrom ($drive, 1)) {
8022             if ($bootindex_hash->{d}) {
8023                 $bootorder->{$ds} = $bootindex_hash->{d};
8024                 $bootindex_hash->{d} += 1;
8025             }
8026         } elsif ($bootindex_hash->{c}) {
8027             $bootorder->{$ds} = $bootindex_hash->{c}
8028                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8029             $bootindex_hash->{c} += 1;
8030         }
8031     });
8032
8033     if ($bootindex_hash->{n}) {
8034         for (my $i = 0; $i < $MAX_NETS; $i++) {
8035             my $netname = "net$i";
8036             next if !$conf->{$netname};
8037             $bootorder->{$netname} = $bootindex_hash->{n};
8038             $bootindex_hash->{n} += 1;
8039         }
8040     }
8041
8042     return $bootorder;
8043 }
8044
8045 # Generate default device list for 'boot: order=' property. Matches legacy
8046 # default boot order, but with explicit device names. This is important, since
8047 # the fallback for when neither 'order' nor the old format is specified relies
8048 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8049 sub get_default_bootdevices {
8050     my ($conf) = @_;
8051
8052     my @ret = ();
8053
8054     # harddisk
8055     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8056     push @ret, $first if $first;
8057
8058     # cdrom
8059     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8060     push @ret, $first if $first;
8061
8062     # network
8063     for (my $i = 0; $i < $MAX_NETS; $i++) {
8064         my $netname = "net$i";
8065         next if !$conf->{$netname};
8066         push @ret, $netname;
8067         last;
8068     }
8069
8070     return \@ret;
8071 }
8072
8073 sub device_bootorder {
8074     my ($conf) = @_;
8075
8076     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8077
8078     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8079
8080     my $bootorder = {};
8081     if (!defined($boot) || $boot->{legacy}) {
8082         $bootorder = bootorder_from_legacy($conf, $boot);
8083     } elsif ($boot->{order}) {
8084         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8085         for my $dev (PVE::Tools::split_list($boot->{order})) {
8086             $bootorder->{$dev} = $i++;
8087         }
8088     }
8089
8090     return $bootorder;
8091 }
8092
8093 sub register_qmeventd_handle {
8094     my ($vmid) = @_;
8095
8096     my $fh;
8097     my $peer = "/var/run/qmeventd.sock";
8098     my $count = 0;
8099
8100     for (;;) {
8101         $count++;
8102         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8103         last if $fh;
8104         if ($! != EINTR && $! != EAGAIN) {
8105             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8106         }
8107         if ($count > 4) {
8108             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8109               . "after $count retries\n";
8110         }
8111         usleep(25000);
8112     }
8113
8114     # send handshake to mark VM as backing up
8115     print $fh to_json({vzdump => {vmid => "$vmid"}});
8116
8117     # return handle to be closed later when inhibit is no longer required
8118     return $fh;
8119 }
8120
8121 # bash completion helper
8122
8123 sub complete_backup_archives {
8124     my ($cmdname, $pname, $cvalue) = @_;
8125
8126     my $cfg = PVE::Storage::config();
8127
8128     my $storeid;
8129
8130     if ($cvalue =~ m/^([^:]+):/) {
8131         $storeid = $1;
8132     }
8133
8134     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8135
8136     my $res = [];
8137     foreach my $id (keys %$data) {
8138         foreach my $item (@{$data->{$id}}) {
8139             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8140             push @$res, $item->{volid} if defined($item->{volid});
8141         }
8142     }
8143
8144     return $res;
8145 }
8146
8147 my $complete_vmid_full = sub {
8148     my ($running) = @_;
8149
8150     my $idlist = vmstatus();
8151
8152     my $res = [];
8153
8154     foreach my $id (keys %$idlist) {
8155         my $d = $idlist->{$id};
8156         if (defined($running)) {
8157             next if $d->{template};
8158             next if $running && $d->{status} ne 'running';
8159             next if !$running && $d->{status} eq 'running';
8160         }
8161         push @$res, $id;
8162
8163     }
8164     return $res;
8165 };
8166
8167 sub complete_vmid {
8168     return &$complete_vmid_full();
8169 }
8170
8171 sub complete_vmid_stopped {
8172     return &$complete_vmid_full(0);
8173 }
8174
8175 sub complete_vmid_running {
8176     return &$complete_vmid_full(1);
8177 }
8178
8179 sub complete_storage {
8180
8181     my $cfg = PVE::Storage::config();
8182     my $ids = $cfg->{ids};
8183
8184     my $res = [];
8185     foreach my $sid (keys %$ids) {
8186         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8187         next if !$ids->{$sid}->{content}->{images};
8188         push @$res, $sid;
8189     }
8190
8191     return $res;
8192 }
8193
8194 sub complete_migration_storage {
8195     my ($cmd, $param, $current_value, $all_args) = @_;
8196
8197     my $targetnode = @$all_args[1];
8198
8199     my $cfg = PVE::Storage::config();
8200     my $ids = $cfg->{ids};
8201
8202     my $res = [];
8203     foreach my $sid (keys %$ids) {
8204         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8205         next if !$ids->{$sid}->{content}->{images};
8206         push @$res, $sid;
8207     }
8208
8209     return $res;
8210 }
8211
8212 sub vm_is_paused {
8213     my ($vmid) = @_;
8214     my $qmpstatus = eval {
8215         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8216         mon_cmd($vmid, "query-status");
8217     };
8218     warn "$@\n" if $@;
8219     return $qmpstatus && $qmpstatus->{status} eq "paused";
8220 }
8221
8222 sub check_volume_storage_type {
8223     my ($storecfg, $vol) = @_;
8224
8225     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8226     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8227     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8228
8229     die "storage '$storeid' does not support content-type '$vtype'\n"
8230         if !$scfg->{content}->{$vtype};
8231
8232     return 1;
8233 }
8234
8235 1;