PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RESTEnvironment qw(log_warn);
  40 use PVE::RPCEnvironment;
  41 use PVE::Storage;
  42 use PVE::SysFSTools;
  43 use PVE::Systemd;
  44 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  45
  46 use PVE::QMPClient;
  47 use PVE::QemuConfig;
  48 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  49 use PVE::QemuServer::Cloudinit;
  50 use PVE::QemuServer::CGroup;
  51 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  52 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  53 use PVE::QemuServer::Machine;
  54 use PVE::QemuServer::Memory;
  55 use PVE::QemuServer::Monitor qw(mon_cmd);
  56 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  57 use PVE::QemuServer::USB qw(parse_usb_device);
  58
  59 my $have_sdn;
  60 eval {
  61     require PVE::Network::SDN::Zones;
  62     $have_sdn = 1;
  63 };
  64
  65 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  66 my $OVMF = {
  67     x86_64 => {
  68         '4m-no-smm' => [
  69             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  70             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  71         ],
  72         '4m-no-smm-ms' => [
  73             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  74             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  75         ],
  76         '4m' => [
  77             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  78             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  79         ],
  80         '4m-ms' => [
  81             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  82             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  83         ],
  84         default => [
  85             "$EDK2_FW_BASE/OVMF_CODE.fd",
  86             "$EDK2_FW_BASE/OVMF_VARS.fd",
  87         ],
  88     },
  89     aarch64 => {
  90         default => [
  91             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  92             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  93         ],
  94     },
  95 };
  96
  97 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  98
  99 # Note about locking: we use flock on the config file protect against concurent actions.
 100 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 101 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 102 # But you can ignore this kind of lock with the --skiplock flag.
 103
 104 cfs_register_file('/qemu-server/',
 105                   \&parse_vm_config,
 106                   \&write_vm_config);
 107
 108 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 109     description => "Some command save/restore state from this location.",
 110     type => 'string',
 111     maxLength => 128,
 112     optional => 1,
 113 });
 114
 115 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 116         description => "Specifies the Qemu machine type.",
 117         type => 'string',
 118         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 119         maxLength => 40,
 120         optional => 1,
 121 });
 122
 123 #no warnings 'redefine';
 124
 125 my $nodename_cache;
 126 sub nodename {
 127     $nodename_cache //= PVE::INotify::nodename();
 128     return $nodename_cache;
 129 }
 130
 131 my $watchdog_fmt = {
 132     model => {
 133         default_key => 1,
 134         type => 'string',
 135         enum => [qw(i6300esb ib700)],
 136         description => "Watchdog type to emulate.",
 137         default => 'i6300esb',
 138         optional => 1,
 139     },
 140     action => {
 141         type => 'string',
 142         enum => [qw(reset shutdown poweroff pause debug none)],
 143         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 144         optional => 1,
 145     },
 146 };
 147 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 148
 149 my $agent_fmt = {
 150     enabled => {
 151         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 152         type => 'boolean',
 153         default => 0,
 154         default_key => 1,
 155     },
 156     fstrim_cloned_disks => {
 157         description => "Run fstrim after moving a disk or migrating the VM.",
 158         type => 'boolean',
 159         optional => 1,
 160         default => 0
 161     },
 162     type => {
 163         description => "Select the agent type",
 164         type => 'string',
 165         default => 'virtio',
 166         optional => 1,
 167         enum => [qw(virtio isa)],
 168     },
 169 };
 170
 171 my $vga_fmt = {
 172     type => {
 173         description => "Select the VGA type.",
 174         type => 'string',
 175         default => 'std',
 176         optional => 1,
 177         default_key => 1,
 178         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 179     },
 180     memory => {
 181         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 182         type => 'integer',
 183         optional => 1,
 184         minimum => 4,
 185         maximum => 512,
 186     },
 187 };
 188
 189 my $ivshmem_fmt = {
 190     size => {
 191         type => 'integer',
 192         minimum => 1,
 193         description => "The size of the file in MB.",
 194     },
 195     name => {
 196         type => 'string',
 197         pattern => '[a-zA-Z0-9\-]+',
 198         optional => 1,
 199         format_description => 'string',
 200         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 201     },
 202 };
 203
 204 my $audio_fmt = {
 205     device => {
 206         type => 'string',
 207         enum => [qw(ich9-intel-hda intel-hda AC97)],
 208         description =>  "Configure an audio device."
 209     },
 210     driver =>  {
 211         type => 'string',
 212         enum => ['spice', 'none'],
 213         default => 'spice',
 214         optional => 1,
 215         description => "Driver backend for the audio device."
 216     },
 217 };
 218
 219 my $spice_enhancements_fmt = {
 220     foldersharing => {
 221         type => 'boolean',
 222         optional => 1,
 223         default => '0',
 224         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 225     },
 226     videostreaming =>  {
 227         type => 'string',
 228         enum => ['off', 'all', 'filter'],
 229         default => 'off',
 230         optional => 1,
 231         description => "Enable video streaming. Uses compression for detected video streams."
 232     },
 233 };
 234
 235 my $rng_fmt = {
 236     source => {
 237         type => 'string',
 238         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 239         default_key => 1,
 240         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 241             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 242             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 243             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 244             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 245             ." a hardware RNG from the host.",
 246     },
 247     max_bytes => {
 248         type => 'integer',
 249         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 250             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 251             ." `0` to disable limiting (potentially dangerous!).",
 252         optional => 1,
 253
 254         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 255         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 256         # reading from /dev/urandom
 257         default => 1024,
 258     },
 259     period => {
 260         type => 'integer',
 261         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 262             ." the guest to retrieve another 'max_bytes' of entropy.",
 263         optional => 1,
 264         default => 1000,
 265     },
 266 };
 267
 268 my $meta_info_fmt = {
 269     'ctime' => {
 270         type => 'integer',
 271         description => "The guest creation timestamp as UNIX epoch time",
 272         minimum => 0,
 273         optional => 1,
 274     },
 275     'creation-qemu' => {
 276         type => 'string',
 277         description => "The QEMU (machine) version from the time this VM was created.",
 278         pattern => '\d+(\.\d+)+',
 279         optional => 1,
 280     },
 281 };
 282
 283 my $confdesc = {
 284     onboot => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Specifies whether a VM will be started during system bootup.",
 288         default => 0,
 289     },
 290     autostart => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Automatic restart after crash (currently ignored).",
 294         default => 0,
 295     },
 296     hotplug => {
 297         optional => 1,
 298         type => 'string', format => 'pve-hotplug-features',
 299         description => "Selectively enable hotplug features. This is a comma separated list of"
 300             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 301             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 302         default => 'network,disk,usb',
 303     },
 304     reboot => {
 305         optional => 1,
 306         type => 'boolean',
 307         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 308         default => 1,
 309     },
 310     lock => {
 311         optional => 1,
 312         type => 'string',
 313         description => "Lock/unlock the VM.",
 314         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 315     },
 316     cpulimit => {
 317         optional => 1,
 318         type => 'number',
 319         description => "Limit of CPU usage.",
 320         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 321             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 322         minimum => 0,
 323         maximum => 128,
 324         default => 0,
 325     },
 326     cpuunits => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 330         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 331             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 332             ." weights of all the other running VMs.",
 333         minimum => 1,
 334         maximum => 262144,
 335         default => 'cgroup v1: 1024, cgroup v2: 100',
 336     },
 337     memory => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 341             ." you use the balloon device.",
 342         minimum => 16,
 343         default => 512,
 344     },
 345     balloon => {
 346         optional => 1,
 347         type => 'integer',
 348         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 349         minimum => 0,
 350     },
 351     shares => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 355             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 356             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 357         minimum => 0,
 358         maximum => 50000,
 359         default => 1000,
 360     },
 361     keyboard => {
 362         optional => 1,
 363         type => 'string',
 364         description => "Keyboard layout for VNC server. This option is generally not required and"
 365         ." is often better handled from within the guest OS.",
 366         enum => PVE::Tools::kvmkeymaplist(),
 367         default => undef,
 368     },
 369     name => {
 370         optional => 1,
 371         type => 'string', format => 'dns-name',
 372         description => "Set a name for the VM. Only used on the configuration web interface.",
 373     },
 374     scsihw => {
 375         optional => 1,
 376         type => 'string',
 377         description => "SCSI controller model",
 378         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 379         default => 'lsi',
 380     },
 381     description => {
 382         optional => 1,
 383         type => 'string',
 384         description => "Description for the VM. Shown in the web-interface VM's summary."
 385             ." This is saved as comment inside the configuration file.",
 386         maxLength => 1024 * 8,
 387     },
 388     ostype => {
 389         optional => 1,
 390         type => 'string',
 391         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 392         description => "Specify guest operating system.",
 393         verbose_description => <<EODESC,
 394 Specify guest operating system. This is used to enable special
 395 optimization/features for specific operating systems:
 396
 397 [horizontal]
 398 other;; unspecified OS
 399 wxp;; Microsoft Windows XP
 400 w2k;; Microsoft Windows 2000
 401 w2k3;; Microsoft Windows 2003
 402 w2k8;; Microsoft Windows 2008
 403 wvista;; Microsoft Windows Vista
 404 win7;; Microsoft Windows 7
 405 win8;; Microsoft Windows 8/2012/2012r2
 406 win10;; Microsoft Windows 10/2016/2019
 407 win11;; Microsoft Windows 11/2022
 408 l24;; Linux 2.4 Kernel
 409 l26;; Linux 2.6 - 5.X Kernel
 410 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 411 EODESC
 412     },
 413     boot => {
 414         optional => 1,
 415         type => 'string', format => 'pve-qm-boot',
 416         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 417             ." key or 'legacy=' is deprecated.",
 418     },
 419     bootdisk => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-bootdisk',
 422         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 423         pattern => '(ide|sata|scsi|virtio)\d+',
 424     },
 425     smp => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of CPUs. Please use option -sockets instead.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     sockets => {
 433         optional => 1,
 434         type => 'integer',
 435         description => "The number of CPU sockets.",
 436         minimum => 1,
 437         default => 1,
 438     },
 439     cores => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "The number of cores per socket.",
 443         minimum => 1,
 444         default => 1,
 445     },
 446     numa => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable NUMA.",
 450         default => 0,
 451     },
 452     hugepages => {
 453         optional => 1,
 454         type => 'string',
 455         description => "Enable/disable hugepages memory.",
 456         enum => [qw(any 2 1024)],
 457     },
 458     keephugepages => {
 459         optional => 1,
 460         type => 'boolean',
 461         default => 0,
 462         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 463             ." after VM shutdown and can be used for subsequent starts.",
 464     },
 465     vcpus => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "Number of hotplugged vcpus.",
 469         minimum => 1,
 470         default => 0,
 471     },
 472     acpi => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable ACPI.",
 476         default => 1,
 477     },
 478     agent => {
 479         optional => 1,
 480         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 481         type => 'string',
 482         format => $agent_fmt,
 483     },
 484     kvm => {
 485         optional => 1,
 486         type => 'boolean',
 487         description => "Enable/disable KVM hardware virtualization.",
 488         default => 1,
 489     },
 490     tdf => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable time drift fix.",
 494         default => 0,
 495     },
 496     localtime => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 500             ." the `ostype` indicates a Microsoft Windows OS.",
 501     },
 502     freeze => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 506     },
 507     vga => {
 508         optional => 1,
 509         type => 'string', format => $vga_fmt,
 510         description => "Configure the VGA hardware.",
 511         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 512             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 513             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 514             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 515             ." display server. For win* OS you can select how many independent displays you want,"
 516             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 517             ." using a serial device as terminal.",
 518     },
 519     watchdog => {
 520         optional => 1,
 521         type => 'string', format => 'pve-qm-watchdog',
 522         description => "Create a virtual hardware watchdog device.",
 523         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 524             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 525             ." else the watchdog will reset the guest (or execute the respective action specified)",
 526     },
 527     startdate => {
 528         optional => 1,
 529         type => 'string',
 530         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 531         description => "Set the initial date of the real time clock. Valid format for date are:"
 532             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 533         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 534         default => 'now',
 535     },
 536     startup =>  get_standard_option('pve-startup-order'),
 537     template => {
 538         optional => 1,
 539         type => 'boolean',
 540         description => "Enable/disable Template.",
 541         default => 0,
 542     },
 543     args => {
 544         optional => 1,
 545         type => 'string',
 546         description => "Arbitrary arguments passed to kvm.",
 547         verbose_description => <<EODESCR,
 548 Arbitrary arguments passed to kvm, for example:
 549
 550 args: -no-reboot -no-hpet
 551
 552 NOTE: this option is for experts only.
 553 EODESCR
 554     },
 555     tablet => {
 556         optional => 1,
 557         type => 'boolean',
 558         default => 1,
 559         description => "Enable/disable the USB tablet device.",
 560         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 561             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 562             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 563             ." may consider disabling this to save some context switches. This is turned off by"
 564             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 565     },
 566     migrate_speed => {
 567         optional => 1,
 568         type => 'integer',
 569         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 570         minimum => 0,
 571         default => 0,
 572     },
 573     migrate_downtime => {
 574         optional => 1,
 575         type => 'number',
 576         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 577         minimum => 0,
 578         default => 0.1,
 579     },
 580     cdrom => {
 581         optional => 1,
 582         type => 'string', format => 'pve-qm-ide',
 583         typetext => '<volume>',
 584         description => "This is an alias for option -ide2",
 585     },
 586     cpu => {
 587         optional => 1,
 588         description => "Emulated CPU type.",
 589         type => 'string',
 590         format => 'pve-vm-cpu-conf',
 591     },
 592     parent => get_standard_option('pve-snapshot-name', {
 593         optional => 1,
 594         description => "Parent snapshot name. This is used internally, and should not be modified.",
 595     }),
 596     snaptime => {
 597         optional => 1,
 598         description => "Timestamp for snapshots.",
 599         type => 'integer',
 600         minimum => 0,
 601     },
 602     vmstate => {
 603         optional => 1,
 604         type => 'string', format => 'pve-volume-id',
 605         description => "Reference to a volume which stores the VM state. This is used internally"
 606             ." for snapshots.",
 607     },
 608     vmstatestorage => get_standard_option('pve-storage-id', {
 609         description => "Default storage for VM state volumes/files.",
 610         optional => 1,
 611     }),
 612     runningmachine => get_standard_option('pve-qemu-machine', {
 613         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 614             ." for snapshots.",
 615     }),
 616     runningcpu => {
 617         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 618             ." internally for snapshots.",
 619         optional => 1,
 620         type => 'string',
 621         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 622         format_description => 'QEMU -cpu parameter'
 623     },
 624     machine => get_standard_option('pve-qemu-machine'),
 625     arch => {
 626         description => "Virtual processor architecture. Defaults to the host.",
 627         optional => 1,
 628         type => 'string',
 629         enum => [qw(x86_64 aarch64)],
 630     },
 631     smbios1 => {
 632         description => "Specify SMBIOS type 1 fields.",
 633         type => 'string', format => 'pve-qm-smbios1',
 634         maxLength => 512,
 635         optional => 1,
 636     },
 637     protection => {
 638         optional => 1,
 639         type => 'boolean',
 640         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 641             ." remove disk operations.",
 642         default => 0,
 643     },
 644     bios => {
 645         optional => 1,
 646         type => 'string',
 647         enum => [ qw(seabios ovmf) ],
 648         description => "Select BIOS implementation.",
 649         default => 'seabios',
 650     },
 651     vmgenid => {
 652         type => 'string',
 653         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 654         format_description => 'UUID',
 655         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 656             ." to disable explicitly.",
 657         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 658             ." value identifier to the guest OS. This allows to notify the guest operating system"
 659             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 660             ." execution or creation from a template). The guest operating system notices the"
 661             ." change, and is then able to react as appropriate by marking its copies of"
 662             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 663             ."Note that auto-creation only works when done through API/CLI create or update methods"
 664             .", but not when manually editing the config file.",
 665         default => "1 (autogenerated)",
 666         optional => 1,
 667     },
 668     hookscript => {
 669         type => 'string',
 670         format => 'pve-volume-id',
 671         optional => 1,
 672         description => "Script that will be executed during various steps in the vms lifetime.",
 673     },
 674     ivshmem => {
 675         type => 'string',
 676         format => $ivshmem_fmt,
 677         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 678             ." the host.",
 679         optional => 1,
 680     },
 681     audio0 => {
 682         type => 'string',
 683         format => $audio_fmt,
 684         description => "Configure a audio device, useful in combination with QXL/Spice.",
 685         optional => 1
 686     },
 687     spice_enhancements => {
 688         type => 'string',
 689         format => $spice_enhancements_fmt,
 690         description => "Configure additional enhancements for SPICE.",
 691         optional => 1
 692     },
 693     tags => {
 694         type => 'string', format => 'pve-tag-list',
 695         description => 'Tags of the VM. This is only meta information.',
 696         optional => 1,
 697     },
 698     rng0 => {
 699         type => 'string',
 700         format => $rng_fmt,
 701         description => "Configure a VirtIO-based Random Number Generator.",
 702         optional => 1,
 703     },
 704     meta => {
 705         type => 'string',
 706         format => $meta_info_fmt,
 707         description => "Some (read-only) meta-information about this guest.",
 708         optional => 1,
 709     },
 710 };
 711
 712 my $cicustom_fmt = {
 713     meta => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all meta data passed to the VM via"
 717             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via'
 725             .' cloud-init.',
 726         format => 'pve-volume-id',
 727         format_description => 'volume',
 728     },
 729     user => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all user data passed to the VM via'
 733             .' cloud-init.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     vendor => {
 738     type => 'string',
 739     optional => 1,
 740     description => 'Specify a custom file containing all vendor data passed to the VM via'
 741      .' cloud-init.',
 742     format => 'pve-volume-id',
 743     format_description => 'volume',
 744     },
 745 };
 746 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 747
 748 my $confdesc_cloudinit = {
 749     citype => {
 750         optional => 1,
 751         type => 'string',
 752         description => 'Specifies the cloud-init configuration format. The default depends on the'
 753             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 754             .' and `configdrive2` for windows.',
 755         enum => ['configdrive2', 'nocloud', 'opennebula'],
 756     },
 757     ciuser => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: User name to change ssh keys and password for instead of the"
 761             ." image's configured default user.",
 762     },
 763     cipassword => {
 764         optional => 1,
 765         type => 'string',
 766         description => 'cloud-init: Password to assign the user. Using this is generally not'
 767             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 768             .' support hashed passwords.',
 769     },
 770     cicustom => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'cloud-init: Specify custom files to replace the automatically generated'
 774             .' ones at start.',
 775         format => 'pve-qm-cicustom',
 776     },
 777     searchdomain => {
 778         optional => 1,
 779         type => 'string',
 780         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 781             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 782             .' are set.',
 783     },
 784     nameserver => {
 785         optional => 1,
 786         type => 'string', format => 'address-list',
 787         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 788             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 789             .' are set.',
 790     },
 791     sshkeys => {
 792         optional => 1,
 793         type => 'string',
 794         format => 'urlencoded',
 795         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 796     },
 797 };
 798
 799 # what about other qemu settings ?
 800 #cpu => 'string',
 801 #machine => 'string',
 802 #fda => 'file',
 803 #fdb => 'file',
 804 #mtdblock => 'file',
 805 #sd => 'file',
 806 #pflash => 'file',
 807 #snapshot => 'bool',
 808 #bootp => 'file',
 809 ##tftp => 'dir',
 810 ##smb => 'dir',
 811 #kernel => 'file',
 812 #append => 'string',
 813 #initrd => 'file',
 814 ##soundhw => 'string',
 815
 816 while (my ($k, $v) = each %$confdesc) {
 817     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 818 }
 819
 820 my $MAX_USB_DEVICES = 5;
 821 my $MAX_NETS = 32;
 822 my $MAX_SERIAL_PORTS = 4;
 823 my $MAX_PARALLEL_PORTS = 3;
 824 my $MAX_NUMA = 8;
 825
 826 my $numa_fmt = {
 827     cpus => {
 828         type => "string",
 829         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 830         description => "CPUs accessing this NUMA node.",
 831         format_description => "id[-id];...",
 832     },
 833     memory => {
 834         type => "number",
 835         description => "Amount of memory this NUMA node provides.",
 836         optional => 1,
 837     },
 838     hostnodes => {
 839         type => "string",
 840         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 841         description => "Host NUMA nodes to use.",
 842         format_description => "id[-id];...",
 843         optional => 1,
 844     },
 845     policy => {
 846         type => 'string',
 847         enum => [qw(preferred bind interleave)],
 848         description => "NUMA allocation policy.",
 849         optional => 1,
 850     },
 851 };
 852 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 853 my $numadesc = {
 854     optional => 1,
 855     type => 'string', format => $numa_fmt,
 856     description => "NUMA topology.",
 857 };
 858 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 859
 860 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 861     $confdesc->{"numa$i"} = $numadesc;
 862 }
 863
 864 my $nic_model_list = [
 865     'e1000',
 866     'e1000-82540em',
 867     'e1000-82544gc',
 868     'e1000-82545em',
 869     'e1000e',
 870     'i82551',
 871     'i82557b',
 872     'i82559er',
 873     'ne2k_isa',
 874     'ne2k_pci',
 875     'pcnet',
 876     'rtl8139',
 877     'virtio',
 878     'vmxnet3',
 879 ];
 880 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 881
 882 my $net_fmt_bridge_descr = <<__EOD__;
 883 Bridge to attach the network device to. The Proxmox VE standard bridge
 884 is called 'vmbr0'.
 885
 886 If you do not specify a bridge, we create a kvm user (NATed) network
 887 device, which provides DHCP and DNS services. The following addresses
 888 are used:
 889
 890  10.0.2.2   Gateway
 891  10.0.2.3   DNS Server
 892  10.0.2.4   SMB Server
 893
 894 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 895 __EOD__
 896
 897 my $net_fmt = {
 898     macaddr  => get_standard_option('mac-addr', {
 899         description => "MAC address. That address must be unique withing your network. This is"
 900             ." automatically generated if not specified.",
 901     }),
 902     model => {
 903         type => 'string',
 904         description => "Network Card Model. The 'virtio' model provides the best performance with"
 905             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 906             ." best to use 'e1000'.",
 907         enum => $nic_model_list,
 908         default_key => 1,
 909     },
 910     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 911     bridge => get_standard_option('pve-bridge-id', {
 912         description => $net_fmt_bridge_descr,
 913         optional => 1,
 914     }),
 915     queues => {
 916         type => 'integer',
 917         minimum => 0, maximum => 16,
 918         description => 'Number of packet queues to be used on the device.',
 919         optional => 1,
 920     },
 921     rate => {
 922         type => 'number',
 923         minimum => 0,
 924         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 925         optional => 1,
 926     },
 927     tag => {
 928         type => 'integer',
 929         minimum => 1, maximum => 4094,
 930         description => 'VLAN tag to apply to packets on this interface.',
 931         optional => 1,
 932     },
 933     trunks => {
 934         type => 'string',
 935         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 936         description => 'VLAN trunks to pass through this interface.',
 937         format_description => 'vlanid[;vlanid...]',
 938         optional => 1,
 939     },
 940     firewall => {
 941         type => 'boolean',
 942         description => 'Whether this interface should be protected by the firewall.',
 943         optional => 1,
 944     },
 945     link_down => {
 946         type => 'boolean',
 947         description => 'Whether this interface should be disconnected (like pulling the plug).',
 948         optional => 1,
 949     },
 950     mtu => {
 951         type => 'integer',
 952         minimum => 1, maximum => 65520,
 953         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 954         optional => 1,
 955     },
 956 };
 957
 958 my $netdesc = {
 959     optional => 1,
 960     type => 'string', format => $net_fmt,
 961     description => "Specify network devices.",
 962 };
 963
 964 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 965
 966 my $ipconfig_fmt = {
 967     ip => {
 968         type => 'string',
 969         format => 'pve-ipv4-config',
 970         format_description => 'IPv4Format/CIDR',
 971         description => 'IPv4 address in CIDR format.',
 972         optional => 1,
 973         default => 'dhcp',
 974     },
 975     gw => {
 976         type => 'string',
 977         format => 'ipv4',
 978         format_description => 'GatewayIPv4',
 979         description => 'Default gateway for IPv4 traffic.',
 980         optional => 1,
 981         requires => 'ip',
 982     },
 983     ip6 => {
 984         type => 'string',
 985         format => 'pve-ipv6-config',
 986         format_description => 'IPv6Format/CIDR',
 987         description => 'IPv6 address in CIDR format.',
 988         optional => 1,
 989         default => 'dhcp',
 990     },
 991     gw6 => {
 992         type => 'string',
 993         format => 'ipv6',
 994         format_description => 'GatewayIPv6',
 995         description => 'Default gateway for IPv6 traffic.',
 996         optional => 1,
 997         requires => 'ip6',
 998     },
 999 };
1000 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1001 my $ipconfigdesc = {
1002     optional => 1,
1003     type => 'string', format => 'pve-qm-ipconfig',
1004     description => <<'EODESCR',
1005 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1006
1007 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1008
1009 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1010 gateway should be provided.
1011 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1012 cloud-init 19.4 or newer.
1013
1014 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1015 dhcp on IPv4.
1016 EODESCR
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1019
1020 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1021     $confdesc->{"net$i"} = $netdesc;
1022     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1023 }
1024
1025 foreach my $key (keys %$confdesc_cloudinit) {
1026     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1027 }
1028
1029 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1030 sub verify_volume_id_or_qm_path {
1031     my ($volid, $noerr) = @_;
1032
1033     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1034
1035     return verify_volume_id_or_absolute_path($volid, $noerr);
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1039 sub verify_volume_id_or_absolute_path {
1040     my ($volid, $noerr) = @_;
1041
1042     return $volid if $volid =~ m|^/|;
1043
1044     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1045     if ($@) {
1046         return if $noerr;
1047         die $@;
1048     }
1049     return $volid;
1050 }
1051
1052 my $usb_fmt = {
1053     host => {
1054         default_key => 1,
1055         type => 'string', format => 'pve-qm-usb-device',
1056         format_description => 'HOSTUSBDEVICE|spice',
1057         description => <<EODESCR,
1058 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1059
1060  'bus-port(.port)*' (decimal numbers) or
1061  'vendor_id:product_id' (hexadeciaml numbers) or
1062  'spice'
1063
1064 You can use the 'lsusb -t' command to list existing usb devices.
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 The value 'spice' can be used to add a usb redirection devices for spice.
1070 EODESCR
1071     },
1072     usb3 => {
1073         optional => 1,
1074         type => 'boolean',
1075         description => "Specifies whether if given host option is a USB3 device or port.",
1076         default => 0,
1077     },
1078 };
1079
1080 my $usbdesc = {
1081     optional => 1,
1082     type => 'string', format => $usb_fmt,
1083     description => "Configure an USB device (n is 0 to 4).",
1084 };
1085 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return  $cdrom_path if $cdrom_path;
1265
1266     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1267     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1268     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1269 }
1270
1271 sub get_iso_path {
1272     my ($storecfg, $vmid, $cdrom) = @_;
1273
1274     if ($cdrom eq 'cdrom') {
1275         return get_cdrom_path();
1276     } elsif ($cdrom eq 'none') {
1277         return '';
1278     } elsif ($cdrom =~ m|^/|) {
1279         return $cdrom;
1280     } else {
1281         return PVE::Storage::path($storecfg, $cdrom);
1282     }
1283 }
1284
1285 # try to convert old style file names to volume IDs
1286 sub filename_to_volume_id {
1287     my ($vmid, $file, $media) = @_;
1288
1289      if (!($file eq 'none' || $file eq 'cdrom' ||
1290           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1291
1292         return if $file =~ m|/|;
1293
1294         if ($media && $media eq 'cdrom') {
1295             $file = "local:iso/$file";
1296         } else {
1297             $file = "local:$vmid/$file";
1298         }
1299     }
1300
1301     return $file;
1302 }
1303
1304 sub verify_media_type {
1305     my ($opt, $vtype, $media) = @_;
1306
1307     return if !$media;
1308
1309     my $etype;
1310     if ($media eq 'disk') {
1311         $etype = 'images';
1312     } elsif ($media eq 'cdrom') {
1313         $etype = 'iso';
1314     } else {
1315         die "internal error";
1316     }
1317
1318     return if ($vtype eq $etype);
1319
1320     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1321 }
1322
1323 sub cleanup_drive_path {
1324     my ($opt, $storecfg, $drive) = @_;
1325
1326     # try to convert filesystem paths to volume IDs
1327
1328     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1329         ($drive->{file} !~ m|^/dev/.+|) &&
1330         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1331         ($drive->{file} !~ m/^\d+$/)) {
1332         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1333         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1334             if !$vtype;
1335         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1336         verify_media_type($opt, $vtype, $drive->{media});
1337         $drive->{file} = $volid;
1338     }
1339
1340     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1341 }
1342
1343 sub parse_hotplug_features {
1344     my ($data) = @_;
1345
1346     my $res = {};
1347
1348     return $res if $data eq '0';
1349
1350     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1351
1352     foreach my $feature (PVE::Tools::split_list($data)) {
1353         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1354             $res->{$1} = 1;
1355         } else {
1356             die "invalid hotplug feature '$feature'\n";
1357         }
1358     }
1359     return $res;
1360 }
1361
1362 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1363 sub pve_verify_hotplug_features {
1364     my ($value, $noerr) = @_;
1365
1366     return $value if parse_hotplug_features($value);
1367
1368     return if $noerr;
1369
1370     die "unable to parse hotplug option\n";
1371 }
1372
1373 sub scsi_inquiry {
1374     my($fh, $noerr) = @_;
1375
1376     my $SG_IO = 0x2285;
1377     my $SG_GET_VERSION_NUM = 0x2282;
1378
1379     my $versionbuf = "\x00" x 8;
1380     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1381     if (!$ret) {
1382         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1383         return;
1384     }
1385     my $version = unpack("I", $versionbuf);
1386     if ($version < 30000) {
1387         die "scsi generic interface too old\n"  if !$noerr;
1388         return;
1389     }
1390
1391     my $buf = "\x00" x 36;
1392     my $sensebuf = "\x00" x 8;
1393     my $cmd = pack("C x3 C x1", 0x12, 36);
1394
1395     # see /usr/include/scsi/sg.h
1396     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1397
1398     my $packet = pack(
1399         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1400     );
1401
1402     $ret = ioctl($fh, $SG_IO, $packet);
1403     if (!$ret) {
1404         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1405         return;
1406     }
1407
1408     my @res = unpack($sg_io_hdr_t, $packet);
1409     if ($res[17] || $res[18]) {
1410         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my $res = {};
1415     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1416
1417     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1418     $res->{type} &= 0x1F;
1419
1420     return $res;
1421 }
1422
1423 sub path_is_scsi {
1424     my ($path) = @_;
1425
1426     my $fh = IO::File->new("+<$path") || return;
1427     my $res = scsi_inquiry($fh, 1);
1428     close($fh);
1429
1430     return $res;
1431 }
1432
1433 sub print_tabletdevice_full {
1434     my ($conf, $arch) = @_;
1435
1436     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1437
1438     # we use uhci for old VMs because tablet driver was buggy in older qemu
1439     my $usbbus;
1440     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1441         $usbbus = 'ehci';
1442     } else {
1443         $usbbus = 'uhci';
1444     }
1445
1446     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1447 }
1448
1449 sub print_keyboarddevice_full {
1450     my ($conf, $arch) = @_;
1451
1452     return if $arch ne 'aarch64';
1453
1454     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1455 }
1456
1457 my sub get_drive_id {
1458     my ($drive) = @_;
1459     return "$drive->{interface}$drive->{index}";
1460 }
1461
1462 sub print_drivedevice_full {
1463     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1464
1465     my $device = '';
1466     my $maxdev = 0;
1467
1468     my $drive_id = get_drive_id($drive);
1469     if ($drive->{interface} eq 'virtio') {
1470         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1471         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1472         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1473     } elsif ($drive->{interface} eq 'scsi') {
1474
1475         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1476         my $unit = $drive->{index} % $maxdev;
1477         my $devicetype = 'hd';
1478         my $path = '';
1479         if (drive_is_cdrom($drive)) {
1480             $devicetype = 'cd';
1481         } else {
1482             if ($drive->{file} =~ m|^/|) {
1483                 $path = $drive->{file};
1484                 if (my $info = path_is_scsi($path)) {
1485                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1486                         $devicetype = 'block';
1487                     } elsif ($info->{type} == 1) { # tape
1488                         $devicetype = 'generic';
1489                     }
1490                 }
1491             } else {
1492                  $path = PVE::Storage::path($storecfg, $drive->{file});
1493             }
1494
1495             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1496             my $version = extract_version($machine_type, kvm_user_version());
1497             if ($path =~ m/^iscsi\:\/\// &&
1498                !min_version($version, 4, 1)) {
1499                 $devicetype = 'generic';
1500             }
1501         }
1502
1503         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1504             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1505         } else {
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1507                 .",lun=$drive->{index}";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1512             $device .= ",rotation_rate=1";
1513         }
1514         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1515
1516     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1517         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype";
1523         if ($drive->{interface} eq 'ide') {
1524             $device .= ",bus=ide.$controller,unit=$unit";
1525         } else {
1526             $device .= ",bus=ahci$controller.$unit";
1527         }
1528         $device .= ",drive=drive-$drive_id,id=$drive_id";
1529
1530         if ($devicetype eq 'hd') {
1531             if (my $model = $drive->{model}) {
1532                 $model = URI::Escape::uri_unescape($model);
1533                 $device .= ",model=$model";
1534             }
1535             if ($drive->{ssd}) {
1536                 $device .= ",rotation_rate=1";
1537             }
1538         }
1539         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1540     } elsif ($drive->{interface} eq 'usb') {
1541         die "implement me";
1542         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1543     } else {
1544         die "unsupported interface type";
1545     }
1546
1547     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1548
1549     if (my $serial = $drive->{serial}) {
1550         $serial = URI::Escape::uri_unescape($serial);
1551         $device .= ",serial=$serial";
1552     }
1553
1554
1555     return $device;
1556 }
1557
1558 sub get_initiator_name {
1559     my $initiator;
1560
1561     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1562     while (defined(my $line = <$fh>)) {
1563         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1564         $initiator = $1;
1565         last;
1566     }
1567     $fh->close();
1568
1569     return $initiator;
1570 }
1571
1572 sub print_drive_commandline_full {
1573     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1574
1575     my $path;
1576     my $volid = $drive->{file};
1577     my $format = $drive->{format};
1578     my $drive_id = get_drive_id($drive);
1579
1580     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1581     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1582
1583     if (drive_is_cdrom($drive)) {
1584         $path = get_iso_path($storecfg, $vmid, $volid);
1585         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1586     } else {
1587         if ($storeid) {
1588             $path = PVE::Storage::path($storecfg, $volid);
1589             $format //= qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format //= "raw";
1593         }
1594    }
1595
1596    my $is_rbd = $path =~ m/^rbd:/;
1597
1598     my $opts = '';
1599     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1600     foreach my $o (@qemu_drive_options) {
1601         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1602     }
1603
1604     # snapshot only accepts on|off
1605     if (defined($drive->{snapshot})) {
1606         my $v = $drive->{snapshot} ? 'on' : 'off';
1607         $opts .= ",snapshot=$v";
1608     }
1609
1610     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1611         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1612     }
1613
1614     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1615         my ($dir, $qmpname) = @$type;
1616         if (my $v = $drive->{"mbps$dir"}) {
1617             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1618         }
1619         if (my $v = $drive->{"mbps${dir}_max"}) {
1620             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1621         }
1622         if (my $v = $drive->{"bps${dir}_max_length"}) {
1623             $opts .= ",throttling.bps$qmpname-max-length=$v";
1624         }
1625         if (my $v = $drive->{"iops${dir}"}) {
1626             $opts .= ",throttling.iops$qmpname=$v";
1627         }
1628         if (my $v = $drive->{"iops${dir}_max"}) {
1629             $opts .= ",throttling.iops$qmpname-max=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}_max_length"}) {
1632             $opts .= ",throttling.iops$qmpname-max-length=$v";
1633         }
1634     }
1635
1636     if ($pbs_name) {
1637         $format = "rbd" if $is_rbd;
1638         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1639             if !$format;
1640         $opts .= ",format=alloc-track,file.driver=$format";
1641     } elsif ($format) {
1642         $opts .= ",format=$format";
1643     }
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # io_uring with cache mode writeback or writethrough on krbd will hang...
1655     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1656
1657     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1658     # sometimes, just plain disable...
1659     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1660
1661     # io_uring causes problems when used with CIFS since kernel 5.15
1662     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1663     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1664
1665     if (!$drive->{aio}) {
1666         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1667             # io_uring supports all cache modes
1668             $opts .= ",aio=io_uring";
1669         } else {
1670             # aio native works only with O_DIRECT
1671             if($cache_direct) {
1672                 $opts .= ",aio=native";
1673             } else {
1674                 $opts .= ",aio=threads";
1675             }
1676         }
1677     }
1678
1679     if (!drive_is_cdrom($drive)) {
1680         my $detectzeroes;
1681         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1682             $detectzeroes = 'off';
1683         } elsif ($drive->{discard}) {
1684             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1685         } else {
1686             # This used to be our default with discard not being specified:
1687             $detectzeroes = 'on';
1688         }
1689
1690         # note: 'detect-zeroes' works per blockdev and we want it to persist
1691         # after the alloc-track is removed, so put it on 'file' directly
1692         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1693         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1694     }
1695
1696     if ($pbs_name) {
1697         $opts .= ",backing=$pbs_name";
1698         $opts .= ",auto-remove=on";
1699     }
1700
1701     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1702     my $file_param = "file";
1703     if ($pbs_name) {
1704         # non-rbd drivers require the underlying file to be a seperate block
1705         # node, so add a second .file indirection
1706         $file_param .= ".file" if !$is_rbd;
1707         $file_param .= ".filename";
1708     }
1709     my $pathinfo = $path ? "$file_param=$path," : '';
1710
1711     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1712 }
1713
1714 sub print_pbs_blockdev {
1715     my ($pbs_conf, $pbs_name) = @_;
1716     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1717     $blockdev .= ",repository=$pbs_conf->{repository}";
1718     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1719     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1720     $blockdev .= ",archive=$pbs_conf->{archive}";
1721     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1722     return $blockdev;
1723 }
1724
1725 sub print_netdevice_full {
1726     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1727
1728     my $device = $net->{model};
1729     if ($net->{model} eq 'virtio') {
1730          $device = 'virtio-net-pci';
1731      };
1732
1733     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1734     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1735     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1736         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1737         # and out of each queue plus one config interrupt and control vector queue
1738         my $vectors = $net->{queues} * 2 + 2;
1739         $tmpstr .= ",vectors=$vectors,mq=on";
1740     }
1741     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1742
1743     if (my $mtu = $net->{mtu}) {
1744         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1745             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1746             if ($mtu == 1) {
1747                  $mtu = $bridge_mtu;
1748             } elsif ($mtu < 576) {
1749                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1750             } elsif ($mtu > $bridge_mtu) {
1751                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1752             }
1753             $tmpstr .= ",host_mtu=$mtu";
1754         } else {
1755             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1756         }
1757     }
1758
1759     if ($use_old_bios_files) {
1760         my $romfile;
1761         if ($device eq 'virtio-net-pci') {
1762             $romfile = 'pxe-virtio.rom';
1763         } elsif ($device eq 'e1000') {
1764             $romfile = 'pxe-e1000.rom';
1765         } elsif ($device eq 'e1000e') {
1766             $romfile = 'pxe-e1000e.rom';
1767         } elsif ($device eq 'ne2k') {
1768             $romfile = 'pxe-ne2k_pci.rom';
1769         } elsif ($device eq 'pcnet') {
1770             $romfile = 'pxe-pcnet.rom';
1771         } elsif ($device eq 'rtl8139') {
1772             $romfile = 'pxe-rtl8139.rom';
1773         }
1774         $tmpstr .= ",romfile=$romfile" if $romfile;
1775     }
1776
1777     return $tmpstr;
1778 }
1779
1780 sub print_netdev_full {
1781     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1782
1783     my $i = '';
1784     if ($netid =~ m/^net(\d+)$/) {
1785         $i = int($1);
1786     }
1787
1788     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1789
1790     my $ifname = "tap${vmid}i$i";
1791
1792     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1793     die "interface name '$ifname' is too long (max 15 character)\n"
1794         if length($ifname) >= 16;
1795
1796     my $vhostparam = '';
1797     if (is_native($arch)) {
1798         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1799     }
1800
1801     my $vmname = $conf->{name} || "vm$vmid";
1802
1803     my $netdev = "";
1804     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1805
1806     if ($net->{bridge}) {
1807         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1808             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1809     } else {
1810         $netdev = "type=user,id=$netid,hostname=$vmname";
1811     }
1812
1813     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1814
1815     return $netdev;
1816 }
1817
1818 my $vga_map = {
1819     'cirrus' => 'cirrus-vga',
1820     'std' => 'VGA',
1821     'vmware' => 'vmware-svga',
1822     'virtio' => 'virtio-vga',
1823     'virtio-gl' => 'virtio-vga-gl',
1824 };
1825
1826 sub print_vga_device {
1827     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1828
1829     my $type = $vga_map->{$vga->{type}};
1830     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1831         $type = 'virtio-gpu';
1832     }
1833     my $vgamem_mb = $vga->{memory};
1834
1835     my $max_outputs = '';
1836     if ($qxlnum) {
1837         $type = $id ? 'qxl' : 'qxl-vga';
1838
1839         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1840             # set max outputs so linux can have up to 4 qxl displays with one device
1841             if (min_version($machine_version, 4, 1)) {
1842                 $max_outputs = ",max_outputs=4";
1843             }
1844         }
1845     }
1846
1847     die "no devicetype for $vga->{type}\n" if !$type;
1848
1849     my $memory = "";
1850     if ($vgamem_mb) {
1851         if ($vga->{type} =~ /^virtio/) {
1852             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1853             $memory = ",max_hostmem=$bytes";
1854         } elsif ($qxlnum) {
1855             # from https://www.spice-space.org/multiple-monitors.html
1856             $memory = ",vgamem_mb=$vga->{memory}";
1857             my $ram = $vgamem_mb * 4;
1858             my $vram = $vgamem_mb * 2;
1859             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1860         } else {
1861             $memory = ",vgamem_mb=$vga->{memory}";
1862         }
1863     } elsif ($qxlnum && $id) {
1864         $memory = ",ram_size=67108864,vram_size=33554432";
1865     }
1866
1867     my $edidoff = "";
1868     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1869         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1870     }
1871
1872     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1873     my $vgaid = "vga" . ($id // '');
1874     my $pciaddr;
1875     if ($q35 && $vgaid eq 'vga') {
1876         # the first display uses pcie.0 bus on q35 machines
1877         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1878     } else {
1879         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1880     }
1881
1882     if ($vga->{type} eq 'virtio-gl') {
1883         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1884         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1885             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1886
1887         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1888             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1889     }
1890
1891     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1892 }
1893
1894 sub parse_number_sets {
1895     my ($set) = @_;
1896     my $res = [];
1897     foreach my $part (split(/;/, $set)) {
1898         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1899             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1900             push @$res, [ $1, $2 ];
1901         } else {
1902             die "invalid range: $part\n";
1903         }
1904     }
1905     return $res;
1906 }
1907
1908 sub parse_numa {
1909     my ($data) = @_;
1910
1911     my $res = parse_property_string($numa_fmt, $data);
1912     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1913     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1914     return $res;
1915 }
1916
1917 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1918 sub parse_net {
1919     my ($data) = @_;
1920
1921     my $res = eval { parse_property_string($net_fmt, $data) };
1922     if ($@) {
1923         warn $@;
1924         return;
1925     }
1926     if (!defined($res->{macaddr})) {
1927         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1928         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1929     }
1930     return $res;
1931 }
1932
1933 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1934 sub parse_ipconfig {
1935     my ($data) = @_;
1936
1937     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1938     if ($@) {
1939         warn $@;
1940         return;
1941     }
1942
1943     if ($res->{gw} && !$res->{ip}) {
1944         warn 'gateway specified without specifying an IP address';
1945         return;
1946     }
1947     if ($res->{gw6} && !$res->{ip6}) {
1948         warn 'IPv6 gateway specified without specifying an IPv6 address';
1949         return;
1950     }
1951     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1952         warn 'gateway specified together with DHCP';
1953         return;
1954     }
1955     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1956         # gw6 + auto/dhcp
1957         warn "IPv6 gateway specified together with $res->{ip6} address";
1958         return;
1959     }
1960
1961     if (!$res->{ip} && !$res->{ip6}) {
1962         return { ip => 'dhcp', ip6 => 'dhcp' };
1963     }
1964
1965     return $res;
1966 }
1967
1968 sub print_net {
1969     my $net = shift;
1970
1971     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1972 }
1973
1974 sub add_random_macs {
1975     my ($settings) = @_;
1976
1977     foreach my $opt (keys %$settings) {
1978         next if $opt !~ m/^net(\d+)$/;
1979         my $net = parse_net($settings->{$opt});
1980         next if !$net;
1981         $settings->{$opt} = print_net($net);
1982     }
1983 }
1984
1985 sub vm_is_volid_owner {
1986     my ($storecfg, $vmid, $volid) = @_;
1987
1988     if ($volid !~  m|^/|) {
1989         my ($path, $owner);
1990         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1991         if ($owner && ($owner == $vmid)) {
1992             return 1;
1993         }
1994     }
1995
1996     return;
1997 }
1998
1999 sub vmconfig_register_unused_drive {
2000     my ($storecfg, $vmid, $conf, $drive) = @_;
2001
2002     if (drive_is_cloudinit($drive)) {
2003         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2004         warn $@ if $@;
2005         delete $conf->{cloudinit};
2006     } elsif (!drive_is_cdrom($drive)) {
2007         my $volid = $drive->{file};
2008         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2009             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2010         }
2011     }
2012 }
2013
2014 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2015 my $smbios1_fmt = {
2016     uuid => {
2017         type => 'string',
2018         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2019         format_description => 'UUID',
2020         description => "Set SMBIOS1 UUID.",
2021         optional => 1,
2022     },
2023     version => {
2024         type => 'string',
2025         pattern => '[A-Za-z0-9+\/]+={0,2}',
2026         format_description => 'Base64 encoded string',
2027         description => "Set SMBIOS1 version.",
2028         optional => 1,
2029     },
2030     serial => {
2031         type => 'string',
2032         pattern => '[A-Za-z0-9+\/]+={0,2}',
2033         format_description => 'Base64 encoded string',
2034         description => "Set SMBIOS1 serial number.",
2035         optional => 1,
2036     },
2037     manufacturer => {
2038         type => 'string',
2039         pattern => '[A-Za-z0-9+\/]+={0,2}',
2040         format_description => 'Base64 encoded string',
2041         description => "Set SMBIOS1 manufacturer.",
2042         optional => 1,
2043     },
2044     product => {
2045         type => 'string',
2046         pattern => '[A-Za-z0-9+\/]+={0,2}',
2047         format_description => 'Base64 encoded string',
2048         description => "Set SMBIOS1 product ID.",
2049         optional => 1,
2050     },
2051     sku => {
2052         type => 'string',
2053         pattern => '[A-Za-z0-9+\/]+={0,2}',
2054         format_description => 'Base64 encoded string',
2055         description => "Set SMBIOS1 SKU string.",
2056         optional => 1,
2057     },
2058     family => {
2059         type => 'string',
2060         pattern => '[A-Za-z0-9+\/]+={0,2}',
2061         format_description => 'Base64 encoded string',
2062         description => "Set SMBIOS1 family string.",
2063         optional => 1,
2064     },
2065     base64 => {
2066         type => 'boolean',
2067         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2068         optional => 1,
2069     },
2070 };
2071
2072 sub parse_smbios1 {
2073     my ($data) = @_;
2074
2075     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2076     warn $@ if $@;
2077     return $res;
2078 }
2079
2080 sub print_smbios1 {
2081     my ($smbios1) = @_;
2082     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2083 }
2084
2085 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2086
2087 sub parse_watchdog {
2088     my ($value) = @_;
2089
2090     return if !$value;
2091
2092     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2093     warn $@ if $@;
2094     return $res;
2095 }
2096
2097 sub parse_guest_agent {
2098     my ($conf) = @_;
2099
2100     return {} if !defined($conf->{agent});
2101
2102     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2103     warn $@ if $@;
2104
2105     # if the agent is disabled ignore the other potentially set properties
2106     return {} if !$res->{enabled};
2107     return $res;
2108 }
2109
2110 sub get_qga_key {
2111     my ($conf, $key) = @_;
2112     return undef if !defined($conf->{agent});
2113
2114     my $agent = parse_guest_agent($conf);
2115     return $agent->{$key};
2116 }
2117
2118 sub parse_vga {
2119     my ($value) = @_;
2120
2121     return {} if !$value;
2122     my $res = eval { parse_property_string($vga_fmt, $value) };
2123     warn $@ if $@;
2124     return $res;
2125 }
2126
2127 sub parse_rng {
2128     my ($value) = @_;
2129
2130     return if !$value;
2131
2132     my $res = eval { parse_property_string($rng_fmt, $value) };
2133     warn $@ if $@;
2134     return $res;
2135 }
2136
2137 sub parse_meta_info {
2138     my ($value) = @_;
2139
2140     return if !$value;
2141
2142     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2143     warn $@ if $@;
2144     return $res;
2145 }
2146
2147 sub new_meta_info_string {
2148     my () = @_; # for now do not allow to override any value
2149
2150     return PVE::JSONSchema::print_property_string(
2151         {
2152             'creation-qemu' => kvm_user_version(),
2153             ctime => "". int(time()),
2154         },
2155         $meta_info_fmt
2156     );
2157 }
2158
2159 sub qemu_created_version_fixups {
2160     my ($conf, $forcemachine, $kvmver) = @_;
2161
2162     my $meta = parse_meta_info($conf->{meta}) // {};
2163     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2164
2165     # check if we need to apply some handling for VMs that always use the latest machine version but
2166     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2167     # would be required (we do not want to pin machine version for non-windows OS type)
2168     if (
2169         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2170         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2171         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2172         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2173     ) {
2174         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2175         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2176             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2177             # and thus with the predictable interface naming of systemd
2178             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2179         }
2180     }
2181     return;
2182 }
2183
2184 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2185 sub verify_usb_device {
2186     my ($value, $noerr) = @_;
2187
2188     return $value if parse_usb_device($value);
2189
2190     return if $noerr;
2191
2192     die "unable to parse usb device\n";
2193 }
2194
2195 # add JSON properties for create and set function
2196 sub json_config_properties {
2197     my ($prop, $with_disk_alloc) = @_;
2198
2199     my $skip_json_config_opts = {
2200         parent => 1,
2201         snaptime => 1,
2202         vmstate => 1,
2203         runningmachine => 1,
2204         runningcpu => 1,
2205         meta => 1,
2206     };
2207
2208     foreach my $opt (keys %$confdesc) {
2209         next if $skip_json_config_opts->{$opt};
2210
2211         if ($with_disk_alloc && is_valid_drivename($opt)) {
2212             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2213         } else {
2214             $prop->{$opt} = $confdesc->{$opt};
2215         }
2216     }
2217
2218     return $prop;
2219 }
2220
2221 # Properties that we can read from an OVF file
2222 sub json_ovf_properties {
2223     my $prop = {};
2224
2225     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2226         $prop->{$device} = {
2227             type => 'string',
2228             format => 'pve-volume-id-or-absolute-path',
2229             description => "Disk image that gets imported to $device",
2230             optional => 1,
2231         };
2232     }
2233
2234     $prop->{cores} = {
2235         type => 'integer',
2236         description => "The number of CPU cores.",
2237         optional => 1,
2238     };
2239     $prop->{memory} = {
2240         type => 'integer',
2241         description => "Amount of RAM for the VM in MB.",
2242         optional => 1,
2243     };
2244     $prop->{name} = {
2245         type => 'string',
2246         description => "Name of the VM.",
2247         optional => 1,
2248     };
2249
2250     return $prop;
2251 }
2252
2253 # return copy of $confdesc_cloudinit to generate documentation
2254 sub cloudinit_config_properties {
2255
2256     return dclone($confdesc_cloudinit);
2257 }
2258
2259 sub check_type {
2260     my ($key, $value) = @_;
2261
2262     die "unknown setting '$key'\n" if !$confdesc->{$key};
2263
2264     my $type = $confdesc->{$key}->{type};
2265
2266     if (!defined($value)) {
2267         die "got undefined value\n";
2268     }
2269
2270     if ($value =~ m/[\n\r]/) {
2271         die "property contains a line feed\n";
2272     }
2273
2274     if ($type eq 'boolean') {
2275         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2276         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2277         die "type check ('boolean') failed - got '$value'\n";
2278     } elsif ($type eq 'integer') {
2279         return int($1) if $value =~ m/^(\d+)$/;
2280         die "type check ('integer') failed - got '$value'\n";
2281     } elsif ($type eq 'number') {
2282         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2283         die "type check ('number') failed - got '$value'\n";
2284     } elsif ($type eq 'string') {
2285         if (my $fmt = $confdesc->{$key}->{format}) {
2286             PVE::JSONSchema::check_format($fmt, $value);
2287             return $value;
2288         }
2289         $value =~ s/^\"(.*)\"$/$1/;
2290         return $value;
2291     } else {
2292         die "internal error"
2293     }
2294 }
2295
2296 sub destroy_vm {
2297     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2298
2299     my $conf = PVE::QemuConfig->load_config($vmid);
2300
2301     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2302
2303     if ($conf->{template}) {
2304         # check if any base image is still used by a linked clone
2305         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2306                 my ($ds, $drive) = @_;
2307                 return if drive_is_cdrom($drive);
2308
2309                 my $volid = $drive->{file};
2310                 return if !$volid || $volid =~ m|^/|;
2311
2312                 die "base volume '$volid' is still in use by linked cloned\n"
2313                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2314
2315         });
2316     }
2317
2318     my $volids = {};
2319     my $remove_owned_drive = sub {
2320         my ($ds, $drive) = @_;
2321         return if drive_is_cdrom($drive, 1);
2322
2323         my $volid = $drive->{file};
2324         return if !$volid || $volid =~ m|^/|;
2325         return if $volids->{$volid};
2326
2327         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2328         return if !$path || !$owner || ($owner != $vmid);
2329
2330         $volids->{$volid} = 1;
2331         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2332         warn "Could not remove disk '$volid', check manually: $@" if $@;
2333     };
2334
2335     # only remove disks owned by this VM (referenced in the config)
2336     my $include_opts = {
2337         include_unused => 1,
2338         extra_keys => ['vmstate'],
2339     };
2340     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2341
2342     for my $snap (values %{$conf->{snapshots}}) {
2343         next if !defined($snap->{vmstate});
2344         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2345         next if !defined($drive);
2346         $remove_owned_drive->('vmstate', $drive);
2347     }
2348
2349     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2350
2351     if ($purge_unreferenced) { # also remove unreferenced disk
2352         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2353         PVE::Storage::foreach_volid($vmdisks, sub {
2354             my ($volid, $sid, $volname, $d) = @_;
2355             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2356             warn $@ if $@;
2357         });
2358     }
2359
2360     if (defined $replacement_conf) {
2361         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2362     } else {
2363         PVE::QemuConfig->destroy_config($vmid);
2364     }
2365 }
2366
2367 sub parse_vm_config {
2368     my ($filename, $raw, $strict) = @_;
2369
2370     return if !defined($raw);
2371
2372     my $res = {
2373         digest => Digest::SHA::sha1_hex($raw),
2374         snapshots => {},
2375         pending => {},
2376         cloudinit => {},
2377     };
2378
2379     my $handle_error = sub {
2380         my ($msg) = @_;
2381
2382         if ($strict) {
2383             die $msg;
2384         } else {
2385             warn $msg;
2386         }
2387     };
2388
2389     $filename =~ m|/qemu-server/(\d+)\.conf$|
2390         || die "got strange filename '$filename'";
2391
2392     my $vmid = $1;
2393
2394     my $conf = $res;
2395     my $descr;
2396     my $section = '';
2397
2398     my @lines = split(/\n/, $raw);
2399     foreach my $line (@lines) {
2400         next if $line =~ m/^\s*$/;
2401
2402         if ($line =~ m/^\[PENDING\]\s*$/i) {
2403             $section = 'pending';
2404             if (defined($descr)) {
2405                 $descr =~ s/\s+$//;
2406                 $conf->{description} = $descr;
2407             }
2408             $descr = undef;
2409             $conf = $res->{$section} = {};
2410             next;
2411         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2412             $section = 'cloudinit';
2413             $descr = undef;
2414             $conf = $res->{$section} = {};
2415             next;
2416
2417         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2418             $section = $1;
2419             if (defined($descr)) {
2420                 $descr =~ s/\s+$//;
2421                 $conf->{description} = $descr;
2422             }
2423             $descr = undef;
2424             $conf = $res->{snapshots}->{$section} = {};
2425             next;
2426         }
2427
2428         if ($line =~ m/^\#(.*)$/) {
2429             $descr = '' if !defined($descr);
2430             $descr .= PVE::Tools::decode_text($1) . "\n";
2431             next;
2432         }
2433
2434         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2435             $descr = '' if !defined($descr);
2436             $descr .= PVE::Tools::decode_text($2);
2437         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2438             $conf->{snapstate} = $1;
2439         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2440             my $key = $1;
2441             my $value = $2;
2442             $conf->{$key} = $value;
2443         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2444             my $value = $1;
2445             if ($section eq 'pending') {
2446                 $conf->{delete} = $value; # we parse this later
2447             } else {
2448                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2449             }
2450         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2451             my $key = $1;
2452             my $value = $2;
2453             eval { $value = check_type($key, $value); };
2454             if ($@) {
2455                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2456             } else {
2457                 $key = 'ide2' if $key eq 'cdrom';
2458                 my $fmt = $confdesc->{$key}->{format};
2459                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2460                     my $v = parse_drive($key, $value);
2461                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2462                         $v->{file} = $volid;
2463                         $value = print_drive($v);
2464                     } else {
2465                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2466                         next;
2467                     }
2468                 }
2469
2470                 $conf->{$key} = $value;
2471             }
2472         } else {
2473             $handle_error->("vm $vmid - unable to parse config: $line\n");
2474         }
2475     }
2476
2477     if (defined($descr)) {
2478         $descr =~ s/\s+$//;
2479         $conf->{description} = $descr;
2480     }
2481     delete $res->{snapstate}; # just to be sure
2482
2483     return $res;
2484 }
2485
2486 sub write_vm_config {
2487     my ($filename, $conf) = @_;
2488
2489     delete $conf->{snapstate}; # just to be sure
2490
2491     if ($conf->{cdrom}) {
2492         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2493         $conf->{ide2} = $conf->{cdrom};
2494         delete $conf->{cdrom};
2495     }
2496
2497     # we do not use 'smp' any longer
2498     if ($conf->{sockets}) {
2499         delete $conf->{smp};
2500     } elsif ($conf->{smp}) {
2501         $conf->{sockets} = $conf->{smp};
2502         delete $conf->{cores};
2503         delete $conf->{smp};
2504     }
2505
2506     my $used_volids = {};
2507
2508     my $cleanup_config = sub {
2509         my ($cref, $pending, $snapname) = @_;
2510
2511         foreach my $key (keys %$cref) {
2512             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2513                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2514             my $value = $cref->{$key};
2515             if ($key eq 'delete') {
2516                 die "propertry 'delete' is only allowed in [PENDING]\n"
2517                     if !$pending;
2518                 # fixme: check syntax?
2519                 next;
2520             }
2521             eval { $value = check_type($key, $value); };
2522             die "unable to parse value of '$key' - $@" if $@;
2523
2524             $cref->{$key} = $value;
2525
2526             if (!$snapname && is_valid_drivename($key)) {
2527                 my $drive = parse_drive($key, $value);
2528                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2529             }
2530         }
2531     };
2532
2533     &$cleanup_config($conf);
2534
2535     &$cleanup_config($conf->{pending}, 1);
2536
2537     &$cleanup_config($conf->{cloudinit});
2538
2539     foreach my $snapname (keys %{$conf->{snapshots}}) {
2540         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2541         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2542     }
2543
2544     # remove 'unusedX' settings if we re-add a volume
2545     foreach my $key (keys %$conf) {
2546         my $value = $conf->{$key};
2547         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2548             delete $conf->{$key};
2549         }
2550     }
2551
2552     my $generate_raw_config = sub {
2553         my ($conf, $pending) = @_;
2554
2555         my $raw = '';
2556
2557         # add description as comment to top of file
2558         if (defined(my $descr = $conf->{description})) {
2559             if ($descr) {
2560                 foreach my $cl (split(/\n/, $descr)) {
2561                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2562                 }
2563             } else {
2564                 $raw .= "#\n" if $pending;
2565             }
2566         }
2567
2568         foreach my $key (sort keys %$conf) {
2569             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2570             $raw .= "$key: $conf->{$key}\n";
2571         }
2572         return $raw;
2573     };
2574
2575     my $raw = &$generate_raw_config($conf);
2576
2577     if (scalar(keys %{$conf->{pending}})){
2578         $raw .= "\n[PENDING]\n";
2579         $raw .= &$generate_raw_config($conf->{pending}, 1);
2580     }
2581
2582     if (scalar(keys %{$conf->{cloudinit}})){
2583         $raw .= "\n[special:cloudinit]\n";
2584         $raw .= &$generate_raw_config($conf->{cloudinit});
2585     }
2586
2587     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2588         $raw .= "\n[$snapname]\n";
2589         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2590     }
2591
2592     return $raw;
2593 }
2594
2595 sub load_defaults {
2596
2597     my $res = {};
2598
2599     # we use static defaults from our JSON schema configuration
2600     foreach my $key (keys %$confdesc) {
2601         if (defined(my $default = $confdesc->{$key}->{default})) {
2602             $res->{$key} = $default;
2603         }
2604     }
2605
2606     return $res;
2607 }
2608
2609 sub config_list {
2610     my $vmlist = PVE::Cluster::get_vmlist();
2611     my $res = {};
2612     return $res if !$vmlist || !$vmlist->{ids};
2613     my $ids = $vmlist->{ids};
2614     my $nodename = nodename();
2615
2616     foreach my $vmid (keys %$ids) {
2617         my $d = $ids->{$vmid};
2618         next if !$d->{node} || $d->{node} ne $nodename;
2619         next if !$d->{type} || $d->{type} ne 'qemu';
2620         $res->{$vmid}->{exists} = 1;
2621     }
2622     return $res;
2623 }
2624
2625 # test if VM uses local resources (to prevent migration)
2626 sub check_local_resources {
2627     my ($conf, $noerr) = @_;
2628
2629     my @loc_res = ();
2630
2631     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2632     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2633
2634     push @loc_res, "ivshmem" if $conf->{ivshmem};
2635
2636     foreach my $k (keys %$conf) {
2637         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2638         # sockets are safe: they will recreated be on the target side post-migrate
2639         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2640         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2641     }
2642
2643     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2644
2645     return \@loc_res;
2646 }
2647
2648 # check if used storages are available on all nodes (use by migrate)
2649 sub check_storage_availability {
2650     my ($storecfg, $conf, $node) = @_;
2651
2652     PVE::QemuConfig->foreach_volume($conf, sub {
2653         my ($ds, $drive) = @_;
2654
2655         my $volid = $drive->{file};
2656         return if !$volid;
2657
2658         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2659         return if !$sid;
2660
2661         # check if storage is available on both nodes
2662         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2663         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2664
2665         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2666
2667         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2668             if !$scfg->{content}->{$vtype};
2669    });
2670 }
2671
2672 # list nodes where all VM images are available (used by has_feature API)
2673 sub shared_nodes {
2674     my ($conf, $storecfg) = @_;
2675
2676     my $nodelist = PVE::Cluster::get_nodelist();
2677     my $nodehash = { map { $_ => 1 } @$nodelist };
2678     my $nodename = nodename();
2679
2680     PVE::QemuConfig->foreach_volume($conf, sub {
2681         my ($ds, $drive) = @_;
2682
2683         my $volid = $drive->{file};
2684         return if !$volid;
2685
2686         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2687         if ($storeid) {
2688             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2689             if ($scfg->{disable}) {
2690                 $nodehash = {};
2691             } elsif (my $avail = $scfg->{nodes}) {
2692                 foreach my $node (keys %$nodehash) {
2693                     delete $nodehash->{$node} if !$avail->{$node};
2694                 }
2695             } elsif (!$scfg->{shared}) {
2696                 foreach my $node (keys %$nodehash) {
2697                     delete $nodehash->{$node} if $node ne $nodename
2698                 }
2699             }
2700         }
2701     });
2702
2703     return $nodehash
2704 }
2705
2706 sub check_local_storage_availability {
2707     my ($conf, $storecfg) = @_;
2708
2709     my $nodelist = PVE::Cluster::get_nodelist();
2710     my $nodehash = { map { $_ => {} } @$nodelist };
2711
2712     PVE::QemuConfig->foreach_volume($conf, sub {
2713         my ($ds, $drive) = @_;
2714
2715         my $volid = $drive->{file};
2716         return if !$volid;
2717
2718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2719         if ($storeid) {
2720             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2721
2722             if ($scfg->{disable}) {
2723                 foreach my $node (keys %$nodehash) {
2724                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2725                 }
2726             } elsif (my $avail = $scfg->{nodes}) {
2727                 foreach my $node (keys %$nodehash) {
2728                     if (!$avail->{$node}) {
2729                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2730                     }
2731                 }
2732             }
2733         }
2734     });
2735
2736     foreach my $node (values %$nodehash) {
2737         if (my $unavail = $node->{unavailable_storages}) {
2738             $node->{unavailable_storages} = [ sort keys %$unavail ];
2739         }
2740     }
2741
2742     return $nodehash
2743 }
2744
2745 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2746 sub check_running {
2747     my ($vmid, $nocheck, $node) = @_;
2748
2749     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2750     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2751 }
2752
2753 sub vzlist {
2754
2755     my $vzlist = config_list();
2756
2757     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2758
2759     while (defined(my $de = $fd->read)) {
2760         next if $de !~ m/^(\d+)\.pid$/;
2761         my $vmid = $1;
2762         next if !defined($vzlist->{$vmid});
2763         if (my $pid = check_running($vmid)) {
2764             $vzlist->{$vmid}->{pid} = $pid;
2765         }
2766     }
2767
2768     return $vzlist;
2769 }
2770
2771 our $vmstatus_return_properties = {
2772     vmid => get_standard_option('pve-vmid'),
2773     status => {
2774         description => "Qemu process status.",
2775         type => 'string',
2776         enum => ['stopped', 'running'],
2777     },
2778     maxmem => {
2779         description => "Maximum memory in bytes.",
2780         type => 'integer',
2781         optional => 1,
2782         renderer => 'bytes',
2783     },
2784     maxdisk => {
2785         description => "Root disk size in bytes.",
2786         type => 'integer',
2787         optional => 1,
2788         renderer => 'bytes',
2789     },
2790     name => {
2791         description => "VM name.",
2792         type => 'string',
2793         optional => 1,
2794     },
2795     qmpstatus => {
2796         description => "Qemu QMP agent status.",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     pid => {
2801         description => "PID of running qemu process.",
2802         type => 'integer',
2803         optional => 1,
2804     },
2805     uptime => {
2806         description => "Uptime.",
2807         type => 'integer',
2808         optional => 1,
2809         renderer => 'duration',
2810     },
2811     cpus => {
2812         description => "Maximum usable CPUs.",
2813         type => 'number',
2814         optional => 1,
2815     },
2816     lock => {
2817         description => "The current config lock, if any.",
2818         type => 'string',
2819         optional => 1,
2820     },
2821     tags => {
2822         description  => "The current configured tags, if any",
2823         type => 'string',
2824         optional => 1,
2825     },
2826     'running-machine' => {
2827         description => "The currently running machine type (if running).",
2828         type => 'string',
2829         optional => 1,
2830     },
2831     'running-qemu' => {
2832         description => "The currently running QEMU version (if running).",
2833         type => 'string',
2834         optional => 1,
2835     },
2836 };
2837
2838 my $last_proc_pid_stat;
2839
2840 # get VM status information
2841 # This must be fast and should not block ($full == false)
2842 # We only query KVM using QMP if $full == true (this can be slow)
2843 sub vmstatus {
2844     my ($opt_vmid, $full) = @_;
2845
2846     my $res = {};
2847
2848     my $storecfg = PVE::Storage::config();
2849
2850     my $list = vzlist();
2851     my $defaults = load_defaults();
2852
2853     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2854
2855     my $cpucount = $cpuinfo->{cpus} || 1;
2856
2857     foreach my $vmid (keys %$list) {
2858         next if $opt_vmid && ($vmid ne $opt_vmid);
2859
2860         my $conf = PVE::QemuConfig->load_config($vmid);
2861
2862         my $d = { vmid => int($vmid) };
2863         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2864
2865         # fixme: better status?
2866         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2867
2868         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2869         if (defined($size)) {
2870             $d->{disk} = 0; # no info available
2871             $d->{maxdisk} = $size;
2872         } else {
2873             $d->{disk} = 0;
2874             $d->{maxdisk} = 0;
2875         }
2876
2877         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2878             * ($conf->{cores} || $defaults->{cores});
2879         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2880         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2881
2882         $d->{name} = $conf->{name} || "VM $vmid";
2883         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2884             : $defaults->{memory}*(1024*1024);
2885
2886         if ($conf->{balloon}) {
2887             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2888             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2889                 : $defaults->{shares};
2890         }
2891
2892         $d->{uptime} = 0;
2893         $d->{cpu} = 0;
2894         $d->{mem} = 0;
2895
2896         $d->{netout} = 0;
2897         $d->{netin} = 0;
2898
2899         $d->{diskread} = 0;
2900         $d->{diskwrite} = 0;
2901
2902         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2903
2904         $d->{serial} = 1 if conf_has_serial($conf);
2905         $d->{lock} = $conf->{lock} if $conf->{lock};
2906         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2907
2908         $res->{$vmid} = $d;
2909     }
2910
2911     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2912     foreach my $dev (keys %$netdev) {
2913         next if $dev !~ m/^tap([1-9]\d*)i/;
2914         my $vmid = $1;
2915         my $d = $res->{$vmid};
2916         next if !$d;
2917
2918         $d->{netout} += $netdev->{$dev}->{receive};
2919         $d->{netin} += $netdev->{$dev}->{transmit};
2920
2921         if ($full) {
2922             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2923             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2924         }
2925
2926     }
2927
2928     my $ctime = gettimeofday;
2929
2930     foreach my $vmid (keys %$list) {
2931
2932         my $d = $res->{$vmid};
2933         my $pid = $d->{pid};
2934         next if !$pid;
2935
2936         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2937         next if !$pstat; # not running
2938
2939         my $used = $pstat->{utime} + $pstat->{stime};
2940
2941         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2942
2943         if ($pstat->{vsize}) {
2944             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2945         }
2946
2947         my $old = $last_proc_pid_stat->{$pid};
2948         if (!$old) {
2949             $last_proc_pid_stat->{$pid} = {
2950                 time => $ctime,
2951                 used => $used,
2952                 cpu => 0,
2953             };
2954             next;
2955         }
2956
2957         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2958
2959         if ($dtime > 1000) {
2960             my $dutime = $used -  $old->{used};
2961
2962             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2963             $last_proc_pid_stat->{$pid} = {
2964                 time => $ctime,
2965                 used => $used,
2966                 cpu => $d->{cpu},
2967             };
2968         } else {
2969             $d->{cpu} = $old->{cpu};
2970         }
2971     }
2972
2973     return $res if !$full;
2974
2975     my $qmpclient = PVE::QMPClient->new();
2976
2977     my $ballooncb = sub {
2978         my ($vmid, $resp) = @_;
2979
2980         my $info = $resp->{'return'};
2981         return if !$info->{max_mem};
2982
2983         my $d = $res->{$vmid};
2984
2985         # use memory assigned to VM
2986         $d->{maxmem} = $info->{max_mem};
2987         $d->{balloon} = $info->{actual};
2988
2989         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2990             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2991             $d->{freemem} = $info->{free_mem};
2992         }
2993
2994         $d->{ballooninfo} = $info;
2995     };
2996
2997     my $blockstatscb = sub {
2998         my ($vmid, $resp) = @_;
2999         my $data = $resp->{'return'} || [];
3000         my $totalrdbytes = 0;
3001         my $totalwrbytes = 0;
3002
3003         for my $blockstat (@$data) {
3004             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3005             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3006
3007             $blockstat->{device} =~ s/drive-//;
3008             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3009         }
3010         $res->{$vmid}->{diskread} = $totalrdbytes;
3011         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3012     };
3013
3014     my $machinecb = sub {
3015         my ($vmid, $resp) = @_;
3016         my $data = $resp->{'return'} || [];
3017
3018         $res->{$vmid}->{'running-machine'} =
3019             PVE::QemuServer::Machine::current_from_query_machines($data);
3020     };
3021
3022     my $versioncb = sub {
3023         my ($vmid, $resp) = @_;
3024         my $data = $resp->{'return'} // {};
3025         my $version = 'unknown';
3026
3027         if (my $v = $data->{qemu}) {
3028             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3029         }
3030
3031         $res->{$vmid}->{'running-qemu'} = $version;
3032     };
3033
3034     my $statuscb = sub {
3035         my ($vmid, $resp) = @_;
3036
3037         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3038         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3039         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3040         # this fails if ballon driver is not loaded, so this must be
3041         # the last commnand (following command are aborted if this fails).
3042         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3043
3044         my $status = 'unknown';
3045         if (!defined($status = $resp->{'return'}->{status})) {
3046             warn "unable to get VM status\n";
3047             return;
3048         }
3049
3050         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3051     };
3052
3053     foreach my $vmid (keys %$list) {
3054         next if $opt_vmid && ($vmid ne $opt_vmid);
3055         next if !$res->{$vmid}->{pid}; # not running
3056         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3057     }
3058
3059     $qmpclient->queue_execute(undef, 2);
3060
3061     foreach my $vmid (keys %$list) {
3062         next if $opt_vmid && ($vmid ne $opt_vmid);
3063         next if !$res->{$vmid}->{pid}; #not running
3064
3065         # we can't use the $qmpclient since it might have already aborted on
3066         # 'query-balloon', but this might also fail for older versions...
3067         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3068         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3069     }
3070
3071     foreach my $vmid (keys %$list) {
3072         next if $opt_vmid && ($vmid ne $opt_vmid);
3073         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3074     }
3075
3076     return $res;
3077 }
3078
3079 sub conf_has_serial {
3080     my ($conf) = @_;
3081
3082     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3083         if ($conf->{"serial$i"}) {
3084             return 1;
3085         }
3086     }
3087
3088     return 0;
3089 }
3090
3091 sub conf_has_audio {
3092     my ($conf, $id) = @_;
3093
3094     $id //= 0;
3095     my $audio = $conf->{"audio$id"};
3096     return if !defined($audio);
3097
3098     my $audioproperties = parse_property_string($audio_fmt, $audio);
3099     my $audiodriver = $audioproperties->{driver} // 'spice';
3100
3101     return {
3102         dev => $audioproperties->{device},
3103         dev_id => "audiodev$id",
3104         backend => $audiodriver,
3105         backend_id => "$audiodriver-backend${id}",
3106     };
3107 }
3108
3109 sub audio_devs {
3110     my ($audio, $audiopciaddr, $machine_version) = @_;
3111
3112     my $devs = [];
3113
3114     my $id = $audio->{dev_id};
3115     my $audiodev = "";
3116     if (min_version($machine_version, 4, 2)) {
3117         $audiodev = ",audiodev=$audio->{backend_id}";
3118     }
3119
3120     if ($audio->{dev} eq 'AC97') {
3121         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3122     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3123         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3124         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3125         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3126     } else {
3127         die "unkown audio device '$audio->{dev}', implement me!";
3128     }
3129
3130     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3131
3132     return $devs;
3133 }
3134
3135 sub get_tpm_paths {
3136     my ($vmid) = @_;
3137     return {
3138         socket => "/var/run/qemu-server/$vmid.swtpm",
3139         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3140     };
3141 }
3142
3143 sub add_tpm_device {
3144     my ($vmid, $devices, $conf) = @_;
3145
3146     return if !$conf->{tpmstate0};
3147
3148     my $paths = get_tpm_paths($vmid);
3149
3150     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3151     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3152     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3153 }
3154
3155 sub start_swtpm {
3156     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3157
3158     return if !$tpmdrive;
3159
3160     my $state;
3161     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3162     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3163     if ($storeid) {
3164         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3165     } else {
3166         $state = $tpm->{file};
3167     }
3168
3169     my $paths = get_tpm_paths($vmid);
3170
3171     # during migration, we will get state from remote
3172     #
3173     if (!$migration) {
3174         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3175         my $setup_cmd = [
3176             "swtpm_setup",
3177             "--tpmstate",
3178             "file://$state",
3179             "--createek",
3180             "--create-ek-cert",
3181             "--create-platform-cert",
3182             "--lock-nvram",
3183             "--config",
3184             "/etc/swtpm_setup.conf", # do not use XDG configs
3185             "--runas",
3186             "0", # force creation as root, error if not possible
3187             "--not-overwrite", # ignore existing state, do not modify
3188         ];
3189
3190         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3191         # TPM 2.0 supports ECC crypto, use if possible
3192         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3193
3194         run_command($setup_cmd, outfunc => sub {
3195             print "swtpm_setup: $1\n";
3196         });
3197     }
3198
3199     my $emulator_cmd = [
3200         "swtpm",
3201         "socket",
3202         "--tpmstate",
3203         "backend-uri=file://$state,mode=0600",
3204         "--ctrl",
3205         "type=unixio,path=$paths->{socket},mode=0600",
3206         "--pid",
3207         "file=$paths->{pid}",
3208         "--terminate", # terminate on QEMU disconnect
3209         "--daemon",
3210     ];
3211     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3212     run_command($emulator_cmd, outfunc => sub { print $1; });
3213
3214     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3215     while (! -e $paths->{pid}) {
3216         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3217         usleep(50_000);
3218     }
3219
3220     # return untainted PID of swtpm daemon so it can be killed on error
3221     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3222     return $1;
3223 }
3224
3225 sub vga_conf_has_spice {
3226     my ($vga) = @_;
3227
3228     my $vgaconf = parse_vga($vga);
3229     my $vgatype = $vgaconf->{type};
3230     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3231
3232     return $1 || 1;
3233 }
3234
3235 sub is_native($) {
3236     my ($arch) = @_;
3237     return get_host_arch() eq $arch;
3238 }
3239
3240 sub get_vm_arch {
3241     my ($conf) = @_;
3242     return $conf->{arch} // get_host_arch();
3243 }
3244
3245 my $default_machines = {
3246     x86_64 => 'pc',
3247     aarch64 => 'virt',
3248 };
3249
3250 sub get_installed_machine_version {
3251     my ($kvmversion) = @_;
3252     $kvmversion = kvm_user_version() if !defined($kvmversion);
3253     $kvmversion =~ m/^(\d+\.\d+)/;
3254     return $1;
3255 }
3256
3257 sub windows_get_pinned_machine_version {
3258     my ($machine, $base_version, $kvmversion) = @_;
3259
3260     my $pin_version = $base_version;
3261     if (!defined($base_version) ||
3262         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3263     ) {
3264         $pin_version = get_installed_machine_version($kvmversion);
3265     }
3266     if (!$machine || $machine eq 'pc') {
3267         $machine = "pc-i440fx-$pin_version";
3268     } elsif ($machine eq 'q35') {
3269         $machine = "pc-q35-$pin_version";
3270     } elsif ($machine eq 'virt') {
3271         $machine = "virt-$pin_version";
3272     } else {
3273         warn "unknown machine type '$machine', not touching that!\n";
3274     }
3275
3276     return $machine;
3277 }
3278
3279 sub get_vm_machine {
3280     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3281
3282     my $machine = $forcemachine || $conf->{machine};
3283
3284     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3285         $kvmversion //= kvm_user_version();
3286         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3287         # layout which confuses windows quite a bit and may result in various regressions..
3288         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3289         if (windows_version($conf->{ostype})) {
3290             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3291         }
3292         $arch //= 'x86_64';
3293         $machine ||= $default_machines->{$arch};
3294         if ($add_pve_version) {
3295             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3296             $machine .= "+pve$pvever";
3297         }
3298     }
3299
3300     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3301         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3302         $machine = $1 if $is_pxe;
3303
3304         # for version-pinned machines that do not include a pve-version (e.g.
3305         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3306         $machine .= '+pve0';
3307
3308         $machine .= '.pxe' if $is_pxe;
3309     }
3310
3311     return $machine;
3312 }
3313
3314 sub get_ovmf_files($$$) {
3315     my ($arch, $efidisk, $smm) = @_;
3316
3317     my $types = $OVMF->{$arch}
3318         or die "no OVMF images known for architecture '$arch'\n";
3319
3320     my $type = 'default';
3321     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3322         $type = $smm ? "4m" : "4m-no-smm";
3323         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3324     }
3325
3326     return $types->{$type}->@*;
3327 }
3328
3329 my $Arch2Qemu = {
3330     aarch64 => '/usr/bin/qemu-system-aarch64',
3331     x86_64 => '/usr/bin/qemu-system-x86_64',
3332 };
3333 sub get_command_for_arch($) {
3334     my ($arch) = @_;
3335     return '/usr/bin/kvm' if is_native($arch);
3336
3337     my $cmd = $Arch2Qemu->{$arch}
3338         or die "don't know how to emulate architecture '$arch'\n";
3339     return $cmd;
3340 }
3341
3342 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3343 # to use in a QEMU command line (-cpu element), first array_intersect the result
3344 # of query_supported_ with query_understood_. This is necessary because:
3345 #
3346 # a) query_understood_ returns flags the host cannot use and
3347 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3348 #    flags, but CPU settings - with most of them being flags. Those settings
3349 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3350 #
3351 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3352 # expensive. If you need the value returned from this, you can get it much
3353 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3354 # $accel being 'kvm' or 'tcg'.
3355 #
3356 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3357 # changes, automatically populating pmxcfs.
3358 #
3359 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3360 # since kvm and tcg machines support different flags
3361 #
3362 sub query_supported_cpu_flags {
3363     my ($arch) = @_;
3364
3365     $arch //= get_host_arch();
3366     my $default_machine = $default_machines->{$arch};
3367
3368     my $flags = {};
3369
3370     # FIXME: Once this is merged, the code below should work for ARM as well:
3371     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3372     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3373         $arch eq "aarch64";
3374
3375     my $kvm_supported = defined(kvm_version());
3376     my $qemu_cmd = get_command_for_arch($arch);
3377     my $fakevmid = -1;
3378     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3379
3380     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3381     my $query_supported_run_qemu = sub {
3382         my ($kvm) = @_;
3383
3384         my $flags = {};
3385         my $cmd = [
3386             $qemu_cmd,
3387             '-machine', $default_machine,
3388             '-display', 'none',
3389             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3390             '-mon', 'chardev=qmp,mode=control',
3391             '-pidfile', $pidfile,
3392             '-S', '-daemonize'
3393         ];
3394
3395         if (!$kvm) {
3396             push @$cmd, '-accel', 'tcg';
3397         }
3398
3399         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3400         die "QEMU flag querying VM exited with code " . $rc if $rc;
3401
3402         eval {
3403             my $cmd_result = mon_cmd(
3404                 $fakevmid,
3405                 'query-cpu-model-expansion',
3406                 type => 'full',
3407                 model => { name => 'host' }
3408             );
3409
3410             my $props = $cmd_result->{model}->{props};
3411             foreach my $prop (keys %$props) {
3412                 next if $props->{$prop} ne '1';
3413                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3414                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3415                 # We only keep those with underscores, to match /proc/cpuinfo
3416                 $prop =~ s/\.|-/_/g;
3417                 $flags->{$prop} = 1;
3418             }
3419         };
3420         my $err = $@;
3421
3422         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3423         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3424
3425         die $err if $err;
3426
3427         return [ sort keys %$flags ];
3428     };
3429
3430     # We need to query QEMU twice, since KVM and TCG have different supported flags
3431     PVE::QemuConfig->lock_config($fakevmid, sub {
3432         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3433         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3434
3435         if ($kvm_supported) {
3436             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3437             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3438         }
3439     });
3440
3441     return $flags;
3442 }
3443
3444 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3445 my $understood_cpu_flag_dir = "/usr/share/kvm";
3446 sub query_understood_cpu_flags {
3447     my $arch = get_host_arch();
3448     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3449
3450     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3451         if ! -e $filepath;
3452
3453     my $raw = file_get_contents($filepath);
3454     $raw =~ s/^\s+|\s+$//g;
3455     my @flags = split(/\s+/, $raw);
3456
3457     return \@flags;
3458 }
3459
3460 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3461 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3462 my sub should_disable_smm {
3463     my ($conf, $vga) = @_;
3464
3465     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3466         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3467 }
3468
3469 sub config_to_command {
3470     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3471         $pbs_backing) = @_;
3472
3473     my $cmd = [];
3474     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3475     my $devices = [];
3476     my $bridges = {};
3477     my $ostype = $conf->{ostype};
3478     my $winversion = windows_version($ostype);
3479     my $kvm = $conf->{kvm};
3480     my $nodename = nodename();
3481
3482     my $arch = get_vm_arch($conf);
3483     my $kvm_binary = get_command_for_arch($arch);
3484     my $kvmver = kvm_user_version($kvm_binary);
3485
3486     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3487         $kvmver //= "undefined";
3488         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3489     }
3490
3491     my $add_pve_version = min_version($kvmver, 4, 1);
3492
3493     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3494     my $machine_version = extract_version($machine_type, $kvmver);
3495     $kvm //= 1 if is_native($arch);
3496
3497     $machine_version =~ m/(\d+)\.(\d+)/;
3498     my ($machine_major, $machine_minor) = ($1, $2);
3499
3500     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3501         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3502     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3503         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3504             ." please upgrade node '$nodename'\n"
3505     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3506         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3507         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3508             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3509             ." node '$nodename'\n";
3510     }
3511
3512     # if a specific +pve version is required for a feature, use $version_guard
3513     # instead of min_version to allow machines to be run with the minimum
3514     # required version
3515     my $required_pve_version = 0;
3516     my $version_guard = sub {
3517         my ($major, $minor, $pve) = @_;
3518         return 0 if !min_version($machine_version, $major, $minor, $pve);
3519         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3520         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3521         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3522         return 1;
3523     };
3524
3525     if ($kvm && !defined kvm_version()) {
3526         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3527             ." or enable in BIOS.\n";
3528     }
3529
3530     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3531     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3532     my $use_old_bios_files = undef;
3533     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3534
3535     push @$cmd, $kvm_binary;
3536
3537     push @$cmd, '-id', $vmid;
3538
3539     my $vmname = $conf->{name} || "vm$vmid";
3540
3541     push @$cmd, '-name', "$vmname,debug-threads=on";
3542
3543     push @$cmd, '-no-shutdown';
3544
3545     my $use_virtio = 0;
3546
3547     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3548     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3549     push @$cmd, '-mon', "chardev=qmp,mode=control";
3550
3551     if (min_version($machine_version, 2, 12)) {
3552         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3553         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3554     }
3555
3556     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3557
3558     push @$cmd, '-daemonize';
3559
3560     if ($conf->{smbios1}) {
3561         my $smbios_conf = parse_smbios1($conf->{smbios1});
3562         if ($smbios_conf->{base64}) {
3563             # Do not pass base64 flag to qemu
3564             delete $smbios_conf->{base64};
3565             my $smbios_string = "";
3566             foreach my $key (keys %$smbios_conf) {
3567                 my $value;
3568                 if ($key eq "uuid") {
3569                     $value = $smbios_conf->{uuid}
3570                 } else {
3571                     $value = decode_base64($smbios_conf->{$key});
3572                 }
3573                 # qemu accepts any binary data, only commas need escaping by double comma
3574                 $value =~ s/,/,,/g;
3575                 $smbios_string .= "," . $key . "=" . $value if $value;
3576             }
3577             push @$cmd, '-smbios', "type=1" . $smbios_string;
3578         } else {
3579             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3580         }
3581     }
3582
3583     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3584         my $d;
3585         if (my $efidisk = $conf->{efidisk0}) {
3586             $d = parse_drive('efidisk0', $efidisk);
3587         }
3588
3589         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3590         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3591
3592         my ($path, $format);
3593         my $read_only_str = '';
3594         if ($d) {
3595             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3596             $format = $d->{format};
3597             if ($storeid) {
3598                 $path = PVE::Storage::path($storecfg, $d->{file});
3599                 if (!defined($format)) {
3600                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3601                     $format = qemu_img_format($scfg, $volname);
3602                 }
3603             } else {
3604                 $path = $d->{file};
3605                 die "efidisk format must be specified\n"
3606                     if !defined($format);
3607             }
3608
3609             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3610         } else {
3611             log_warn("no efidisk configured! Using temporary efivars disk.");
3612             $path = "/tmp/$vmid-ovmf.fd";
3613             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3614             $format = 'raw';
3615         }
3616
3617         my $size_str = "";
3618
3619         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3620             $size_str = ",size=" . (-s $ovmf_vars);
3621         }
3622
3623         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3624         my $cache = "";
3625         if ($path =~ m/^rbd:/) {
3626                 $cache = ',cache=writeback';
3627                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3628         }
3629
3630         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3631         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3632     }
3633
3634     if ($q35) { # tell QEMU to load q35 config early
3635         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3636         if (min_version($machine_version, 4, 0)) {
3637             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3638         } else {
3639             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3640         }
3641     }
3642
3643     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3644         push @$cmd, $fixups->@*;
3645     }
3646
3647     if ($conf->{vmgenid}) {
3648         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3649     }
3650
3651     # add usb controllers
3652     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3653         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3654     push @$devices, @usbcontrollers if @usbcontrollers;
3655     my $vga = parse_vga($conf->{vga});
3656
3657     my $qxlnum = vga_conf_has_spice($conf->{vga});
3658     $vga->{type} = 'qxl' if $qxlnum;
3659
3660     if (!$vga->{type}) {
3661         if ($arch eq 'aarch64') {
3662             $vga->{type} = 'virtio';
3663         } elsif (min_version($machine_version, 2, 9)) {
3664             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3665         } else {
3666             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3667         }
3668     }
3669
3670     # enable absolute mouse coordinates (needed by vnc)
3671     my $tablet = $conf->{tablet};
3672     if (!defined($tablet)) {
3673         $tablet = $defaults->{tablet};
3674         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3675         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3676     }
3677
3678     if ($tablet) {
3679         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3680         my $kbd = print_keyboarddevice_full($conf, $arch);
3681         push @$devices, '-device', $kbd if defined($kbd);
3682     }
3683
3684     my $bootorder = device_bootorder($conf);
3685
3686     # host pci device passthrough
3687     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3688         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3689
3690     # usb devices
3691     my $usb_dev_features = {};
3692     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3693
3694     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3695         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3696     push @$devices, @usbdevices if @usbdevices;
3697
3698     # serial devices
3699     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3700         my $path = $conf->{"serial$i"} or next;
3701         if ($path eq 'socket') {
3702             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3703             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3704             # On aarch64, serial0 is the UART device. Qemu only allows
3705             # connecting UART devices via the '-serial' command line, as
3706             # the device has a fixed slot on the hardware...
3707             if ($arch eq 'aarch64' && $i == 0) {
3708                 push @$devices, '-serial', "chardev:serial$i";
3709             } else {
3710                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3711             }
3712         } else {
3713             die "no such serial device\n" if ! -c $path;
3714             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3715             push @$devices, '-device', "isa-serial,chardev=serial$i";
3716         }
3717     }
3718
3719     # parallel devices
3720     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3721         if (my $path = $conf->{"parallel$i"}) {
3722             die "no such parallel device\n" if ! -c $path;
3723             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3724             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3725             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3726         }
3727     }
3728
3729     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3730         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3731         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3732         push @$devices, @$audio_devs;
3733     }
3734
3735     add_tpm_device($vmid, $devices, $conf);
3736
3737     my $sockets = 1;
3738     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3739     $sockets = $conf->{sockets} if  $conf->{sockets};
3740
3741     my $cores = $conf->{cores} || 1;
3742
3743     my $maxcpus = $sockets * $cores;
3744
3745     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3746
3747     my $allowed_vcpus = $cpuinfo->{cpus};
3748
3749     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3750
3751     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3752         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753         for (my $i = 2; $i <= $vcpus; $i++)  {
3754             my $cpustr = print_cpu_device($conf,$i);
3755             push @$cmd, '-device', $cpustr;
3756         }
3757
3758     } else {
3759
3760         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3761     }
3762     push @$cmd, '-nodefaults';
3763
3764     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3765
3766     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3767
3768     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3769
3770     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3771         push @$devices, '-device', print_vga_device(
3772             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3773
3774         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3775
3776         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3777         push @$cmd,  '-vnc', "unix:$socket,password=on";
3778     } else {
3779         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3780         push @$cmd, '-nographic';
3781     }
3782
3783     # time drift fix
3784     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3785     my $useLocaltime = $conf->{localtime};
3786
3787     if ($winversion >= 5) { # windows
3788         $useLocaltime = 1 if !defined($conf->{localtime});
3789
3790         # use time drift fix when acpi is enabled
3791         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3792             $tdf = 1 if !defined($conf->{tdf});
3793         }
3794     }
3795
3796     if ($winversion >= 6) {
3797         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3798         push @$cmd, '-no-hpet';
3799     }
3800
3801     push @$rtcFlags, 'driftfix=slew' if $tdf;
3802
3803     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3804         push @$rtcFlags, "base=$conf->{startdate}";
3805     } elsif ($useLocaltime) {
3806         push @$rtcFlags, 'base=localtime';
3807     }
3808
3809     if ($forcecpu) {
3810         push @$cmd, '-cpu', $forcecpu;
3811     } else {
3812         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3813     }
3814
3815     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3816
3817     push @$cmd, '-S' if $conf->{freeze};
3818
3819     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3820
3821     my $guest_agent = parse_guest_agent($conf);
3822
3823     if ($guest_agent->{enabled}) {
3824         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3825         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3826
3827         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3828             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3829             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3830             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3831         } elsif ($guest_agent->{type} eq 'isa') {
3832             push @$devices, '-device', "isa-serial,chardev=qga0";
3833         }
3834     }
3835
3836     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3837     if ($rng && $version_guard->(4, 1, 2)) {
3838         check_rng_source($rng->{source});
3839
3840         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3841         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3842         my $limiter_str = "";
3843         if ($max_bytes) {
3844             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3845         }
3846
3847         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3848         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3849         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3850     }
3851
3852     my $spice_port;
3853
3854     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3855         if ($qxlnum > 1) {
3856             if ($winversion){
3857                 for (my $i = 1; $i < $qxlnum; $i++){
3858                     push @$devices, '-device', print_vga_device(
3859                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3860                 }
3861             } else {
3862                 # assume other OS works like Linux
3863                 my ($ram, $vram) = ("134217728", "67108864");
3864                 if ($vga->{memory}) {
3865                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3866                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3867                 }
3868                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3869                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3870             }
3871         }
3872
3873         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3874
3875         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3876         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3877         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3878
3879         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3880         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3881         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3882
3883         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3884         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3885
3886         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3887         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3888         if ($spice_enhancement->{foldersharing}) {
3889             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3890             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3891         }
3892
3893         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3894         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3895             if $spice_enhancement->{videostreaming};
3896
3897         push @$devices, '-spice', "$spice_opts";
3898     }
3899
3900     # enable balloon by default, unless explicitly disabled
3901     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3902         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3903         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3904         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3905         push @$devices, '-device', $ballooncmd;
3906     }
3907
3908     if ($conf->{watchdog}) {
3909         my $wdopts = parse_watchdog($conf->{watchdog});
3910         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3911         my $watchdog = $wdopts->{model} || 'i6300esb';
3912         push @$devices, '-device', "$watchdog$pciaddr";
3913         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3914     }
3915
3916     my $vollist = [];
3917     my $scsicontroller = {};
3918     my $ahcicontroller = {};
3919     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3920
3921     # Add iscsi initiator name if available
3922     if (my $initiator = get_initiator_name()) {
3923         push @$devices, '-iscsi', "initiator-name=$initiator";
3924     }
3925
3926     PVE::QemuConfig->foreach_volume($conf, sub {
3927         my ($ds, $drive) = @_;
3928
3929         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3930             check_volume_storage_type($storecfg, $drive->{file});
3931             push @$vollist, $drive->{file};
3932         }
3933
3934         # ignore efidisk here, already added in bios/fw handling code above
3935         return if $drive->{interface} eq 'efidisk';
3936         # similar for TPM
3937         return if $drive->{interface} eq 'tpmstate';
3938
3939         $use_virtio = 1 if $ds =~ m/^virtio/;
3940
3941         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3942
3943         if ($drive->{interface} eq 'virtio'){
3944            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3945         }
3946
3947         if ($drive->{interface} eq 'scsi') {
3948
3949             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3950
3951             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3952                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3953
3954             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3955             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3956
3957             my $iothread = '';
3958             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3959                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3960                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3961             } elsif ($drive->{iothread}) {
3962                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3963             }
3964
3965             my $queues = '';
3966             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3967                 $queues = ",num_queues=$drive->{queues}";
3968             }
3969
3970             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3971                 if !$scsicontroller->{$controller};
3972             $scsicontroller->{$controller}=1;
3973         }
3974
3975         if ($drive->{interface} eq 'sata') {
3976             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3977             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3978             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3979                 if !$ahcicontroller->{$controller};
3980             $ahcicontroller->{$controller}=1;
3981         }
3982
3983         my $pbs_conf = $pbs_backing->{$ds};
3984         my $pbs_name = undef;
3985         if ($pbs_conf) {
3986             $pbs_name = "drive-$ds-pbs";
3987             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3988         }
3989
3990         my $drive_cmd = print_drive_commandline_full(
3991             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3992
3993         # extra protection for templates, but SATA and IDE don't support it..
3994         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3995
3996         push @$devices, '-drive',$drive_cmd;
3997         push @$devices, '-device', print_drivedevice_full(
3998             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3999     });
4000
4001     for (my $i = 0; $i < $MAX_NETS; $i++) {
4002         my $netname = "net$i";
4003
4004         next if !$conf->{$netname};
4005         my $d = parse_net($conf->{$netname});
4006         next if !$d;
4007
4008         $use_virtio = 1 if $d->{model} eq 'virtio';
4009
4010         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4011
4012         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4013         push @$devices, '-netdev', $netdevfull;
4014
4015         my $netdevicefull = print_netdevice_full(
4016             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4017
4018         push @$devices, '-device', $netdevicefull;
4019     }
4020
4021     if ($conf->{ivshmem}) {
4022         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4023
4024         my $bus;
4025         if ($q35) {
4026             $bus = print_pcie_addr("ivshmem");
4027         } else {
4028             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4029         }
4030
4031         my $ivshmem_name = $ivshmem->{name} // $vmid;
4032         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4033
4034         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4035         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4036             .",size=$ivshmem->{size}M";
4037     }
4038
4039     # pci.4 is nested in pci.1
4040     $bridges->{1} = 1 if $bridges->{4};
4041
4042     if (!$q35) { # add pci bridges
4043         if (min_version($machine_version, 2, 3)) {
4044            $bridges->{1} = 1;
4045            $bridges->{2} = 1;
4046         }
4047         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4048     }
4049
4050     for my $k (sort {$b cmp $a} keys %$bridges) {
4051         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4052
4053         my $k_name = $k;
4054         if ($k == 2 && $legacy_igd) {
4055             $k_name = "$k-igd";
4056         }
4057         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4058         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4059
4060         if ($q35) { # add after -readconfig pve-q35.cfg
4061             splice @$devices, 2, 0, '-device', $devstr;
4062         } else {
4063             unshift @$devices, '-device', $devstr if $k > 0;
4064         }
4065     }
4066
4067     if (!$kvm) {
4068         push @$machineFlags, 'accel=tcg';
4069     }
4070
4071     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4072
4073     my $machine_type_min = $machine_type;
4074     if ($add_pve_version) {
4075         $machine_type_min =~ s/\+pve\d+$//;
4076         $machine_type_min .= "+pve$required_pve_version";
4077     }
4078     push @$machineFlags, "type=${machine_type_min}";
4079
4080     push @$cmd, @$devices;
4081     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4082     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4083     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4084
4085     if (my $vmstate = $conf->{vmstate}) {
4086         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4087         push @$vollist, $vmstate;
4088         push @$cmd, '-loadstate', $statepath;
4089         print "activating and using '$vmstate' as vmstate\n";
4090     }
4091
4092     if (PVE::QemuConfig->is_template($conf)) {
4093         # needed to workaround base volumes being read-only
4094         push @$cmd, '-snapshot';
4095     }
4096
4097     # add custom args
4098     if ($conf->{args}) {
4099         my $aa = PVE::Tools::split_args($conf->{args});
4100         push @$cmd, @$aa;
4101     }
4102
4103     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4104 }
4105
4106 sub check_rng_source {
4107     my ($source) = @_;
4108
4109     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4110     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4111         if ! -e $source;
4112
4113     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4114     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4115         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4116         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4117         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4118             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4119             ." to the host.\n";
4120     }
4121 }
4122
4123 sub spice_port {
4124     my ($vmid) = @_;
4125
4126     my $res = mon_cmd($vmid, 'query-spice');
4127
4128     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4129 }
4130
4131 sub vm_devices_list {
4132     my ($vmid) = @_;
4133
4134     my $res = mon_cmd($vmid, 'query-pci');
4135     my $devices_to_check = [];
4136     my $devices = {};
4137     foreach my $pcibus (@$res) {
4138         push @$devices_to_check, @{$pcibus->{devices}},
4139     }
4140
4141     while (@$devices_to_check) {
4142         my $to_check = [];
4143         for my $d (@$devices_to_check) {
4144             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4145             next if !$d->{'pci_bridge'};
4146
4147             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4148             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4149         }
4150         $devices_to_check = $to_check;
4151     }
4152
4153     my $resblock = mon_cmd($vmid, 'query-block');
4154     foreach my $block (@$resblock) {
4155         if($block->{device} =~ m/^drive-(\S+)/){
4156                 $devices->{$1} = 1;
4157         }
4158     }
4159
4160     my $resmice = mon_cmd($vmid, 'query-mice');
4161     foreach my $mice (@$resmice) {
4162         if ($mice->{name} eq 'QEMU HID Tablet') {
4163             $devices->{tablet} = 1;
4164             last;
4165         }
4166     }
4167
4168     # for usb devices there is no query-usb
4169     # but we can iterate over the entries in
4170     # qom-list path=/machine/peripheral
4171     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4172     foreach my $per (@$resperipheral) {
4173         if ($per->{name} =~ m/^usb\d+$/) {
4174             $devices->{$per->{name}} = 1;
4175         }
4176     }
4177
4178     return $devices;
4179 }
4180
4181 sub vm_deviceplug {
4182     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4183
4184     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4185
4186     my $devices_list = vm_devices_list($vmid);
4187     return 1 if defined($devices_list->{$deviceid});
4188
4189     # add PCI bridge if we need it for the device
4190     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4191
4192     if ($deviceid eq 'tablet') {
4193         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4194     } elsif ($deviceid eq 'keyboard') {
4195         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4196     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4197         die "usb hotplug currently not reliable\n";
4198         # since we can't reliably hot unplug all added usb devices and usb
4199         # passthrough breaks live migration we disable usb hotplugging for now
4200         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4201     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4202         qemu_iothread_add($vmid, $deviceid, $device);
4203
4204         qemu_driveadd($storecfg, $vmid, $device);
4205         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4206
4207         qemu_deviceadd($vmid, $devicefull);
4208         eval { qemu_deviceaddverify($vmid, $deviceid); };
4209         if (my $err = $@) {
4210             eval { qemu_drivedel($vmid, $deviceid); };
4211             warn $@ if $@;
4212             die $err;
4213         }
4214     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4215         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4216         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4217         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4218
4219         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4220
4221         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4222             qemu_iothread_add($vmid, $deviceid, $device);
4223             $devicefull .= ",iothread=iothread-$deviceid";
4224         }
4225
4226         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4227             $devicefull .= ",num_queues=$device->{queues}";
4228         }
4229
4230         qemu_deviceadd($vmid, $devicefull);
4231         qemu_deviceaddverify($vmid, $deviceid);
4232     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4233         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4234         qemu_driveadd($storecfg, $vmid, $device);
4235
4236         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4237         eval { qemu_deviceadd($vmid, $devicefull); };
4238         if (my $err = $@) {
4239             eval { qemu_drivedel($vmid, $deviceid); };
4240             warn $@ if $@;
4241             die $err;
4242         }
4243     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4244         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4245
4246         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4247         my $use_old_bios_files = undef;
4248         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4249
4250         my $netdevicefull = print_netdevice_full(
4251             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4252         qemu_deviceadd($vmid, $netdevicefull);
4253         eval {
4254             qemu_deviceaddverify($vmid, $deviceid);
4255             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4256         };
4257         if (my $err = $@) {
4258             eval { qemu_netdevdel($vmid, $deviceid); };
4259             warn $@ if $@;
4260             die $err;
4261         }
4262     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4263         my $bridgeid = $2;
4264         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4265         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4266
4267         qemu_deviceadd($vmid, $devicefull);
4268         qemu_deviceaddverify($vmid, $deviceid);
4269     } else {
4270         die "can't hotplug device '$deviceid'\n";
4271     }
4272
4273     return 1;
4274 }
4275
4276 # fixme: this should raise exceptions on error!
4277 sub vm_deviceunplug {
4278     my ($vmid, $conf, $deviceid) = @_;
4279
4280     my $devices_list = vm_devices_list($vmid);
4281     return 1 if !defined($devices_list->{$deviceid});
4282
4283     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4284     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4285
4286     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4287         qemu_devicedel($vmid, $deviceid);
4288     } elsif ($deviceid =~ m/^usb\d+$/) {
4289         die "usb hotplug currently not reliable\n";
4290         # when unplugging usb devices this way, there may be remaining usb
4291         # controllers/hubs so we disable it for now
4292         #qemu_devicedel($vmid, $deviceid);
4293         #qemu_devicedelverify($vmid, $deviceid);
4294     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4295         my $device = parse_drive($deviceid, $conf->{$deviceid});
4296
4297         qemu_devicedel($vmid, $deviceid);
4298         qemu_devicedelverify($vmid, $deviceid);
4299         qemu_drivedel($vmid, $deviceid);
4300         qemu_iothread_del($vmid, $deviceid, $device);
4301     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4302         qemu_devicedel($vmid, $deviceid);
4303         qemu_devicedelverify($vmid, $deviceid);
4304     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4305         my $device = parse_drive($deviceid, $conf->{$deviceid});
4306
4307         qemu_devicedel($vmid, $deviceid);
4308         qemu_devicedelverify($vmid, $deviceid);
4309         qemu_drivedel($vmid, $deviceid);
4310         qemu_deletescsihw($conf, $vmid, $deviceid);
4311
4312         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4313             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4314     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4315         qemu_devicedel($vmid, $deviceid);
4316         qemu_devicedelverify($vmid, $deviceid);
4317         qemu_netdevdel($vmid, $deviceid);
4318     } else {
4319         die "can't unplug device '$deviceid'\n";
4320     }
4321
4322     return 1;
4323 }
4324
4325 sub qemu_deviceadd {
4326     my ($vmid, $devicefull) = @_;
4327
4328     $devicefull = "driver=".$devicefull;
4329     my %options =  split(/[=,]/, $devicefull);
4330
4331     mon_cmd($vmid, "device_add" , %options);
4332 }
4333
4334 sub qemu_devicedel {
4335     my ($vmid, $deviceid) = @_;
4336
4337     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4338 }
4339
4340 sub qemu_iothread_add {
4341     my ($vmid, $deviceid, $device) = @_;
4342
4343     if ($device->{iothread}) {
4344         my $iothreads = vm_iothreads_list($vmid);
4345         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4346     }
4347 }
4348
4349 sub qemu_iothread_del {
4350     my ($vmid, $deviceid, $device) = @_;
4351
4352     if ($device->{iothread}) {
4353         my $iothreads = vm_iothreads_list($vmid);
4354         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4355     }
4356 }
4357
4358 sub qemu_objectadd {
4359     my ($vmid, $objectid, $qomtype) = @_;
4360
4361     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4362
4363     return 1;
4364 }
4365
4366 sub qemu_objectdel {
4367     my ($vmid, $objectid) = @_;
4368
4369     mon_cmd($vmid, "object-del", id => $objectid);
4370
4371     return 1;
4372 }
4373
4374 sub qemu_driveadd {
4375     my ($storecfg, $vmid, $device) = @_;
4376
4377     my $kvmver = get_running_qemu_version($vmid);
4378     my $io_uring = min_version($kvmver, 6, 0);
4379     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4380     $drive =~ s/\\/\\\\/g;
4381     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4382
4383     # If the command succeeds qemu prints: "OK"
4384     return 1 if $ret =~ m/OK/s;
4385
4386     die "adding drive failed: $ret\n";
4387 }
4388
4389 sub qemu_drivedel {
4390     my ($vmid, $deviceid) = @_;
4391
4392     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4393     $ret =~ s/^\s+//;
4394
4395     return 1 if $ret eq "";
4396
4397     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4398     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4399
4400     die "deleting drive $deviceid failed : $ret\n";
4401 }
4402
4403 sub qemu_deviceaddverify {
4404     my ($vmid, $deviceid) = @_;
4405
4406     for (my $i = 0; $i <= 5; $i++) {
4407          my $devices_list = vm_devices_list($vmid);
4408          return 1 if defined($devices_list->{$deviceid});
4409          sleep 1;
4410     }
4411
4412     die "error on hotplug device '$deviceid'\n";
4413 }
4414
4415
4416 sub qemu_devicedelverify {
4417     my ($vmid, $deviceid) = @_;
4418
4419     # need to verify that the device is correctly removed as device_del
4420     # is async and empty return is not reliable
4421
4422     for (my $i = 0; $i <= 5; $i++) {
4423          my $devices_list = vm_devices_list($vmid);
4424          return 1 if !defined($devices_list->{$deviceid});
4425          sleep 1;
4426     }
4427
4428     die "error on hot-unplugging device '$deviceid'\n";
4429 }
4430
4431 sub qemu_findorcreatescsihw {
4432     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4433
4434     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4435
4436     my $scsihwid="$controller_prefix$controller";
4437     my $devices_list = vm_devices_list($vmid);
4438
4439     if (!defined($devices_list->{$scsihwid})) {
4440         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4441     }
4442
4443     return 1;
4444 }
4445
4446 sub qemu_deletescsihw {
4447     my ($conf, $vmid, $opt) = @_;
4448
4449     my $device = parse_drive($opt, $conf->{$opt});
4450
4451     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4452         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4453         return 1;
4454     }
4455
4456     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4457
4458     my $devices_list = vm_devices_list($vmid);
4459     foreach my $opt (keys %{$devices_list}) {
4460         if (is_valid_drivename($opt)) {
4461             my $drive = parse_drive($opt, $conf->{$opt});
4462             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4463                 return 1;
4464             }
4465         }
4466     }
4467
4468     my $scsihwid="scsihw$controller";
4469
4470     vm_deviceunplug($vmid, $conf, $scsihwid);
4471
4472     return 1;
4473 }
4474
4475 sub qemu_add_pci_bridge {
4476     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4477
4478     my $bridges = {};
4479
4480     my $bridgeid;
4481
4482     print_pci_addr($device, $bridges, $arch, $machine_type);
4483
4484     while (my ($k, $v) = each %$bridges) {
4485         $bridgeid = $k;
4486     }
4487     return 1 if !defined($bridgeid) || $bridgeid < 1;
4488
4489     my $bridge = "pci.$bridgeid";
4490     my $devices_list = vm_devices_list($vmid);
4491
4492     if (!defined($devices_list->{$bridge})) {
4493         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4494     }
4495
4496     return 1;
4497 }
4498
4499 sub qemu_set_link_status {
4500     my ($vmid, $device, $up) = @_;
4501
4502     mon_cmd($vmid, "set_link", name => $device,
4503                up => $up ? JSON::true : JSON::false);
4504 }
4505
4506 sub qemu_netdevadd {
4507     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4508
4509     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4510     my %options =  split(/[=,]/, $netdev);
4511
4512     if (defined(my $vhost = $options{vhost})) {
4513         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4514     }
4515
4516     if (defined(my $queues = $options{queues})) {
4517         $options{queues} = $queues + 0;
4518     }
4519
4520     mon_cmd($vmid, "netdev_add",  %options);
4521     return 1;
4522 }
4523
4524 sub qemu_netdevdel {
4525     my ($vmid, $deviceid) = @_;
4526
4527     mon_cmd($vmid, "netdev_del", id => $deviceid);
4528 }
4529
4530 sub qemu_usb_hotplug {
4531     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4532
4533     return if !$device;
4534
4535     # remove the old one first
4536     vm_deviceunplug($vmid, $conf, $deviceid);
4537
4538     # check if xhci controller is necessary and available
4539     if ($device->{usb3}) {
4540
4541         my $devicelist = vm_devices_list($vmid);
4542
4543         if (!$devicelist->{xhci}) {
4544             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4545             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4546         }
4547     }
4548     my $d = parse_usb_device($device->{host});
4549     $d->{usb3} = $device->{usb3};
4550
4551     # add the new one
4552     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4553 }
4554
4555 sub qemu_cpu_hotplug {
4556     my ($vmid, $conf, $vcpus) = @_;
4557
4558     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4559
4560     my $sockets = 1;
4561     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4562     $sockets = $conf->{sockets} if  $conf->{sockets};
4563     my $cores = $conf->{cores} || 1;
4564     my $maxcpus = $sockets * $cores;
4565
4566     $vcpus = $maxcpus if !$vcpus;
4567
4568     die "you can't add more vcpus than maxcpus\n"
4569         if $vcpus > $maxcpus;
4570
4571     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4572
4573     if ($vcpus < $currentvcpus) {
4574
4575         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4576
4577             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4578                 qemu_devicedel($vmid, "cpu$i");
4579                 my $retry = 0;
4580                 my $currentrunningvcpus = undef;
4581                 while (1) {
4582                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4583                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4584                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4585                     $retry++;
4586                     sleep 1;
4587                 }
4588                 #update conf after each succesfull cpu unplug
4589                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4590                 PVE::QemuConfig->write_config($vmid, $conf);
4591             }
4592         } else {
4593             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4594         }
4595
4596         return;
4597     }
4598
4599     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600     die "vcpus in running vm does not match its configuration\n"
4601         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4602
4603     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4604
4605         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4606             my $cpustr = print_cpu_device($conf, $i);
4607             qemu_deviceadd($vmid, $cpustr);
4608
4609             my $retry = 0;
4610             my $currentrunningvcpus = undef;
4611             while (1) {
4612                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4613                 last if scalar(@{$currentrunningvcpus}) == $i;
4614                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4615                 sleep 1;
4616                 $retry++;
4617             }
4618             #update conf after each succesfull cpu hotplug
4619             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4620             PVE::QemuConfig->write_config($vmid, $conf);
4621         }
4622     } else {
4623
4624         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4625             mon_cmd($vmid, "cpu-add", id => int($i));
4626         }
4627     }
4628 }
4629
4630 sub qemu_block_set_io_throttle {
4631     my ($vmid, $deviceid,
4632         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4633         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4634         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4635         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4636
4637     return if !check_running($vmid) ;
4638
4639     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4640         bps => int($bps),
4641         bps_rd => int($bps_rd),
4642         bps_wr => int($bps_wr),
4643         iops => int($iops),
4644         iops_rd => int($iops_rd),
4645         iops_wr => int($iops_wr),
4646         bps_max => int($bps_max),
4647         bps_rd_max => int($bps_rd_max),
4648         bps_wr_max => int($bps_wr_max),
4649         iops_max => int($iops_max),
4650         iops_rd_max => int($iops_rd_max),
4651         iops_wr_max => int($iops_wr_max),
4652         bps_max_length => int($bps_max_length),
4653         bps_rd_max_length => int($bps_rd_max_length),
4654         bps_wr_max_length => int($bps_wr_max_length),
4655         iops_max_length => int($iops_max_length),
4656         iops_rd_max_length => int($iops_rd_max_length),
4657         iops_wr_max_length => int($iops_wr_max_length),
4658     );
4659
4660 }
4661
4662 sub qemu_block_resize {
4663     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4664
4665     my $running = check_running($vmid);
4666
4667     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4668
4669     return if !$running;
4670
4671     my $padding = (1024 - $size % 1024) % 1024;
4672     $size = $size + $padding;
4673
4674     mon_cmd(
4675         $vmid,
4676         "block_resize",
4677         device => $deviceid,
4678         size => int($size),
4679         timeout => 60,
4680     );
4681 }
4682
4683 sub qemu_volume_snapshot {
4684     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4685
4686     my $running = check_running($vmid);
4687
4688     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4689         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4690     } else {
4691         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4692     }
4693 }
4694
4695 sub qemu_volume_snapshot_delete {
4696     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4697
4698     my $running = check_running($vmid);
4699
4700     if($running) {
4701
4702         $running = undef;
4703         my $conf = PVE::QemuConfig->load_config($vmid);
4704         PVE::QemuConfig->foreach_volume($conf, sub {
4705             my ($ds, $drive) = @_;
4706             $running = 1 if $drive->{file} eq $volid;
4707         });
4708     }
4709
4710     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4711         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4712     } else {
4713         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4714     }
4715 }
4716
4717 sub set_migration_caps {
4718     my ($vmid, $savevm) = @_;
4719
4720     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4721
4722     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4723     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4724
4725     my $cap_ref = [];
4726
4727     my $enabled_cap = {
4728         "auto-converge" => 1,
4729         "xbzrle" => 1,
4730         "x-rdma-pin-all" => 0,
4731         "zero-blocks" => 0,
4732         "compress" => 0,
4733         "dirty-bitmaps" => $dirty_bitmaps,
4734     };
4735
4736     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4737
4738     for my $supported_capability (@$supported_capabilities) {
4739         push @$cap_ref, {
4740             capability => $supported_capability->{capability},
4741             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4742         };
4743     }
4744
4745     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4746 }
4747
4748 sub foreach_volid {
4749     my ($conf, $func, @param) = @_;
4750
4751     my $volhash = {};
4752
4753     my $test_volid = sub {
4754         my ($key, $drive, $snapname) = @_;
4755
4756         my $volid = $drive->{file};
4757         return if !$volid;
4758
4759         $volhash->{$volid}->{cdrom} //= 1;
4760         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4761
4762         my $replicate = $drive->{replicate} // 1;
4763         $volhash->{$volid}->{replicate} //= 0;
4764         $volhash->{$volid}->{replicate} = 1 if $replicate;
4765
4766         $volhash->{$volid}->{shared} //= 0;
4767         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4768
4769         $volhash->{$volid}->{referenced_in_config} //= 0;
4770         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4771
4772         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4773             if defined($snapname);
4774
4775         my $size = $drive->{size};
4776         $volhash->{$volid}->{size} //= $size if $size;
4777
4778         $volhash->{$volid}->{is_vmstate} //= 0;
4779         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4780
4781         $volhash->{$volid}->{is_tpmstate} //= 0;
4782         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4783
4784         $volhash->{$volid}->{is_unused} //= 0;
4785         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4786
4787         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4788     };
4789
4790     my $include_opts = {
4791         extra_keys => ['vmstate'],
4792         include_unused => 1,
4793     };
4794
4795     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4796     foreach my $snapname (keys %{$conf->{snapshots}}) {
4797         my $snap = $conf->{snapshots}->{$snapname};
4798         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4799     }
4800
4801     foreach my $volid (keys %$volhash) {
4802         &$func($volid, $volhash->{$volid}, @param);
4803     }
4804 }
4805
4806 my $fast_plug_option = {
4807     'lock' => 1,
4808     'name' => 1,
4809     'onboot' => 1,
4810     'shares' => 1,
4811     'startup' => 1,
4812     'description' => 1,
4813     'protection' => 1,
4814     'vmstatestorage' => 1,
4815     'hookscript' => 1,
4816     'tags' => 1,
4817 };
4818
4819 # hotplug changes in [PENDING]
4820 # $selection hash can be used to only apply specified options, for
4821 # example: { cores => 1 } (only apply changed 'cores')
4822 # $errors ref is used to return error messages
4823 sub vmconfig_hotplug_pending {
4824     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4825
4826     my $defaults = load_defaults();
4827     my $arch = get_vm_arch($conf);
4828     my $machine_type = get_vm_machine($conf, undef, $arch);
4829
4830     # commit values which do not have any impact on running VM first
4831     # Note: those option cannot raise errors, we we do not care about
4832     # $selection and always apply them.
4833
4834     my $add_error = sub {
4835         my ($opt, $msg) = @_;
4836         $errors->{$opt} = "hotplug problem - $msg";
4837     };
4838
4839     my $changes = 0;
4840     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4841         if ($fast_plug_option->{$opt}) {
4842             $conf->{$opt} = $conf->{pending}->{$opt};
4843             delete $conf->{pending}->{$opt};
4844             $changes = 1;
4845         }
4846     }
4847
4848     if ($changes) {
4849         PVE::QemuConfig->write_config($vmid, $conf);
4850     }
4851
4852     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4853
4854     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4855     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4856     foreach my $opt (sort keys %$pending_delete_hash) {
4857         next if $selection && !$selection->{$opt};
4858         my $force = $pending_delete_hash->{$opt}->{force};
4859         eval {
4860             if ($opt eq 'hotplug') {
4861                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4862             } elsif ($opt eq 'tablet') {
4863                 die "skip\n" if !$hotplug_features->{usb};
4864                 if ($defaults->{tablet}) {
4865                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4866                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4867                         if $arch eq 'aarch64';
4868                 } else {
4869                     vm_deviceunplug($vmid, $conf, 'tablet');
4870                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4871                 }
4872             } elsif ($opt =~ m/^usb\d+/) {
4873                 die "skip\n";
4874                 # since we cannot reliably hot unplug usb devices we are disabling it
4875                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4876                 #vm_deviceunplug($vmid, $conf, $opt);
4877             } elsif ($opt eq 'vcpus') {
4878                 die "skip\n" if !$hotplug_features->{cpu};
4879                 qemu_cpu_hotplug($vmid, $conf, undef);
4880             } elsif ($opt eq 'balloon') {
4881                 # enable balloon device is not hotpluggable
4882                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4883                 # here we reset the ballooning value to memory
4884                 my $balloon = $conf->{memory} || $defaults->{memory};
4885                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4886             } elsif ($fast_plug_option->{$opt}) {
4887                 # do nothing
4888             } elsif ($opt =~ m/^net(\d+)$/) {
4889                 die "skip\n" if !$hotplug_features->{network};
4890                 vm_deviceunplug($vmid, $conf, $opt);
4891             } elsif (is_valid_drivename($opt)) {
4892                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4893                 vm_deviceunplug($vmid, $conf, $opt);
4894                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4895             } elsif ($opt =~ m/^memory$/) {
4896                 die "skip\n" if !$hotplug_features->{memory};
4897                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4898             } elsif ($opt eq 'cpuunits') {
4899                 $cgroup->change_cpu_shares(undef);
4900             } elsif ($opt eq 'cpulimit') {
4901                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4902             } else {
4903                 die "skip\n";
4904             }
4905         };
4906         if (my $err = $@) {
4907             &$add_error($opt, $err) if $err ne "skip\n";
4908         } else {
4909             delete $conf->{$opt};
4910             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4911         }
4912     }
4913
4914     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4915     $apply_pending_cloudinit = sub {
4916         return if $apply_pending_cloudinit_done; # once is enough
4917         $apply_pending_cloudinit_done = 1; # once is enough
4918
4919         my ($key, $value) = @_;
4920
4921         my @cloudinit_opts = keys %$confdesc_cloudinit;
4922         foreach my $opt (keys %{$conf->{pending}}) {
4923             next if !grep { $_ eq $opt } @cloudinit_opts;
4924             $conf->{$opt} = delete $conf->{pending}->{$opt};
4925         }
4926
4927         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4928         foreach my $opt (sort keys %$pending_delete_hash) {
4929             next if !grep { $_ eq $opt } @cloudinit_opts;
4930             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4931             delete $conf->{$opt};
4932         }
4933
4934         my $new_conf = { %$conf };
4935         $new_conf->{$key} = $value;
4936         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4937     };
4938
4939     foreach my $opt (keys %{$conf->{pending}}) {
4940         next if $selection && !$selection->{$opt};
4941         my $value = $conf->{pending}->{$opt};
4942         eval {
4943             if ($opt eq 'hotplug') {
4944                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4945             } elsif ($opt eq 'tablet') {
4946                 die "skip\n" if !$hotplug_features->{usb};
4947                 if ($value == 1) {
4948                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4949                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4950                         if $arch eq 'aarch64';
4951                 } elsif ($value == 0) {
4952                     vm_deviceunplug($vmid, $conf, 'tablet');
4953                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4954                 }
4955             } elsif ($opt =~ m/^usb\d+$/) {
4956                 die "skip\n";
4957                 # since we cannot reliably hot unplug usb devices we disable it for now
4958                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4959                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4960                 #die "skip\n" if !$d;
4961                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4962             } elsif ($opt eq 'vcpus') {
4963                 die "skip\n" if !$hotplug_features->{cpu};
4964                 qemu_cpu_hotplug($vmid, $conf, $value);
4965             } elsif ($opt eq 'balloon') {
4966                 # enable/disable balloning device is not hotpluggable
4967                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4968                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4969                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4970
4971                 # allow manual ballooning if shares is set to zero
4972                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4973                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4974                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4975                 }
4976             } elsif ($opt =~ m/^net(\d+)$/) {
4977                 # some changes can be done without hotplug
4978                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4979                                     $vmid, $opt, $value, $arch, $machine_type);
4980             } elsif (is_valid_drivename($opt)) {
4981                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4982                 # some changes can be done without hotplug
4983                 my $drive = parse_drive($opt, $value);
4984                 if (drive_is_cloudinit($drive)) {
4985                     &$apply_pending_cloudinit($opt, $value);
4986                 }
4987                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4988                                      $vmid, $opt, $value, $arch, $machine_type);
4989             } elsif ($opt =~ m/^memory$/) { #dimms
4990                 die "skip\n" if !$hotplug_features->{memory};
4991                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4992             } elsif ($opt eq 'cpuunits') {
4993                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
4994                 $cgroup->change_cpu_shares($new_cpuunits);
4995             } elsif ($opt eq 'cpulimit') {
4996                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4997                 $cgroup->change_cpu_quota($cpulimit, 100000);
4998             } elsif ($opt eq 'agent') {
4999                 vmconfig_update_agent($conf, $opt, $value);
5000             } else {
5001                 die "skip\n";  # skip non-hot-pluggable options
5002             }
5003         };
5004         if (my $err = $@) {
5005             &$add_error($opt, $err) if $err ne "skip\n";
5006         } else {
5007             $conf->{$opt} = $value;
5008             delete $conf->{pending}->{$opt};
5009         }
5010     }
5011
5012     PVE::QemuConfig->write_config($vmid, $conf);
5013 }
5014
5015 sub try_deallocate_drive {
5016     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5017
5018     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5019         my $volid = $drive->{file};
5020         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5021             my $sid = PVE::Storage::parse_volume_id($volid);
5022             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5023
5024             # check if the disk is really unused
5025             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5026                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5027             PVE::Storage::vdisk_free($storecfg, $volid);
5028             return 1;
5029         } else {
5030             # If vm is not owner of this disk remove from config
5031             return 1;
5032         }
5033     }
5034
5035     return;
5036 }
5037
5038 sub vmconfig_delete_or_detach_drive {
5039     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5040
5041     my $drive = parse_drive($opt, $conf->{$opt});
5042
5043     my $rpcenv = PVE::RPCEnvironment::get();
5044     my $authuser = $rpcenv->get_user();
5045
5046     if ($force) {
5047         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5048         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5049     } else {
5050         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5051     }
5052 }
5053
5054
5055
5056 sub vmconfig_apply_pending {
5057     my ($vmid, $conf, $storecfg, $errors) = @_;
5058
5059     return if !scalar(keys %{$conf->{pending}});
5060
5061     my $add_apply_error = sub {
5062         my ($opt, $msg) = @_;
5063         my $err_msg = "unable to apply pending change $opt : $msg";
5064         $errors->{$opt} = $err_msg;
5065         warn $err_msg;
5066     };
5067
5068     # cold plug
5069
5070     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5071     foreach my $opt (sort keys %$pending_delete_hash) {
5072         my $force = $pending_delete_hash->{$opt}->{force};
5073         eval {
5074             if ($opt =~ m/^unused/) {
5075                 die "internal error";
5076             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5077                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5078             }
5079         };
5080         if (my $err = $@) {
5081             $add_apply_error->($opt, $err);
5082         } else {
5083             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5084             delete $conf->{$opt};
5085         }
5086     }
5087
5088     PVE::QemuConfig->cleanup_pending($conf);
5089
5090     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5091         next if $opt eq 'delete'; # just to be sure
5092         eval {
5093             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5094                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5095             }
5096         };
5097         if (my $err = $@) {
5098             $add_apply_error->($opt, $err);
5099         } else {
5100             $conf->{$opt} = delete $conf->{pending}->{$opt};
5101         }
5102     }
5103
5104     # write all changes at once to avoid unnecessary i/o
5105     PVE::QemuConfig->write_config($vmid, $conf);
5106 }
5107
5108 sub vmconfig_update_net {
5109     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5110
5111     my $newnet = parse_net($value);
5112
5113     if ($conf->{$opt}) {
5114         my $oldnet = parse_net($conf->{$opt});
5115
5116         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5117             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5118             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5119             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5120
5121             # for non online change, we try to hot-unplug
5122             die "skip\n" if !$hotplug;
5123             vm_deviceunplug($vmid, $conf, $opt);
5124         } else {
5125
5126             die "internal error" if $opt !~ m/net(\d+)/;
5127             my $iface = "tap${vmid}i$1";
5128
5129             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5130                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5131                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5132                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5133                 PVE::Network::tap_unplug($iface);
5134
5135                 if ($have_sdn) {
5136                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5137                 } else {
5138                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5139                 }
5140             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5141                 # Rate can be applied on its own but any change above needs to
5142                 # include the rate in tap_plug since OVS resets everything.
5143                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5144             }
5145
5146             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5147                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5148             }
5149
5150             return 1;
5151         }
5152     }
5153
5154     if ($hotplug) {
5155         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5156     } else {
5157         die "skip\n";
5158     }
5159 }
5160
5161 sub vmconfig_update_agent {
5162     my ($conf, $opt, $value) = @_;
5163
5164     die "skip\n" if !$conf->{$opt};
5165
5166     my $hotplug_options = { fstrim_cloned_disks => 1 };
5167
5168     my $old_agent = parse_guest_agent($conf);
5169     my $agent = parse_guest_agent({$opt => $value});
5170
5171     for my $option (keys %$agent) { # added/changed options
5172         next if defined($hotplug_options->{$option});
5173         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5174     }
5175
5176     for my $option (keys %$old_agent) { # removed options
5177         next if defined($hotplug_options->{$option});
5178         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5179     }
5180
5181     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5182 }
5183
5184 sub vmconfig_update_disk {
5185     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5186
5187     my $drive = parse_drive($opt, $value);
5188
5189     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5190         my $media = $drive->{media} || 'disk';
5191         my $oldmedia = $old_drive->{media} || 'disk';
5192         die "unable to change media type\n" if $media ne $oldmedia;
5193
5194         if (!drive_is_cdrom($old_drive)) {
5195
5196             if ($drive->{file} ne $old_drive->{file}) {
5197
5198                 die "skip\n" if !$hotplug;
5199
5200                 # unplug and register as unused
5201                 vm_deviceunplug($vmid, $conf, $opt);
5202                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5203
5204             } else {
5205                 # update existing disk
5206
5207                 # skip non hotpluggable value
5208                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5209                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5210                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5211                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5212                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5213                     die "skip\n";
5214                 }
5215
5216                 # apply throttle
5217                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5218                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5219                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5220                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5221                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5222                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5223                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5224                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5225                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5226                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5227                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5228                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5229                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5230                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5231                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5232                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5233                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5234                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5235
5236                     qemu_block_set_io_throttle(
5237                         $vmid,"drive-$opt",
5238                         ($drive->{mbps} || 0)*1024*1024,
5239                         ($drive->{mbps_rd} || 0)*1024*1024,
5240                         ($drive->{mbps_wr} || 0)*1024*1024,
5241                          $drive->{iops} || 0,
5242                          $drive->{iops_rd} || 0,
5243                          $drive->{iops_wr} || 0,
5244                         ($drive->{mbps_max} || 0)*1024*1024,
5245                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5246                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5247                          $drive->{iops_max} || 0,
5248                          $drive->{iops_rd_max} || 0,
5249                          $drive->{iops_wr_max} || 0,
5250                          $drive->{bps_max_length} || 1,
5251                          $drive->{bps_rd_max_length} || 1,
5252                          $drive->{bps_wr_max_length} || 1,
5253                          $drive->{iops_max_length} || 1,
5254                          $drive->{iops_rd_max_length} || 1,
5255                          $drive->{iops_wr_max_length} || 1,
5256                     );
5257
5258                 }
5259
5260                 return 1;
5261             }
5262
5263         } else { # cdrom
5264
5265             if ($drive->{file} eq 'none') {
5266                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5267                 if (drive_is_cloudinit($old_drive)) {
5268                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5269                 }
5270             } else {
5271                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5272
5273                 # force eject if locked
5274                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5275
5276                 if ($path) {
5277                     mon_cmd($vmid, "blockdev-change-medium",
5278                         id => "$opt", filename => "$path");
5279                 }
5280             }
5281
5282             return 1;
5283         }
5284     }
5285
5286     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5287     # hotplug new disks
5288     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5289     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5290 }
5291
5292 # called in locked context by incoming migration
5293 sub vm_migrate_get_nbd_disks {
5294     my ($storecfg, $conf, $replicated_volumes) = @_;
5295
5296     my $local_volumes = {};
5297     PVE::QemuConfig->foreach_volume($conf, sub {
5298         my ($ds, $drive) = @_;
5299
5300         return if drive_is_cdrom($drive);
5301         return if $ds eq 'tpmstate0';
5302
5303         my $volid = $drive->{file};
5304
5305         return if !$volid;
5306
5307         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5308
5309         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5310         return if $scfg->{shared};
5311
5312         # replicated disks re-use existing state via bitmap
5313         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5314         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5315     });
5316     return $local_volumes;
5317 }
5318
5319 # called in locked context by incoming migration
5320 sub vm_migrate_alloc_nbd_disks {
5321     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5322
5323     my $nbd = {};
5324     foreach my $opt (sort keys %$source_volumes) {
5325         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5326
5327         if ($use_existing) {
5328             $nbd->{$opt}->{drivestr} = print_drive($drive);
5329             $nbd->{$opt}->{volid} = $volid;
5330             $nbd->{$opt}->{replicated} = 1;
5331             next;
5332         }
5333
5334         # storage mapping + volname = regular migration
5335         # storage mapping + format = remote migration
5336         # order of precedence, filtered by whether storage supports it:
5337         # 1. explicit requested format
5338         # 2. format of current volume
5339         # 3. default format of storage
5340         if (!$storagemap->{identity}) {
5341             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5342             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5343             if (!$format || !grep { $format eq $_ } @$validFormats) {
5344                 if ($volname) {
5345                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5346                     my $fileFormat = qemu_img_format($scfg, $volname);
5347                     $format = $fileFormat
5348                         if grep { $fileFormat eq $_ } @$validFormats;
5349                 }
5350                 $format //= $defFormat;
5351             }
5352         } else {
5353             # can't happen for remote migration, so $volname is always defined
5354             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5355             $format = qemu_img_format($scfg, $volname);
5356         }
5357
5358         my $size = $drive->{size} / 1024;
5359         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5360         my $newdrive = $drive;
5361         $newdrive->{format} = $format;
5362         $newdrive->{file} = $newvolid;
5363         my $drivestr = print_drive($newdrive);
5364         $nbd->{$opt}->{drivestr} = $drivestr;
5365         $nbd->{$opt}->{volid} = $newvolid;
5366     }
5367
5368     return $nbd;
5369 }
5370
5371 # see vm_start_nolock for parameters, additionally:
5372 # migrate_opts:
5373 #   storagemap = parsed storage map for allocating NBD disks
5374 sub vm_start {
5375     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5376
5377     return PVE::QemuConfig->lock_config($vmid, sub {
5378         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5379
5380         die "you can't start a vm if it's a template\n"
5381             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5382
5383         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5384         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5385
5386         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5387
5388         if ($has_backup_lock && $running) {
5389             # a backup is currently running, attempt to start the guest in the
5390             # existing QEMU instance
5391             return vm_resume($vmid);
5392         }
5393
5394         PVE::QemuConfig->check_lock($conf)
5395             if !($params->{skiplock} || $has_suspended_lock);
5396
5397         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5398
5399         die "VM $vmid already running\n" if $running;
5400
5401         if (my $storagemap = $migrate_opts->{storagemap}) {
5402             my $replicated = $migrate_opts->{replicated_volumes};
5403             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5404             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5405
5406             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5407                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5408             }
5409         }
5410
5411         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5412     });
5413 }
5414
5415
5416 # params:
5417 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5418 #   skiplock => 0/1, skip checking for config lock
5419 #   skiptemplate => 0/1, skip checking whether VM is template
5420 #   forcemachine => to force Qemu machine (rollback/migration)
5421 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5422 #   timeout => in seconds
5423 #   paused => start VM in paused state (backup)
5424 #   resume => resume from hibernation
5425 #   pbs-backing => {
5426 #      sata0 => {
5427 #         repository
5428 #         snapshot
5429 #         keyfile
5430 #         archive
5431 #      },
5432 #      virtio2 => ...
5433 #   }
5434 # migrate_opts:
5435 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5436 #   migratedfrom => source node
5437 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5438 #   network => CIDR of migration network
5439 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5440 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5441 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5442 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5443 #       contained in config
5444 sub vm_start_nolock {
5445     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5446
5447     my $statefile = $params->{statefile};
5448     my $resume = $params->{resume};
5449
5450     my $migratedfrom = $migrate_opts->{migratedfrom};
5451     my $migration_type = $migrate_opts->{type};
5452
5453     my $res = {};
5454
5455     # clean up leftover reboot request files
5456     eval { clear_reboot_request($vmid); };
5457     warn $@ if $@;
5458
5459     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5460         vmconfig_apply_pending($vmid, $conf, $storecfg);
5461         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5462     }
5463
5464     # don't regenerate the ISO if the VM is started as part of a live migration
5465     # this way we can reuse the old ISO with the correct config
5466     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5467
5468     # override offline migrated volumes, conf is out of date still
5469     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5470         for my $key (sort keys $offline_volumes->%*) {
5471             my $parsed = parse_drive($key, $conf->{$key});
5472             $parsed->{file} = $offline_volumes->{$key};
5473             $conf->{$key} = print_drive($parsed);
5474         }
5475     }
5476
5477     my $defaults = load_defaults();
5478
5479     # set environment variable useful inside network script
5480     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5481
5482     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5483
5484     my $forcemachine = $params->{forcemachine};
5485     my $forcecpu = $params->{forcecpu};
5486     if ($resume) {
5487         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5488         $forcemachine = $conf->{runningmachine};
5489         $forcecpu = $conf->{runningcpu};
5490         print "Resuming suspended VM\n";
5491     }
5492
5493     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5494         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5495
5496     my $migration_ip;
5497     my $get_migration_ip = sub {
5498         my ($nodename) = @_;
5499
5500         return $migration_ip if defined($migration_ip);
5501
5502         my $cidr = $migrate_opts->{network};
5503
5504         if (!defined($cidr)) {
5505             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5506             $cidr = $dc_conf->{migration}->{network};
5507         }
5508
5509         if (defined($cidr)) {
5510             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5511
5512             die "could not get IP: no address configured on local " .
5513                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5514
5515             die "could not get IP: multiple addresses configured on local " .
5516                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5517
5518             $migration_ip = @$ips[0];
5519         }
5520
5521         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5522             if !defined($migration_ip);
5523
5524         return $migration_ip;
5525     };
5526
5527     my $migrate_uri;
5528     if ($statefile) {
5529         if ($statefile eq 'tcp') {
5530             my $localip = "localhost";
5531             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5532             my $nodename = nodename();
5533
5534             if (!defined($migration_type)) {
5535                 if (defined($datacenterconf->{migration}->{type})) {
5536                     $migration_type = $datacenterconf->{migration}->{type};
5537                 } else {
5538                     $migration_type = 'secure';
5539                 }
5540             }
5541
5542             if ($migration_type eq 'insecure') {
5543                 $localip = $get_migration_ip->($nodename);
5544                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5545             }
5546
5547             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5548             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5549             $migrate_uri = "tcp:${localip}:${migrate_port}";
5550             push @$cmd, '-incoming', $migrate_uri;
5551             push @$cmd, '-S';
5552
5553         } elsif ($statefile eq 'unix') {
5554             # should be default for secure migrations as a ssh TCP forward
5555             # tunnel is not deterministic reliable ready and fails regurarly
5556             # to set up in time, so use UNIX socket forwards
5557             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5558             unlink $socket_addr;
5559
5560             $migrate_uri = "unix:$socket_addr";
5561
5562             push @$cmd, '-incoming', $migrate_uri;
5563             push @$cmd, '-S';
5564
5565         } elsif (-e $statefile) {
5566             push @$cmd, '-loadstate', $statefile;
5567         } else {
5568             my $statepath = PVE::Storage::path($storecfg, $statefile);
5569             push @$vollist, $statefile;
5570             push @$cmd, '-loadstate', $statepath;
5571         }
5572     } elsif ($params->{paused}) {
5573         push @$cmd, '-S';
5574     }
5575
5576     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5577
5578     my $pci_devices = {}; # host pci devices
5579     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5580         my $dev = $conf->{"hostpci$i"} or next;
5581         $pci_devices->{$i} = parse_hostpci($dev);
5582     }
5583
5584     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5585     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5586
5587     # map to a flat list of pci ids
5588     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5589
5590     # reserve all PCI IDs before actually doing anything with them
5591     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5592
5593     eval {
5594         my $uuid;
5595         for my $id (sort keys %$pci_devices) {
5596             my $d = $pci_devices->{$id};
5597             for my $dev ($d->{pciid}->@*) {
5598                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5599
5600                 # nvidia grid needs the uuid of the mdev as qemu parameter
5601                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5602                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5603                 }
5604             }
5605         }
5606         push @$cmd, '-uuid', $uuid if defined($uuid);
5607     };
5608     if (my $err = $@) {
5609         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5610         warn $@ if $@;
5611         die $err;
5612     }
5613
5614     PVE::Storage::activate_volumes($storecfg, $vollist);
5615
5616     eval {
5617         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5618     };
5619     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5620     # timeout should be more than enough here...
5621     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5622
5623     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5624
5625     my %run_params = (
5626         timeout => $statefile ? undef : $start_timeout,
5627         umask => 0077,
5628         noerr => 1,
5629     );
5630
5631     # when migrating, prefix QEMU output so other side can pick up any
5632     # errors that might occur and show the user
5633     if ($migratedfrom) {
5634         $run_params{quiet} = 1;
5635         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5636     }
5637
5638     my %systemd_properties = (
5639         Slice => 'qemu.slice',
5640         KillMode => 'process',
5641         SendSIGKILL => 0,
5642         TimeoutStopUSec => ULONG_MAX, # infinity
5643     );
5644
5645     if (PVE::CGroup::cgroup_mode() == 2) {
5646         $systemd_properties{CPUWeight} = $cpuunits;
5647     } else {
5648         $systemd_properties{CPUShares} = $cpuunits;
5649     }
5650
5651     if (my $cpulimit = $conf->{cpulimit}) {
5652         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5653     }
5654     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5655
5656     my $run_qemu = sub {
5657         PVE::Tools::run_fork sub {
5658             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5659
5660             my $tpmpid;
5661             if (my $tpm = $conf->{tpmstate0}) {
5662                 # start the TPM emulator so QEMU can connect on start
5663                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5664             }
5665
5666             my $exitcode = run_command($cmd, %run_params);
5667             if ($exitcode) {
5668                 if ($tpmpid) {
5669                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5670                     kill 'TERM', $tpmpid;
5671                 }
5672                 die "QEMU exited with code $exitcode\n";
5673             }
5674         };
5675     };
5676
5677     if ($conf->{hugepages}) {
5678
5679         my $code = sub {
5680             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5681             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5682
5683             PVE::QemuServer::Memory::hugepages_mount();
5684             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5685
5686             eval { $run_qemu->() };
5687             if (my $err = $@) {
5688                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5689                     if !$conf->{keephugepages};
5690                 die $err;
5691             }
5692
5693             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5694                 if !$conf->{keephugepages};
5695         };
5696         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5697
5698     } else {
5699         eval { $run_qemu->() };
5700     }
5701
5702     if (my $err = $@) {
5703         # deactivate volumes if start fails
5704         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5705         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5706
5707         die "start failed: $err";
5708     }
5709
5710     # re-reserve all PCI IDs now that we can know the actual VM PID
5711     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5712     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5713     warn $@ if $@;
5714
5715     print "migration listens on $migrate_uri\n" if $migrate_uri;
5716     $res->{migrate_uri} = $migrate_uri;
5717
5718     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5719         eval { mon_cmd($vmid, "cont"); };
5720         warn $@ if $@;
5721     }
5722
5723     #start nbd server for storage migration
5724     if (my $nbd = $migrate_opts->{nbd}) {
5725         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5726
5727         my $migrate_storage_uri;
5728         # nbd_protocol_version > 0 for unix socket support
5729         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5730             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5731             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5732             $migrate_storage_uri = "nbd:unix:$socket_path";
5733         } else {
5734             my $nodename = nodename();
5735             my $localip = $get_migration_ip->($nodename);
5736             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5737             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5738
5739             mon_cmd($vmid, "nbd-server-start", addr => {
5740                 type => 'inet',
5741                 data => {
5742                     host => "${localip}",
5743                     port => "${storage_migrate_port}",
5744                 },
5745             });
5746             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5747             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5748         }
5749
5750         $res->{migrate_storage_uri} = $migrate_storage_uri;
5751
5752         foreach my $opt (sort keys %$nbd) {
5753             my $drivestr = $nbd->{$opt}->{drivestr};
5754             my $volid = $nbd->{$opt}->{volid};
5755             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5756             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5757             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5758             print "re-using replicated volume: $opt - $volid\n"
5759                 if $nbd->{$opt}->{replicated};
5760
5761             $res->{drives}->{$opt} = $nbd->{$opt};
5762             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5763         }
5764     }
5765
5766     if ($migratedfrom) {
5767         eval {
5768             set_migration_caps($vmid);
5769         };
5770         warn $@ if $@;
5771
5772         if ($spice_port) {
5773             print "spice listens on port $spice_port\n";
5774             $res->{spice_port} = $spice_port;
5775             if ($migrate_opts->{spice_ticket}) {
5776                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5777                     $migrate_opts->{spice_ticket});
5778                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5779             }
5780         }
5781
5782     } else {
5783         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5784             if !$statefile && $conf->{balloon};
5785
5786         foreach my $opt (keys %$conf) {
5787             next if $opt !~  m/^net\d+$/;
5788             my $nicconf = parse_net($conf->{$opt});
5789             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5790         }
5791     }
5792
5793     mon_cmd($vmid, 'qom-set',
5794                 path => "machine/peripheral/balloon0",
5795                 property => "guest-stats-polling-interval",
5796                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5797
5798     if ($resume) {
5799         print "Resumed VM, removing state\n";
5800         if (my $vmstate = $conf->{vmstate}) {
5801             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5802             PVE::Storage::vdisk_free($storecfg, $vmstate);
5803         }
5804         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5805         PVE::QemuConfig->write_config($vmid, $conf);
5806     }
5807
5808     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5809
5810     return $res;
5811 }
5812
5813 sub vm_commandline {
5814     my ($storecfg, $vmid, $snapname) = @_;
5815
5816     my $conf = PVE::QemuConfig->load_config($vmid);
5817
5818     my ($forcemachine, $forcecpu);
5819     if ($snapname) {
5820         my $snapshot = $conf->{snapshots}->{$snapname};
5821         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5822
5823         # check for machine or CPU overrides in snapshot
5824         $forcemachine = $snapshot->{runningmachine};
5825         $forcecpu = $snapshot->{runningcpu};
5826
5827         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5828
5829         $conf = $snapshot;
5830     }
5831
5832     my $defaults = load_defaults();
5833
5834     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5835
5836     return PVE::Tools::cmd2string($cmd);
5837 }
5838
5839 sub vm_reset {
5840     my ($vmid, $skiplock) = @_;
5841
5842     PVE::QemuConfig->lock_config($vmid, sub {
5843
5844         my $conf = PVE::QemuConfig->load_config($vmid);
5845
5846         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5847
5848         mon_cmd($vmid, "system_reset");
5849     });
5850 }
5851
5852 sub get_vm_volumes {
5853     my ($conf) = @_;
5854
5855     my $vollist = [];
5856     foreach_volid($conf, sub {
5857         my ($volid, $attr) = @_;
5858
5859         return if $volid =~ m|^/|;
5860
5861         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5862         return if !$sid;
5863
5864         push @$vollist, $volid;
5865     });
5866
5867     return $vollist;
5868 }
5869
5870 sub vm_stop_cleanup {
5871     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5872
5873     eval {
5874
5875         if (!$keepActive) {
5876             my $vollist = get_vm_volumes($conf);
5877             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5878
5879             if (my $tpmdrive = $conf->{tpmstate0}) {
5880                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5881                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5882                 if ($storeid) {
5883                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5884                 }
5885             }
5886         }
5887
5888         foreach my $ext (qw(mon qmp pid vnc qga)) {
5889             unlink "/var/run/qemu-server/${vmid}.$ext";
5890         }
5891
5892         if ($conf->{ivshmem}) {
5893             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5894             # just delete it for now, VMs which have this already open do not
5895             # are affected, but new VMs will get a separated one. If this
5896             # becomes an issue we either add some sort of ref-counting or just
5897             # add a "don't delete on stop" flag to the ivshmem format.
5898             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5899         }
5900
5901         my $ids = [];
5902         foreach my $key (keys %$conf) {
5903             next if $key !~ m/^hostpci(\d+)$/;
5904             my $hostpciindex = $1;
5905             my $d = parse_hostpci($conf->{$key});
5906             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5907
5908             foreach my $pci (@{$d->{pciid}}) {
5909                 my $pciid = $pci->{id};
5910                 push @$ids, $pci->{id};
5911                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5912             }
5913         }
5914         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5915
5916         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5917     };
5918     warn $@ if $@; # avoid errors - just warn
5919 }
5920
5921 # call only in locked context
5922 sub _do_vm_stop {
5923     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5924
5925     my $pid = check_running($vmid, $nocheck);
5926     return if !$pid;
5927
5928     my $conf;
5929     if (!$nocheck) {
5930         $conf = PVE::QemuConfig->load_config($vmid);
5931         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5932         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5933             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5934             $timeout = $opts->{down} if $opts->{down};
5935         }
5936         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5937     }
5938
5939     eval {
5940         if ($shutdown) {
5941             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5942                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5943             } else {
5944                 mon_cmd($vmid, "system_powerdown");
5945             }
5946         } else {
5947             mon_cmd($vmid, "quit");
5948         }
5949     };
5950     my $err = $@;
5951
5952     if (!$err) {
5953         $timeout = 60 if !defined($timeout);
5954
5955         my $count = 0;
5956         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5957             $count++;
5958             sleep 1;
5959         }
5960
5961         if ($count >= $timeout) {
5962             if ($force) {
5963                 warn "VM still running - terminating now with SIGTERM\n";
5964                 kill 15, $pid;
5965             } else {
5966                 die "VM quit/powerdown failed - got timeout\n";
5967             }
5968         } else {
5969             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5970             return;
5971         }
5972     } else {
5973        if (!check_running($vmid, $nocheck)) {
5974             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5975             return;
5976        }
5977        if ($force) {
5978             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5979             kill 15, $pid;
5980         } else {
5981             die "VM quit/powerdown failed\n";
5982         }
5983     }
5984
5985     # wait again
5986     $timeout = 10;
5987
5988     my $count = 0;
5989     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5990         $count++;
5991         sleep 1;
5992     }
5993
5994     if ($count >= $timeout) {
5995         warn "VM still running - terminating now with SIGKILL\n";
5996         kill 9, $pid;
5997         sleep 1;
5998     }
5999
6000     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6001 }
6002
6003 # Note: use $nocheck to skip tests if VM configuration file exists.
6004 # We need that when migration VMs to other nodes (files already moved)
6005 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6006 sub vm_stop {
6007     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6008
6009     $force = 1 if !defined($force) && !$shutdown;
6010
6011     if ($migratedfrom){
6012         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6013         kill 15, $pid if $pid;
6014         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6015         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6016         return;
6017     }
6018
6019     PVE::QemuConfig->lock_config($vmid, sub {
6020         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6021    });
6022 }
6023
6024 sub vm_reboot {
6025     my ($vmid, $timeout) = @_;
6026
6027     PVE::QemuConfig->lock_config($vmid, sub {
6028         eval {
6029
6030             # only reboot if running, as qmeventd starts it again on a stop event
6031             return if !check_running($vmid);
6032
6033             create_reboot_request($vmid);
6034
6035             my $storecfg = PVE::Storage::config();
6036             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6037
6038         };
6039         if (my $err = $@) {
6040             # avoid that the next normal shutdown will be confused for a reboot
6041             clear_reboot_request($vmid);
6042             die $err;
6043         }
6044    });
6045 }
6046
6047 # note: if using the statestorage parameter, the caller has to check privileges
6048 sub vm_suspend {
6049     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6050
6051     my $conf;
6052     my $path;
6053     my $storecfg;
6054     my $vmstate;
6055
6056     PVE::QemuConfig->lock_config($vmid, sub {
6057
6058         $conf = PVE::QemuConfig->load_config($vmid);
6059
6060         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6061         PVE::QemuConfig->check_lock($conf)
6062             if !($skiplock || $is_backing_up);
6063
6064         die "cannot suspend to disk during backup\n"
6065             if $is_backing_up && $includestate;
6066
6067         if ($includestate) {
6068             $conf->{lock} = 'suspending';
6069             my $date = strftime("%Y-%m-%d", localtime(time()));
6070             $storecfg = PVE::Storage::config();
6071             if (!$statestorage) {
6072                 $statestorage = find_vmstate_storage($conf, $storecfg);
6073                 # check permissions for the storage
6074                 my $rpcenv = PVE::RPCEnvironment::get();
6075                 if ($rpcenv->{type} ne 'cli') {
6076                     my $authuser = $rpcenv->get_user();
6077                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6078                 }
6079             }
6080
6081
6082             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6083                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6084             $path = PVE::Storage::path($storecfg, $vmstate);
6085             PVE::QemuConfig->write_config($vmid, $conf);
6086         } else {
6087             mon_cmd($vmid, "stop");
6088         }
6089     });
6090
6091     if ($includestate) {
6092         # save vm state
6093         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6094
6095         eval {
6096             set_migration_caps($vmid, 1);
6097             mon_cmd($vmid, "savevm-start", statefile => $path);
6098             for(;;) {
6099                 my $state = mon_cmd($vmid, "query-savevm");
6100                 if (!$state->{status}) {
6101                     die "savevm not active\n";
6102                 } elsif ($state->{status} eq 'active') {
6103                     sleep(1);
6104                     next;
6105                 } elsif ($state->{status} eq 'completed') {
6106                     print "State saved, quitting\n";
6107                     last;
6108                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6109                     die "query-savevm failed with error '$state->{error}'\n"
6110                 } else {
6111                     die "query-savevm returned status '$state->{status}'\n";
6112                 }
6113             }
6114         };
6115         my $err = $@;
6116
6117         PVE::QemuConfig->lock_config($vmid, sub {
6118             $conf = PVE::QemuConfig->load_config($vmid);
6119             if ($err) {
6120                 # cleanup, but leave suspending lock, to indicate something went wrong
6121                 eval {
6122                     mon_cmd($vmid, "savevm-end");
6123                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6124                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6125                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6126                     PVE::QemuConfig->write_config($vmid, $conf);
6127                 };
6128                 warn $@ if $@;
6129                 die $err;
6130             }
6131
6132             die "lock changed unexpectedly\n"
6133                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6134
6135             mon_cmd($vmid, "quit");
6136             $conf->{lock} = 'suspended';
6137             PVE::QemuConfig->write_config($vmid, $conf);
6138         });
6139     }
6140 }
6141
6142 sub vm_resume {
6143     my ($vmid, $skiplock, $nocheck) = @_;
6144
6145     PVE::QemuConfig->lock_config($vmid, sub {
6146         my $res = mon_cmd($vmid, 'query-status');
6147         my $resume_cmd = 'cont';
6148         my $reset = 0;
6149
6150         if ($res->{status}) {
6151             return if $res->{status} eq 'running'; # job done, go home
6152             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6153             $reset = 1 if $res->{status} eq 'shutdown';
6154         }
6155
6156         if (!$nocheck) {
6157
6158             my $conf = PVE::QemuConfig->load_config($vmid);
6159
6160             PVE::QemuConfig->check_lock($conf)
6161                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6162         }
6163
6164         if ($reset) {
6165             # required if a VM shuts down during a backup and we get a resume
6166             # request before the backup finishes for example
6167             mon_cmd($vmid, "system_reset");
6168         }
6169         mon_cmd($vmid, $resume_cmd);
6170     });
6171 }
6172
6173 sub vm_sendkey {
6174     my ($vmid, $skiplock, $key) = @_;
6175
6176     PVE::QemuConfig->lock_config($vmid, sub {
6177
6178         my $conf = PVE::QemuConfig->load_config($vmid);
6179
6180         # there is no qmp command, so we use the human monitor command
6181         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6182         die $res if $res ne '';
6183     });
6184 }
6185
6186 # vzdump restore implementaion
6187
6188 sub tar_archive_read_firstfile {
6189     my $archive = shift;
6190
6191     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6192
6193     # try to detect archive type first
6194     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6195         die "unable to open file '$archive'\n";
6196     my $firstfile = <$fh>;
6197     kill 15, $pid;
6198     close $fh;
6199
6200     die "ERROR: archive contaions no data\n" if !$firstfile;
6201     chomp $firstfile;
6202
6203     return $firstfile;
6204 }
6205
6206 sub tar_restore_cleanup {
6207     my ($storecfg, $statfile) = @_;
6208
6209     print STDERR "starting cleanup\n";
6210
6211     if (my $fd = IO::File->new($statfile, "r")) {
6212         while (defined(my $line = <$fd>)) {
6213             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6214                 my $volid = $2;
6215                 eval {
6216                     if ($volid =~ m|^/|) {
6217                         unlink $volid || die 'unlink failed\n';
6218                     } else {
6219                         PVE::Storage::vdisk_free($storecfg, $volid);
6220                     }
6221                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6222                 };
6223                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6224             } else {
6225                 print STDERR "unable to parse line in statfile - $line";
6226             }
6227         }
6228         $fd->close();
6229     }
6230 }
6231
6232 sub restore_file_archive {
6233     my ($archive, $vmid, $user, $opts) = @_;
6234
6235     return restore_vma_archive($archive, $vmid, $user, $opts)
6236         if $archive eq '-';
6237
6238     my $info = PVE::Storage::archive_info($archive);
6239     my $format = $opts->{format} // $info->{format};
6240     my $comp = $info->{compression};
6241
6242     # try to detect archive format
6243     if ($format eq 'tar') {
6244         return restore_tar_archive($archive, $vmid, $user, $opts);
6245     } else {
6246         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6247     }
6248 }
6249
6250 # hepler to remove disks that will not be used after restore
6251 my $restore_cleanup_oldconf = sub {
6252     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6253
6254     my $kept_disks = {};
6255
6256     PVE::QemuConfig->foreach_volume($oldconf, sub {
6257         my ($ds, $drive) = @_;
6258
6259         return if drive_is_cdrom($drive, 1);
6260
6261         my $volid = $drive->{file};
6262         return if !$volid || $volid =~ m|^/|;
6263
6264         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6265         return if !$path || !$owner || ($owner != $vmid);
6266
6267         # Note: only delete disk we want to restore
6268         # other volumes will become unused
6269         if ($virtdev_hash->{$ds}) {
6270             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6271             if (my $err = $@) {
6272                 warn $err;
6273             }
6274         } else {
6275             $kept_disks->{$volid} = 1;
6276         }
6277     });
6278
6279     # after the restore we have no snapshots anymore
6280     for my $snapname (keys $oldconf->{snapshots}->%*) {
6281         my $snap = $oldconf->{snapshots}->{$snapname};
6282         if ($snap->{vmstate}) {
6283             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6284             if (my $err = $@) {
6285                 warn $err;
6286             }
6287         }
6288
6289         for my $volid (keys $kept_disks->%*) {
6290             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6291             warn $@ if $@;
6292         }
6293     }
6294 };
6295
6296 # Helper to parse vzdump backup device hints
6297 #
6298 # $rpcenv: Environment, used to ckeck storage permissions
6299 # $user: User ID, to check storage permissions
6300 # $storecfg: Storage configuration
6301 # $fh: the file handle for reading the configuration
6302 # $devinfo: should contain device sizes for all backu-up'ed devices
6303 # $options: backup options (pool, default storage)
6304 #
6305 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6306 my $parse_backup_hints = sub {
6307     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6308
6309     my $check_storage = sub { # assert if an image can be allocate
6310         my ($storeid, $scfg) = @_;
6311         die "Content type 'images' is not available on storage '$storeid'\n"
6312             if !$scfg->{content}->{images};
6313         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6314             if $user ne 'root@pam';
6315     };
6316
6317     my $virtdev_hash = {};
6318     while (defined(my $line = <$fh>)) {
6319         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6320             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6321             die "archive does not contain data for drive '$virtdev'\n"
6322                 if !$devinfo->{$devname};
6323
6324             if (defined($options->{storage})) {
6325                 $storeid = $options->{storage} || 'local';
6326             } elsif (!$storeid) {
6327                 $storeid = 'local';
6328             }
6329             $format = 'raw' if !$format;
6330             $devinfo->{$devname}->{devname} = $devname;
6331             $devinfo->{$devname}->{virtdev} = $virtdev;
6332             $devinfo->{$devname}->{format} = $format;
6333             $devinfo->{$devname}->{storeid} = $storeid;
6334
6335             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6336             $check_storage->($storeid, $scfg); # permission and content type check
6337
6338             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6339         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6340             my $virtdev = $1;
6341             my $drive = parse_drive($virtdev, $2);
6342
6343             if (drive_is_cloudinit($drive)) {
6344                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6345                 $storeid = $options->{storage} if defined ($options->{storage});
6346                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6347                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6348
6349                 $check_storage->($storeid, $scfg); # permission and content type check
6350
6351                 $virtdev_hash->{$virtdev} = {
6352                     format => $format,
6353                     storeid => $storeid,
6354                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6355                     is_cloudinit => 1,
6356                 };
6357             }
6358         }
6359     }
6360
6361     return $virtdev_hash;
6362 };
6363
6364 # Helper to allocate and activate all volumes required for a restore
6365 #
6366 # $storecfg: Storage configuration
6367 # $virtdev_hash: as returned by parse_backup_hints()
6368 #
6369 # Returns: { $virtdev => $volid }
6370 my $restore_allocate_devices = sub {
6371     my ($storecfg, $virtdev_hash, $vmid) = @_;
6372
6373     my $map = {};
6374     foreach my $virtdev (sort keys %$virtdev_hash) {
6375         my $d = $virtdev_hash->{$virtdev};
6376         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6377         my $storeid = $d->{storeid};
6378         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6379
6380         # test if requested format is supported
6381         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6382         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6383         $d->{format} = $defFormat if !$supported;
6384
6385         my $name;
6386         if ($d->{is_cloudinit}) {
6387             $name = "vm-$vmid-cloudinit";
6388             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6389             if ($scfg->{path}) {
6390                 $name .= ".$d->{format}";
6391             }
6392         }
6393
6394         my $volid = PVE::Storage::vdisk_alloc(
6395             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6396
6397         print STDERR "new volume ID is '$volid'\n";
6398         $d->{volid} = $volid;
6399
6400         PVE::Storage::activate_volumes($storecfg, [$volid]);
6401
6402         $map->{$virtdev} = $volid;
6403     }
6404
6405     return $map;
6406 };
6407
6408 sub restore_update_config_line {
6409     my ($cookie, $map, $line, $unique) = @_;
6410
6411     return '' if $line =~ m/^\#qmdump\#/;
6412     return '' if $line =~ m/^\#vzdump\#/;
6413     return '' if $line =~ m/^lock:/;
6414     return '' if $line =~ m/^unused\d+:/;
6415     return '' if $line =~ m/^parent:/;
6416
6417     my $res = '';
6418
6419     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6420     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6421         # try to convert old 1.X settings
6422         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6423         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6424             my ($model, $macaddr) = split(/\=/, $devconfig);
6425             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6426             my $net = {
6427                 model => $model,
6428                 bridge => "vmbr$ind",
6429                 macaddr => $macaddr,
6430             };
6431             my $netstr = print_net($net);
6432
6433             $res .= "net$cookie->{netcount}: $netstr\n";
6434             $cookie->{netcount}++;
6435         }
6436     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6437         my ($id, $netstr) = ($1, $2);
6438         my $net = parse_net($netstr);
6439         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6440         $netstr = print_net($net);
6441         $res .= "$id: $netstr\n";
6442     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6443         my $virtdev = $1;
6444         my $value = $3;
6445         my $di = parse_drive($virtdev, $value);
6446         if (defined($di->{backup}) && !$di->{backup}) {
6447             $res .= "#$line";
6448         } elsif ($map->{$virtdev}) {
6449             delete $di->{format}; # format can change on restore
6450             $di->{file} = $map->{$virtdev};
6451             $value = print_drive($di);
6452             $res .= "$virtdev: $value\n";
6453         } else {
6454             $res .= $line;
6455         }
6456     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6457         my $vmgenid = $1;
6458         if ($vmgenid ne '0') {
6459             # always generate a new vmgenid if there was a valid one setup
6460             $vmgenid = generate_uuid();
6461         }
6462         $res .= "vmgenid: $vmgenid\n";
6463     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6464         my ($uuid, $uuid_str);
6465         UUID::generate($uuid);
6466         UUID::unparse($uuid, $uuid_str);
6467         my $smbios1 = parse_smbios1($2);
6468         $smbios1->{uuid} = $uuid_str;
6469         $res .= $1.print_smbios1($smbios1)."\n";
6470     } else {
6471         $res .= $line;
6472     }
6473
6474     return $res;
6475 }
6476
6477 my $restore_deactivate_volumes = sub {
6478     my ($storecfg, $virtdev_hash) = @_;
6479
6480     my $vollist = [];
6481     for my $dev (values $virtdev_hash->%*) {
6482         push $vollist->@*, $dev->{volid} if $dev->{volid};
6483     }
6484
6485     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6486     print STDERR $@ if $@;
6487 };
6488
6489 my $restore_destroy_volumes = sub {
6490     my ($storecfg, $virtdev_hash) = @_;
6491
6492     for my $dev (values $virtdev_hash->%*) {
6493         my $volid = $dev->{volid} or next;
6494         eval {
6495             PVE::Storage::vdisk_free($storecfg, $volid);
6496             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6497         };
6498         print STDERR "unable to cleanup '$volid' - $@" if $@;
6499     }
6500 };
6501
6502 my $restore_merge_config = sub {
6503     my ($filename, $backup_conf_raw, $override_conf) = @_;
6504
6505     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6506     for my $key (keys $override_conf->%*) {
6507         $backup_conf->{$key} = $override_conf->{$key};
6508     }
6509
6510     return $backup_conf;
6511 };
6512
6513 sub scan_volids {
6514     my ($cfg, $vmid) = @_;
6515
6516     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6517
6518     my $volid_hash = {};
6519     foreach my $storeid (keys %$info) {
6520         foreach my $item (@{$info->{$storeid}}) {
6521             next if !($item->{volid} && $item->{size});
6522             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6523             $volid_hash->{$item->{volid}} = $item;
6524         }
6525     }
6526
6527     return $volid_hash;
6528 }
6529
6530 sub update_disk_config {
6531     my ($vmid, $conf, $volid_hash) = @_;
6532
6533     my $changes;
6534     my $prefix = "VM $vmid";
6535
6536     # used and unused disks
6537     my $referenced = {};
6538
6539     # Note: it is allowed to define multiple storages with same path (alias), so
6540     # we need to check both 'volid' and real 'path' (two different volid can point
6541     # to the same path).
6542
6543     my $referencedpath = {};
6544
6545     # update size info
6546     PVE::QemuConfig->foreach_volume($conf, sub {
6547         my ($opt, $drive) = @_;
6548
6549         my $volid = $drive->{file};
6550         return if !$volid;
6551         my $volume = $volid_hash->{$volid};
6552
6553         # mark volid as "in-use" for next step
6554         $referenced->{$volid} = 1;
6555         if ($volume && (my $path = $volume->{path})) {
6556             $referencedpath->{$path} = 1;
6557         }
6558
6559         return if drive_is_cdrom($drive);
6560         return if !$volume;
6561
6562         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6563         if (defined($updated)) {
6564             $changes = 1;
6565             $conf->{$opt} = print_drive($updated);
6566             print "$prefix ($opt): $msg\n";
6567         }
6568     });
6569
6570     # remove 'unusedX' entry if volume is used
6571     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6572         my ($opt, $drive) = @_;
6573
6574         my $volid = $drive->{file};
6575         return if !$volid;
6576
6577         my $path;
6578         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6579         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6580             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6581             $changes = 1;
6582             delete $conf->{$opt};
6583         }
6584
6585         $referenced->{$volid} = 1;
6586         $referencedpath->{$path} = 1 if $path;
6587     });
6588
6589     foreach my $volid (sort keys %$volid_hash) {
6590         next if $volid =~ m/vm-$vmid-state-/;
6591         next if $referenced->{$volid};
6592         my $path = $volid_hash->{$volid}->{path};
6593         next if !$path; # just to be sure
6594         next if $referencedpath->{$path};
6595         $changes = 1;
6596         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6597         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6598         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6599     }
6600
6601     return $changes;
6602 }
6603
6604 sub rescan {
6605     my ($vmid, $nolock, $dryrun) = @_;
6606
6607     my $cfg = PVE::Storage::config();
6608
6609     print "rescan volumes...\n";
6610     my $volid_hash = scan_volids($cfg, $vmid);
6611
6612     my $updatefn =  sub {
6613         my ($vmid) = @_;
6614
6615         my $conf = PVE::QemuConfig->load_config($vmid);
6616
6617         PVE::QemuConfig->check_lock($conf);
6618
6619         my $vm_volids = {};
6620         foreach my $volid (keys %$volid_hash) {
6621             my $info = $volid_hash->{$volid};
6622             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6623         }
6624
6625         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6626
6627         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6628     };
6629
6630     if (defined($vmid)) {
6631         if ($nolock) {
6632             &$updatefn($vmid);
6633         } else {
6634             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6635         }
6636     } else {
6637         my $vmlist = config_list();
6638         foreach my $vmid (keys %$vmlist) {
6639             if ($nolock) {
6640                 &$updatefn($vmid);
6641             } else {
6642                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6643             }
6644         }
6645     }
6646 }
6647
6648 sub restore_proxmox_backup_archive {
6649     my ($archive, $vmid, $user, $options) = @_;
6650
6651     my $storecfg = PVE::Storage::config();
6652
6653     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6654     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6655
6656     my $fingerprint = $scfg->{fingerprint};
6657     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6658
6659     my $repo = PVE::PBSClient::get_repository($scfg);
6660     my $namespace = $scfg->{namespace};
6661
6662     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6663     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6664     local $ENV{PBS_PASSWORD} = $password;
6665     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6666
6667     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6668         PVE::Storage::parse_volname($storecfg, $archive);
6669
6670     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6671
6672     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6673
6674     my $tmpdir = "/var/tmp/vzdumptmp$$";
6675     rmtree $tmpdir;
6676     mkpath $tmpdir;
6677
6678     my $conffile = PVE::QemuConfig->config_file($vmid);
6679      # disable interrupts (always do cleanups)
6680     local $SIG{INT} =
6681         local $SIG{TERM} =
6682         local $SIG{QUIT} =
6683         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6684
6685     # Note: $oldconf is undef if VM does not exists
6686     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6687     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6688     my $new_conf_raw = '';
6689
6690     my $rpcenv = PVE::RPCEnvironment::get();
6691     my $devinfo = {}; # info about drives included in backup
6692     my $virtdev_hash = {}; # info about allocated drives
6693
6694     eval {
6695         # enable interrupts
6696         local $SIG{INT} =
6697             local $SIG{TERM} =
6698             local $SIG{QUIT} =
6699             local $SIG{HUP} =
6700             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6701
6702         my $cfgfn = "$tmpdir/qemu-server.conf";
6703         my $firewall_config_fn = "$tmpdir/fw.conf";
6704         my $index_fn = "$tmpdir/index.json";
6705
6706         my $cmd = "restore";
6707
6708         my $param = [$pbs_backup_name, "index.json", $index_fn];
6709         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6710         my $index = PVE::Tools::file_get_contents($index_fn);
6711         $index = decode_json($index);
6712
6713         foreach my $info (@{$index->{files}}) {
6714             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6715                 my $devname = $1;
6716                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6717                     $devinfo->{$devname}->{size} = $1;
6718                 } else {
6719                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6720                 }
6721             }
6722         }
6723
6724         my $is_qemu_server_backup = scalar(
6725             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6726         );
6727         if (!$is_qemu_server_backup) {
6728             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6729         }
6730         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6731
6732         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6733         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6734
6735         if ($has_firewall_config) {
6736             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6737             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6738
6739             my $pve_firewall_dir = '/etc/pve/firewall';
6740             mkdir $pve_firewall_dir; # make sure the dir exists
6741             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6742         }
6743
6744         my $fh = IO::File->new($cfgfn, "r") ||
6745             die "unable to read qemu-server.conf - $!\n";
6746
6747         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6748
6749         # fixme: rate limit?
6750
6751         # create empty/temp config
6752         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6753
6754         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6755
6756         # allocate volumes
6757         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6758
6759         foreach my $virtdev (sort keys %$virtdev_hash) {
6760             my $d = $virtdev_hash->{$virtdev};
6761             next if $d->{is_cloudinit}; # no need to restore cloudinit
6762
6763             # this fails if storage is unavailable
6764             my $volid = $d->{volid};
6765             my $path = PVE::Storage::path($storecfg, $volid);
6766
6767             # for live-restore we only want to preload the efidisk and TPM state
6768             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6769
6770             my @ns_arg;
6771             if (defined(my $ns = $scfg->{namespace})) {
6772                 @ns_arg = ('--ns', $ns);
6773             }
6774
6775             my $pbs_restore_cmd = [
6776                 '/usr/bin/pbs-restore',
6777                 '--repository', $repo,
6778                 @ns_arg,
6779                 $pbs_backup_name,
6780                 "$d->{devname}.img.fidx",
6781                 $path,
6782                 '--verbose',
6783                 ];
6784
6785             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6786             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6787
6788             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6789                 push @$pbs_restore_cmd, '--skip-zero';
6790             }
6791
6792             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6793             print "restore proxmox backup image: $dbg_cmdstring\n";
6794             run_command($pbs_restore_cmd);
6795         }
6796
6797         $fh->seek(0, 0) || die "seek failed - $!\n";
6798
6799         my $cookie = { netcount => 0 };
6800         while (defined(my $line = <$fh>)) {
6801             $new_conf_raw .= restore_update_config_line(
6802                 $cookie,
6803                 $map,
6804                 $line,
6805                 $options->{unique},
6806             );
6807         }
6808
6809         $fh->close();
6810     };
6811     my $err = $@;
6812
6813     if ($err || !$options->{live}) {
6814         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6815     }
6816
6817     rmtree $tmpdir;
6818
6819     if ($err) {
6820         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6821         die $err;
6822     }
6823
6824     if ($options->{live}) {
6825         # keep lock during live-restore
6826         $new_conf_raw .= "\nlock: create";
6827     }
6828
6829     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6830     PVE::QemuConfig->write_config($vmid, $new_conf);
6831
6832     eval { rescan($vmid, 1); };
6833     warn $@ if $@;
6834
6835     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6836
6837     if ($options->{live}) {
6838         # enable interrupts
6839         local $SIG{INT} =
6840             local $SIG{TERM} =
6841             local $SIG{QUIT} =
6842             local $SIG{HUP} =
6843             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6844
6845         my $conf = PVE::QemuConfig->load_config($vmid);
6846         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6847
6848         # these special drives are already restored before start
6849         delete $devinfo->{'drive-efidisk0'};
6850         delete $devinfo->{'drive-tpmstate0-backup'};
6851
6852         my $pbs_opts = {
6853             repo => $repo,
6854             keyfile => $keyfile,
6855             snapshot => $pbs_backup_name,
6856             namespace => $namespace,
6857         };
6858         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6859
6860         PVE::QemuConfig->remove_lock($vmid, "create");
6861     }
6862 }
6863
6864 sub pbs_live_restore {
6865     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6866
6867     print "starting VM for live-restore\n";
6868     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6869
6870     my $pbs_backing = {};
6871     for my $ds (keys %$restored_disks) {
6872         $ds =~ m/^drive-(.*)$/;
6873         my $confname = $1;
6874         $pbs_backing->{$confname} = {
6875             repository => $opts->{repo},
6876             snapshot => $opts->{snapshot},
6877             archive => "$ds.img.fidx",
6878         };
6879         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6880         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6881
6882         my $drive = parse_drive($confname, $conf->{$confname});
6883         print "restoring '$ds' to '$drive->{file}'\n";
6884     }
6885
6886     my $drives_streamed = 0;
6887     eval {
6888         # make sure HA doesn't interrupt our restore by stopping the VM
6889         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6890             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6891         }
6892
6893         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6894         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6895         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6896
6897         my $qmeventd_fd = register_qmeventd_handle($vmid);
6898
6899         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6900         # this will effectively collapse the backing image chain consisting of
6901         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6902         # removes itself once all backing images vanish with 'auto-remove=on')
6903         my $jobs = {};
6904         for my $ds (sort keys %$restored_disks) {
6905             my $job_id = "restore-$ds";
6906             mon_cmd($vmid, 'block-stream',
6907                 'job-id' => $job_id,
6908                 device => "$ds",
6909             );
6910             $jobs->{$job_id} = {};
6911         }
6912
6913         mon_cmd($vmid, 'cont');
6914         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6915
6916         print "restore-drive jobs finished successfully, removing all tracking block devices"
6917             ." to disconnect from Proxmox Backup Server\n";
6918
6919         for my $ds (sort keys %$restored_disks) {
6920             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6921         }
6922
6923         close($qmeventd_fd);
6924     };
6925
6926     my $err = $@;
6927
6928     if ($err) {
6929         warn "An error occurred during live-restore: $err\n";
6930         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6931         die "live-restore failed\n";
6932     }
6933 }
6934
6935 sub restore_vma_archive {
6936     my ($archive, $vmid, $user, $opts, $comp) = @_;
6937
6938     my $readfrom = $archive;
6939
6940     my $cfg = PVE::Storage::config();
6941     my $commands = [];
6942     my $bwlimit = $opts->{bwlimit};
6943
6944     my $dbg_cmdstring = '';
6945     my $add_pipe = sub {
6946         my ($cmd) = @_;
6947         push @$commands, $cmd;
6948         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6949         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6950         $readfrom = '-';
6951     };
6952
6953     my $input = undef;
6954     if ($archive eq '-') {
6955         $input = '<&STDIN';
6956     } else {
6957         # If we use a backup from a PVE defined storage we also consider that
6958         # storage's rate limit:
6959         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6960         if (defined($volid)) {
6961             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6962             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6963             if ($readlimit) {
6964                 print STDERR "applying read rate limit: $readlimit\n";
6965                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6966                 $add_pipe->($cstream);
6967             }
6968         }
6969     }
6970
6971     if ($comp) {
6972         my $info = PVE::Storage::decompressor_info('vma', $comp);
6973         my $cmd = $info->{decompressor};
6974         push @$cmd, $readfrom;
6975         $add_pipe->($cmd);
6976     }
6977
6978     my $tmpdir = "/var/tmp/vzdumptmp$$";
6979     rmtree $tmpdir;
6980
6981     # disable interrupts (always do cleanups)
6982     local $SIG{INT} =
6983         local $SIG{TERM} =
6984         local $SIG{QUIT} =
6985         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6986
6987     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6988     POSIX::mkfifo($mapfifo, 0600);
6989     my $fifofh;
6990     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6991
6992     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6993
6994     my $oldtimeout;
6995     my $timeout = 5;
6996
6997     my $devinfo = {}; # info about drives included in backup
6998     my $virtdev_hash = {}; # info about allocated drives
6999
7000     my $rpcenv = PVE::RPCEnvironment::get();
7001
7002     my $conffile = PVE::QemuConfig->config_file($vmid);
7003
7004     # Note: $oldconf is undef if VM does not exist
7005     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7006     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7007     my $new_conf_raw = '';
7008
7009     my %storage_limits;
7010
7011     my $print_devmap = sub {
7012         my $cfgfn = "$tmpdir/qemu-server.conf";
7013
7014         # we can read the config - that is already extracted
7015         my $fh = IO::File->new($cfgfn, "r") ||
7016             die "unable to read qemu-server.conf - $!\n";
7017
7018         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7019         if (-f $fwcfgfn) {
7020             my $pve_firewall_dir = '/etc/pve/firewall';
7021             mkdir $pve_firewall_dir; # make sure the dir exists
7022             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7023         }
7024
7025         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7026
7027         foreach my $info (values %{$virtdev_hash}) {
7028             my $storeid = $info->{storeid};
7029             next if defined($storage_limits{$storeid});
7030
7031             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7032             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7033             $storage_limits{$storeid} = $limit * 1024;
7034         }
7035
7036         foreach my $devname (keys %$devinfo) {
7037             die "found no device mapping information for device '$devname'\n"
7038                 if !$devinfo->{$devname}->{virtdev};
7039         }
7040
7041         # create empty/temp config
7042         if ($oldconf) {
7043             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7044             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7045         }
7046
7047         # allocate volumes
7048         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7049
7050         # print restore information to $fifofh
7051         foreach my $virtdev (sort keys %$virtdev_hash) {
7052             my $d = $virtdev_hash->{$virtdev};
7053             next if $d->{is_cloudinit}; # no need to restore cloudinit
7054
7055             my $storeid = $d->{storeid};
7056             my $volid = $d->{volid};
7057
7058             my $map_opts = '';
7059             if (my $limit = $storage_limits{$storeid}) {
7060                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7061             }
7062
7063             my $write_zeros = 1;
7064             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7065                 $write_zeros = 0;
7066             }
7067
7068             my $path = PVE::Storage::path($cfg, $volid);
7069
7070             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7071
7072             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7073         }
7074
7075         $fh->seek(0, 0) || die "seek failed - $!\n";
7076
7077         my $cookie = { netcount => 0 };
7078         while (defined(my $line = <$fh>)) {
7079             $new_conf_raw .= restore_update_config_line(
7080                 $cookie,
7081                 $map,
7082                 $line,
7083                 $opts->{unique},
7084             );
7085         }
7086
7087         $fh->close();
7088     };
7089
7090     eval {
7091         # enable interrupts
7092         local $SIG{INT} =
7093             local $SIG{TERM} =
7094             local $SIG{QUIT} =
7095             local $SIG{HUP} =
7096             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7097         local $SIG{ALRM} = sub { die "got timeout\n"; };
7098
7099         $oldtimeout = alarm($timeout);
7100
7101         my $parser = sub {
7102             my $line = shift;
7103
7104             print "$line\n";
7105
7106             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7107                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7108                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7109             } elsif ($line =~ m/^CTIME: /) {
7110                 # we correctly received the vma config, so we can disable
7111                 # the timeout now for disk allocation (set to 10 minutes, so
7112                 # that we always timeout if something goes wrong)
7113                 alarm(600);
7114                 &$print_devmap();
7115                 print $fifofh "done\n";
7116                 my $tmp = $oldtimeout || 0;
7117                 $oldtimeout = undef;
7118                 alarm($tmp);
7119                 close($fifofh);
7120                 $fifofh = undef;
7121             }
7122         };
7123
7124         print "restore vma archive: $dbg_cmdstring\n";
7125         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7126     };
7127     my $err = $@;
7128
7129     alarm($oldtimeout) if $oldtimeout;
7130
7131     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7132
7133     close($fifofh) if $fifofh;
7134     unlink $mapfifo;
7135     rmtree $tmpdir;
7136
7137     if ($err) {
7138         $restore_destroy_volumes->($cfg, $virtdev_hash);
7139         die $err;
7140     }
7141
7142     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7143     PVE::QemuConfig->write_config($vmid, $new_conf);
7144
7145     eval { rescan($vmid, 1); };
7146     warn $@ if $@;
7147
7148     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7149 }
7150
7151 sub restore_tar_archive {
7152     my ($archive, $vmid, $user, $opts) = @_;
7153
7154     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7155         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7156         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7157     }
7158
7159     if ($archive ne '-') {
7160         my $firstfile = tar_archive_read_firstfile($archive);
7161         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7162             if $firstfile ne 'qemu-server.conf';
7163     }
7164
7165     my $storecfg = PVE::Storage::config();
7166
7167     # avoid zombie disks when restoring over an existing VM -> cleanup first
7168     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7169     # skiplock=1 because qmrestore has set the 'create' lock itself already
7170     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7171     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7172
7173     my $tocmd = "/usr/lib/qemu-server/qmextract";
7174
7175     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7176     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7177     $tocmd .= ' --prealloc' if $opts->{prealloc};
7178     $tocmd .= ' --info' if $opts->{info};
7179
7180     # tar option "xf" does not autodetect compression when read from STDIN,
7181     # so we pipe to zcat
7182     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7183         PVE::Tools::shellquote("--to-command=$tocmd");
7184
7185     my $tmpdir = "/var/tmp/vzdumptmp$$";
7186     mkpath $tmpdir;
7187
7188     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7189     local $ENV{VZDUMP_VMID} = $vmid;
7190     local $ENV{VZDUMP_USER} = $user;
7191
7192     my $conffile = PVE::QemuConfig->config_file($vmid);
7193     my $new_conf_raw = '';
7194
7195     # disable interrupts (always do cleanups)
7196     local $SIG{INT} =
7197         local $SIG{TERM} =
7198         local $SIG{QUIT} =
7199         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7200
7201     eval {
7202         # enable interrupts
7203         local $SIG{INT} =
7204             local $SIG{TERM} =
7205             local $SIG{QUIT} =
7206             local $SIG{HUP} =
7207             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7208
7209         if ($archive eq '-') {
7210             print "extracting archive from STDIN\n";
7211             run_command($cmd, input => "<&STDIN");
7212         } else {
7213             print "extracting archive '$archive'\n";
7214             run_command($cmd);
7215         }
7216
7217         return if $opts->{info};
7218
7219         # read new mapping
7220         my $map = {};
7221         my $statfile = "$tmpdir/qmrestore.stat";
7222         if (my $fd = IO::File->new($statfile, "r")) {
7223             while (defined (my $line = <$fd>)) {
7224                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7225                     $map->{$1} = $2 if $1;
7226                 } else {
7227                     print STDERR "unable to parse line in statfile - $line\n";
7228                 }
7229             }
7230             $fd->close();
7231         }
7232
7233         my $confsrc = "$tmpdir/qemu-server.conf";
7234
7235         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7236
7237         my $cookie = { netcount => 0 };
7238         while (defined (my $line = <$srcfd>)) {
7239             $new_conf_raw .= restore_update_config_line(
7240                 $cookie,
7241                 $map,
7242                 $line,
7243                 $opts->{unique},
7244             );
7245         }
7246
7247         $srcfd->close();
7248     };
7249     if (my $err = $@) {
7250         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7251         die $err;
7252     }
7253
7254     rmtree $tmpdir;
7255
7256     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7257
7258     PVE::Cluster::cfs_update(); # make sure we read new file
7259
7260     eval { rescan($vmid, 1); };
7261     warn $@ if $@;
7262 };
7263
7264 sub foreach_storage_used_by_vm {
7265     my ($conf, $func) = @_;
7266
7267     my $sidhash = {};
7268
7269     PVE::QemuConfig->foreach_volume($conf, sub {
7270         my ($ds, $drive) = @_;
7271         return if drive_is_cdrom($drive);
7272
7273         my $volid = $drive->{file};
7274
7275         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7276         $sidhash->{$sid} = $sid if $sid;
7277     });
7278
7279     foreach my $sid (sort keys %$sidhash) {
7280         &$func($sid);
7281     }
7282 }
7283
7284 my $qemu_snap_storage = {
7285     rbd => 1,
7286 };
7287 sub do_snapshots_with_qemu {
7288     my ($storecfg, $volid, $deviceid) = @_;
7289
7290     return if $deviceid =~ m/tpmstate0/;
7291
7292     my $storage_name = PVE::Storage::parse_volume_id($volid);
7293     my $scfg = $storecfg->{ids}->{$storage_name};
7294     die "could not find storage '$storage_name'\n" if !defined($scfg);
7295
7296     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7297         return 1;
7298     }
7299
7300     if ($volid =~ m/\.(qcow2|qed)$/){
7301         return 1;
7302     }
7303
7304     return;
7305 }
7306
7307 sub qga_check_running {
7308     my ($vmid, $nowarn) = @_;
7309
7310     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7311     if ($@) {
7312         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7313         return 0;
7314     }
7315     return 1;
7316 }
7317
7318 sub template_create {
7319     my ($vmid, $conf, $disk) = @_;
7320
7321     my $storecfg = PVE::Storage::config();
7322
7323     PVE::QemuConfig->foreach_volume($conf, sub {
7324         my ($ds, $drive) = @_;
7325
7326         return if drive_is_cdrom($drive);
7327         return if $disk && $ds ne $disk;
7328
7329         my $volid = $drive->{file};
7330         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7331
7332         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7333         $drive->{file} = $voliddst;
7334         $conf->{$ds} = print_drive($drive);
7335         PVE::QemuConfig->write_config($vmid, $conf);
7336     });
7337 }
7338
7339 sub convert_iscsi_path {
7340     my ($path) = @_;
7341
7342     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7343         my $portal = $1;
7344         my $target = $2;
7345         my $lun = $3;
7346
7347         my $initiator_name = get_initiator_name();
7348
7349         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7350                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7351     }
7352
7353     die "cannot convert iscsi path '$path', unkown format\n";
7354 }
7355
7356 sub qemu_img_convert {
7357     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7358
7359     my $storecfg = PVE::Storage::config();
7360     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7361     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7362
7363     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7364
7365     my $cachemode;
7366     my $src_path;
7367     my $src_is_iscsi = 0;
7368     my $src_format;
7369
7370     if ($src_storeid) {
7371         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7372         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7373         $src_format = qemu_img_format($src_scfg, $src_volname);
7374         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7375         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7376         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7377     } elsif (-f $src_volid || -b $src_volid) {
7378         $src_path = $src_volid;
7379         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7380             $src_format = $1;
7381         }
7382     }
7383
7384     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7385
7386     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7387     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7388     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7389     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7390
7391     my $cmd = [];
7392     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7393     push @$cmd, '-l', "snapshot.name=$snapname"
7394         if $snapname && $src_format && $src_format eq "qcow2";
7395     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7396     push @$cmd, '-T', $cachemode if defined($cachemode);
7397
7398     if ($src_is_iscsi) {
7399         push @$cmd, '--image-opts';
7400         $src_path = convert_iscsi_path($src_path);
7401     } elsif ($src_format) {
7402         push @$cmd, '-f', $src_format;
7403     }
7404
7405     if ($dst_is_iscsi) {
7406         push @$cmd, '--target-image-opts';
7407         $dst_path = convert_iscsi_path($dst_path);
7408     } else {
7409         push @$cmd, '-O', $dst_format;
7410     }
7411
7412     push @$cmd, $src_path;
7413
7414     if (!$dst_is_iscsi && $is_zero_initialized) {
7415         push @$cmd, "zeroinit:$dst_path";
7416     } else {
7417         push @$cmd, $dst_path;
7418     }
7419
7420     my $parser = sub {
7421         my $line = shift;
7422         if($line =~ m/\((\S+)\/100\%\)/){
7423             my $percent = $1;
7424             my $transferred = int($size * $percent / 100);
7425             my $total_h = render_bytes($size, 1);
7426             my $transferred_h = render_bytes($transferred, 1);
7427
7428             print "transferred $transferred_h of $total_h ($percent%)\n";
7429         }
7430
7431     };
7432
7433     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7434     my $err = $@;
7435     die "copy failed: $err" if $err;
7436 }
7437
7438 sub qemu_img_format {
7439     my ($scfg, $volname) = @_;
7440
7441     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7442         return $1;
7443     } else {
7444         return "raw";
7445     }
7446 }
7447
7448 sub qemu_drive_mirror {
7449     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7450
7451     $jobs = {} if !$jobs;
7452
7453     my $qemu_target;
7454     my $format;
7455     $jobs->{"drive-$drive"} = {};
7456
7457     if ($dst_volid =~ /^nbd:/) {
7458         $qemu_target = $dst_volid;
7459         $format = "nbd";
7460     } else {
7461         my $storecfg = PVE::Storage::config();
7462         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7463
7464         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7465
7466         $format = qemu_img_format($dst_scfg, $dst_volname);
7467
7468         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7469
7470         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7471     }
7472
7473     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7474     $opts->{format} = $format if $format;
7475
7476     if (defined($src_bitmap)) {
7477         $opts->{sync} = 'incremental';
7478         $opts->{bitmap} = $src_bitmap;
7479         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7480     }
7481
7482     if (defined($bwlimit)) {
7483         $opts->{speed} = $bwlimit * 1024;
7484         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7485     } else {
7486         print "drive mirror is starting for drive-$drive\n";
7487     }
7488
7489     # if a job already runs for this device we get an error, catch it for cleanup
7490     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7491     if (my $err = $@) {
7492         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7493         warn "$@\n" if $@;
7494         die "mirroring error: $err\n";
7495     }
7496
7497     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7498 }
7499
7500 # $completion can be either
7501 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7502 # 'cancel': wait until all jobs are ready, block-job-cancel them
7503 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7504 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7505 sub qemu_drive_mirror_monitor {
7506     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7507
7508     $completion //= 'complete';
7509     $op //= "mirror";
7510
7511     eval {
7512         my $err_complete = 0;
7513
7514         my $starttime = time ();
7515         while (1) {
7516             die "block job ('$op') timed out\n" if $err_complete > 300;
7517
7518             my $stats = mon_cmd($vmid, "query-block-jobs");
7519             my $ctime = time();
7520
7521             my $running_jobs = {};
7522             for my $stat (@$stats) {
7523                 next if $stat->{type} ne $op;
7524                 $running_jobs->{$stat->{device}} = $stat;
7525             }
7526
7527             my $readycounter = 0;
7528
7529             for my $job_id (sort keys %$jobs) {
7530                 my $job = $running_jobs->{$job_id};
7531
7532                 my $vanished = !defined($job);
7533                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7534                 if($complete || ($vanished && $completion eq 'auto')) {
7535                     print "$job_id: $op-job finished\n";
7536                     delete $jobs->{$job_id};
7537                     next;
7538                 }
7539
7540                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7541
7542                 my $busy = $job->{busy};
7543                 my $ready = $job->{ready};
7544                 if (my $total = $job->{len}) {
7545                     my $transferred = $job->{offset} || 0;
7546                     my $remaining = $total - $transferred;
7547                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7548
7549                     my $duration = $ctime - $starttime;
7550                     my $total_h = render_bytes($total, 1);
7551                     my $transferred_h = render_bytes($transferred, 1);
7552
7553                     my $status = sprintf(
7554                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7555                         render_duration($duration),
7556                     );
7557
7558                     if ($ready) {
7559                         if ($busy) {
7560                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7561                         } else {
7562                             $status .= ", ready";
7563                         }
7564                     }
7565                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7566                     $jobs->{$job_id}->{ready} = $ready;
7567                 }
7568
7569                 $readycounter++ if $job->{ready};
7570             }
7571
7572             last if scalar(keys %$jobs) == 0;
7573
7574             if ($readycounter == scalar(keys %$jobs)) {
7575                 print "all '$op' jobs are ready\n";
7576
7577                 # do the complete later (or has already been done)
7578                 last if $completion eq 'skip' || $completion eq 'auto';
7579
7580                 if ($vmiddst && $vmiddst != $vmid) {
7581                     my $agent_running = $qga && qga_check_running($vmid);
7582                     if ($agent_running) {
7583                         print "freeze filesystem\n";
7584                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7585                         warn $@ if $@;
7586                     } else {
7587                         print "suspend vm\n";
7588                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7589                         warn $@ if $@;
7590                     }
7591
7592                     # if we clone a disk for a new target vm, we don't switch the disk
7593                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7594
7595                     if ($agent_running) {
7596                         print "unfreeze filesystem\n";
7597                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7598                         warn $@ if $@;
7599                     } else {
7600                         print "resume vm\n";
7601                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7602                         warn $@ if $@;
7603                     }
7604
7605                     last;
7606                 } else {
7607
7608                     for my $job_id (sort keys %$jobs) {
7609                         # try to switch the disk if source and destination are on the same guest
7610                         print "$job_id: Completing block job_id...\n";
7611
7612                         my $op;
7613                         if ($completion eq 'complete') {
7614                             $op = 'block-job-complete';
7615                         } elsif ($completion eq 'cancel') {
7616                             $op = 'block-job-cancel';
7617                         } else {
7618                             die "invalid completion value: $completion\n";
7619                         }
7620                         eval { mon_cmd($vmid, $op, device => $job_id) };
7621                         if ($@ =~ m/cannot be completed/) {
7622                             print "$job_id: block job cannot be completed, trying again.\n";
7623                             $err_complete++;
7624                         }else {
7625                             print "$job_id: Completed successfully.\n";
7626                             $jobs->{$job_id}->{complete} = 1;
7627                         }
7628                     }
7629                 }
7630             }
7631             sleep 1;
7632         }
7633     };
7634     my $err = $@;
7635
7636     if ($err) {
7637         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7638         die "block job ($op) error: $err";
7639     }
7640 }
7641
7642 sub qemu_blockjobs_cancel {
7643     my ($vmid, $jobs) = @_;
7644
7645     foreach my $job (keys %$jobs) {
7646         print "$job: Cancelling block job\n";
7647         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7648         $jobs->{$job}->{cancel} = 1;
7649     }
7650
7651     while (1) {
7652         my $stats = mon_cmd($vmid, "query-block-jobs");
7653
7654         my $running_jobs = {};
7655         foreach my $stat (@$stats) {
7656             $running_jobs->{$stat->{device}} = $stat;
7657         }
7658
7659         foreach my $job (keys %$jobs) {
7660
7661             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7662                 print "$job: Done.\n";
7663                 delete $jobs->{$job};
7664             }
7665         }
7666
7667         last if scalar(keys %$jobs) == 0;
7668
7669         sleep 1;
7670     }
7671 }
7672
7673 sub clone_disk {
7674     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7675
7676     my ($vmid, $running) = $source->@{qw(vmid running)};
7677     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7678
7679     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7680     my ($storage, $format) = $dest->@{qw(storage format)};
7681
7682     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7683
7684     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7685         die "cloning from/to EFI disk requires EFI disk\n"
7686             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7687         die "cloning from/to TPM state requires TPM state\n"
7688             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7689
7690         # This would lead to two device nodes in QEMU pointing to the same backing image!
7691         die "cannot change drive name when cloning disk from/to the same VM\n"
7692             if $use_drive_mirror && $vmid == $newvmid;
7693     }
7694
7695     die "cannot move TPM state while VM is running\n"
7696         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7697
7698     my $newvolid;
7699
7700     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7701     print "$src_drivename " if $src_drivename;
7702     print "($drive->{file})\n";
7703
7704     if (!$full) {
7705         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7706         push @$newvollist, $newvolid;
7707     } else {
7708
7709         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7710         $storeid = $storage if $storage;
7711
7712         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7713
7714         my $name = undef;
7715         my $size = undef;
7716         if (drive_is_cloudinit($drive)) {
7717             $name = "vm-$newvmid-cloudinit";
7718             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7719             if ($scfg->{path}) {
7720                 $name .= ".$dst_format";
7721             }
7722             $snapname = undef;
7723             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7724         } elsif ($dst_drivename eq 'efidisk0') {
7725             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7726         } elsif ($dst_drivename eq 'tpmstate0') {
7727             $dst_format = 'raw';
7728             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7729         } else {
7730             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7731         }
7732         $newvolid = PVE::Storage::vdisk_alloc(
7733             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7734         );
7735         push @$newvollist, $newvolid;
7736
7737         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7738
7739         if (drive_is_cloudinit($drive)) {
7740             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7741             # if this is the case, we have to complete any block-jobs still there from
7742             # previous drive-mirrors
7743             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7744                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7745             }
7746             goto no_data_clone;
7747         }
7748
7749         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7750         if ($use_drive_mirror) {
7751             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7752                 $completion, $qga, $bwlimit);
7753         } else {
7754             # TODO: handle bwlimits
7755             if ($dst_drivename eq 'efidisk0') {
7756                 # the relevant data on the efidisk may be smaller than the source
7757                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7758                 # that is given by the OVMF_VARS.fd
7759                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7760                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7761
7762                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7763
7764                 # better for Ceph if block size is not too small, see bug #3324
7765                 my $bs = 1024*1024;
7766
7767                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7768
7769                 if ($src_format eq 'qcow2' && $snapname) {
7770                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7771                         if !min_version(kvm_user_version(), 6, 2);
7772                     push $cmd->@*, '-l', $snapname;
7773                 }
7774                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7775                 run_command($cmd);
7776             } else {
7777                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7778             }
7779         }
7780     }
7781
7782 no_data_clone:
7783     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7784
7785     my $disk = dclone($drive);
7786     delete $disk->{format};
7787     $disk->{file} = $newvolid;
7788     $disk->{size} = $size if defined($size);
7789
7790     return $disk;
7791 }
7792
7793 sub get_running_qemu_version {
7794     my ($vmid) = @_;
7795     my $res = mon_cmd($vmid, "query-version");
7796     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7797 }
7798
7799 sub qemu_use_old_bios_files {
7800     my ($machine_type) = @_;
7801
7802     return if !$machine_type;
7803
7804     my $use_old_bios_files = undef;
7805
7806     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7807         $machine_type = $1;
7808         $use_old_bios_files = 1;
7809     } else {
7810         my $version = extract_version($machine_type, kvm_user_version());
7811         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7812         # load new efi bios files on migration. So this hack is required to allow
7813         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7814         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7815         $use_old_bios_files = !min_version($version, 2, 4);
7816     }
7817
7818     return ($use_old_bios_files, $machine_type);
7819 }
7820
7821 sub get_efivars_size {
7822     my ($conf, $efidisk) = @_;
7823
7824     my $arch = get_vm_arch($conf);
7825     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7826     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7827     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7828     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7829     return -s $ovmf_vars;
7830 }
7831
7832 sub update_efidisk_size {
7833     my ($conf) = @_;
7834
7835     return if !defined($conf->{efidisk0});
7836
7837     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7838     $disk->{size} = get_efivars_size($conf);
7839     $conf->{efidisk0} = print_drive($disk);
7840
7841     return;
7842 }
7843
7844 sub update_tpmstate_size {
7845     my ($conf) = @_;
7846
7847     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7848     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7849     $conf->{tpmstate0} = print_drive($disk);
7850 }
7851
7852 sub create_efidisk($$$$$$$) {
7853     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7854
7855     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7856     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7857
7858     my $vars_size_b = -s $ovmf_vars;
7859     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7860     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7861     PVE::Storage::activate_volumes($storecfg, [$volid]);
7862
7863     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7864     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7865
7866     return ($volid, $size/1024);
7867 }
7868
7869 sub vm_iothreads_list {
7870     my ($vmid) = @_;
7871
7872     my $res = mon_cmd($vmid, 'query-iothreads');
7873
7874     my $iothreads = {};
7875     foreach my $iothread (@$res) {
7876         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7877     }
7878
7879     return $iothreads;
7880 }
7881
7882 sub scsihw_infos {
7883     my ($conf, $drive) = @_;
7884
7885     my $maxdev = 0;
7886
7887     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7888         $maxdev = 7;
7889     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7890         $maxdev = 1;
7891     } else {
7892         $maxdev = 256;
7893     }
7894
7895     my $controller = int($drive->{index} / $maxdev);
7896     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7897         ? "virtioscsi"
7898         : "scsihw";
7899
7900     return ($maxdev, $controller, $controller_prefix);
7901 }
7902
7903 sub windows_version {
7904     my ($ostype) = @_;
7905
7906     return 0 if !$ostype;
7907
7908     my $winversion = 0;
7909
7910     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7911         $winversion = 5;
7912     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7913         $winversion = 6;
7914     } elsif ($ostype =~ m/^win(\d+)$/) {
7915         $winversion = $1;
7916     }
7917
7918     return $winversion;
7919 }
7920
7921 sub resolve_dst_disk_format {
7922         my ($storecfg, $storeid, $src_volname, $format) = @_;
7923         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7924
7925         if (!$format) {
7926             # if no target format is specified, use the source disk format as hint
7927             if ($src_volname) {
7928                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7929                 $format = qemu_img_format($scfg, $src_volname);
7930             } else {
7931                 return $defFormat;
7932             }
7933         }
7934
7935         # test if requested format is supported - else use default
7936         my $supported = grep { $_ eq $format } @$validFormats;
7937         $format = $defFormat if !$supported;
7938         return $format;
7939 }
7940
7941 # NOTE: if this logic changes, please update docs & possibly gui logic
7942 sub find_vmstate_storage {
7943     my ($conf, $storecfg) = @_;
7944
7945     # first, return storage from conf if set
7946     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7947
7948     my ($target, $shared, $local);
7949
7950     foreach_storage_used_by_vm($conf, sub {
7951         my ($sid) = @_;
7952         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7953         my $dst = $scfg->{shared} ? \$shared : \$local;
7954         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7955     });
7956
7957     # second, use shared storage where VM has at least one disk
7958     # third, use local storage where VM has at least one disk
7959     # fall back to local storage
7960     $target = $shared // $local // 'local';
7961
7962     return $target;
7963 }
7964
7965 sub generate_uuid {
7966     my ($uuid, $uuid_str);
7967     UUID::generate($uuid);
7968     UUID::unparse($uuid, $uuid_str);
7969     return $uuid_str;
7970 }
7971
7972 sub generate_smbios1_uuid {
7973     return "uuid=".generate_uuid();
7974 }
7975
7976 sub nbd_stop {
7977     my ($vmid) = @_;
7978
7979     mon_cmd($vmid, 'nbd-server-stop');
7980 }
7981
7982 sub create_reboot_request {
7983     my ($vmid) = @_;
7984     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7985         or die "failed to create reboot trigger file: $!\n";
7986     close($fh);
7987 }
7988
7989 sub clear_reboot_request {
7990     my ($vmid) = @_;
7991     my $path = "/run/qemu-server/$vmid.reboot";
7992     my $res = 0;
7993
7994     $res = unlink($path);
7995     die "could not remove reboot request for $vmid: $!"
7996         if !$res && $! != POSIX::ENOENT;
7997
7998     return $res;
7999 }
8000
8001 sub bootorder_from_legacy {
8002     my ($conf, $bootcfg) = @_;
8003
8004     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8005     my $bootindex_hash = {};
8006     my $i = 1;
8007     foreach my $o (split(//, $boot)) {
8008         $bootindex_hash->{$o} = $i*100;
8009         $i++;
8010     }
8011
8012     my $bootorder = {};
8013
8014     PVE::QemuConfig->foreach_volume($conf, sub {
8015         my ($ds, $drive) = @_;
8016
8017         if (drive_is_cdrom ($drive, 1)) {
8018             if ($bootindex_hash->{d}) {
8019                 $bootorder->{$ds} = $bootindex_hash->{d};
8020                 $bootindex_hash->{d} += 1;
8021             }
8022         } elsif ($bootindex_hash->{c}) {
8023             $bootorder->{$ds} = $bootindex_hash->{c}
8024                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8025             $bootindex_hash->{c} += 1;
8026         }
8027     });
8028
8029     if ($bootindex_hash->{n}) {
8030         for (my $i = 0; $i < $MAX_NETS; $i++) {
8031             my $netname = "net$i";
8032             next if !$conf->{$netname};
8033             $bootorder->{$netname} = $bootindex_hash->{n};
8034             $bootindex_hash->{n} += 1;
8035         }
8036     }
8037
8038     return $bootorder;
8039 }
8040
8041 # Generate default device list for 'boot: order=' property. Matches legacy
8042 # default boot order, but with explicit device names. This is important, since
8043 # the fallback for when neither 'order' nor the old format is specified relies
8044 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8045 sub get_default_bootdevices {
8046     my ($conf) = @_;
8047
8048     my @ret = ();
8049
8050     # harddisk
8051     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8052     push @ret, $first if $first;
8053
8054     # cdrom
8055     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8056     push @ret, $first if $first;
8057
8058     # network
8059     for (my $i = 0; $i < $MAX_NETS; $i++) {
8060         my $netname = "net$i";
8061         next if !$conf->{$netname};
8062         push @ret, $netname;
8063         last;
8064     }
8065
8066     return \@ret;
8067 }
8068
8069 sub device_bootorder {
8070     my ($conf) = @_;
8071
8072     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8073
8074     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8075
8076     my $bootorder = {};
8077     if (!defined($boot) || $boot->{legacy}) {
8078         $bootorder = bootorder_from_legacy($conf, $boot);
8079     } elsif ($boot->{order}) {
8080         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8081         for my $dev (PVE::Tools::split_list($boot->{order})) {
8082             $bootorder->{$dev} = $i++;
8083         }
8084     }
8085
8086     return $bootorder;
8087 }
8088
8089 sub register_qmeventd_handle {
8090     my ($vmid) = @_;
8091
8092     my $fh;
8093     my $peer = "/var/run/qmeventd.sock";
8094     my $count = 0;
8095
8096     for (;;) {
8097         $count++;
8098         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8099         last if $fh;
8100         if ($! != EINTR && $! != EAGAIN) {
8101             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8102         }
8103         if ($count > 4) {
8104             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8105               . "after $count retries\n";
8106         }
8107         usleep(25000);
8108     }
8109
8110     # send handshake to mark VM as backing up
8111     print $fh to_json({vzdump => {vmid => "$vmid"}});
8112
8113     # return handle to be closed later when inhibit is no longer required
8114     return $fh;
8115 }
8116
8117 # bash completion helper
8118
8119 sub complete_backup_archives {
8120     my ($cmdname, $pname, $cvalue) = @_;
8121
8122     my $cfg = PVE::Storage::config();
8123
8124     my $storeid;
8125
8126     if ($cvalue =~ m/^([^:]+):/) {
8127         $storeid = $1;
8128     }
8129
8130     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8131
8132     my $res = [];
8133     foreach my $id (keys %$data) {
8134         foreach my $item (@{$data->{$id}}) {
8135             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8136             push @$res, $item->{volid} if defined($item->{volid});
8137         }
8138     }
8139
8140     return $res;
8141 }
8142
8143 my $complete_vmid_full = sub {
8144     my ($running) = @_;
8145
8146     my $idlist = vmstatus();
8147
8148     my $res = [];
8149
8150     foreach my $id (keys %$idlist) {
8151         my $d = $idlist->{$id};
8152         if (defined($running)) {
8153             next if $d->{template};
8154             next if $running && $d->{status} ne 'running';
8155             next if !$running && $d->{status} eq 'running';
8156         }
8157         push @$res, $id;
8158
8159     }
8160     return $res;
8161 };
8162
8163 sub complete_vmid {
8164     return &$complete_vmid_full();
8165 }
8166
8167 sub complete_vmid_stopped {
8168     return &$complete_vmid_full(0);
8169 }
8170
8171 sub complete_vmid_running {
8172     return &$complete_vmid_full(1);
8173 }
8174
8175 sub complete_storage {
8176
8177     my $cfg = PVE::Storage::config();
8178     my $ids = $cfg->{ids};
8179
8180     my $res = [];
8181     foreach my $sid (keys %$ids) {
8182         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8183         next if !$ids->{$sid}->{content}->{images};
8184         push @$res, $sid;
8185     }
8186
8187     return $res;
8188 }
8189
8190 sub complete_migration_storage {
8191     my ($cmd, $param, $current_value, $all_args) = @_;
8192
8193     my $targetnode = @$all_args[1];
8194
8195     my $cfg = PVE::Storage::config();
8196     my $ids = $cfg->{ids};
8197
8198     my $res = [];
8199     foreach my $sid (keys %$ids) {
8200         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8201         next if !$ids->{$sid}->{content}->{images};
8202         push @$res, $sid;
8203     }
8204
8205     return $res;
8206 }
8207
8208 sub vm_is_paused {
8209     my ($vmid) = @_;
8210     my $qmpstatus = eval {
8211         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8212         mon_cmd($vmid, "query-status");
8213     };
8214     warn "$@\n" if $@;
8215     return $qmpstatus && $qmpstatus->{status} eq "paused";
8216 }
8217
8218 sub check_volume_storage_type {
8219     my ($storecfg, $vol) = @_;
8220
8221     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8222     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8223     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8224
8225     die "storage '$storeid' does not support content-type '$vtype'\n"
8226         if !$scfg->{content}->{$vtype};
8227
8228     return 1;
8229 }
8230
8231 1;