PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RESTEnvironment qw(log_warn);
  40 use PVE::RPCEnvironment;
  41 use PVE::Storage;
  42 use PVE::SysFSTools;
  43 use PVE::Systemd;
  44 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  45
  46 use PVE::QMPClient;
  47 use PVE::QemuConfig;
  48 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  49 use PVE::QemuServer::Cloudinit;
  50 use PVE::QemuServer::CGroup;
  51 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  52 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  53 use PVE::QemuServer::Machine;
  54 use PVE::QemuServer::Memory;
  55 use PVE::QemuServer::Monitor qw(mon_cmd);
  56 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  57 use PVE::QemuServer::USB qw(parse_usb_device);
  58
  59 my $have_sdn;
  60 eval {
  61     require PVE::Network::SDN::Zones;
  62     $have_sdn = 1;
  63 };
  64
  65 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  66 my $OVMF = {
  67     x86_64 => {
  68         '4m-no-smm' => [
  69             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  70             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  71         ],
  72         '4m-no-smm-ms' => [
  73             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  74             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  75         ],
  76         '4m' => [
  77             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  78             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  79         ],
  80         '4m-ms' => [
  81             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  82             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  83         ],
  84         default => [
  85             "$EDK2_FW_BASE/OVMF_CODE.fd",
  86             "$EDK2_FW_BASE/OVMF_VARS.fd",
  87         ],
  88     },
  89     aarch64 => {
  90         default => [
  91             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  92             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  93         ],
  94     },
  95 };
  96
  97 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  98
  99 # Note about locking: we use flock on the config file protect against concurent actions.
 100 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 101 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 102 # But you can ignore this kind of lock with the --skiplock flag.
 103
 104 cfs_register_file('/qemu-server/',
 105                   \&parse_vm_config,
 106                   \&write_vm_config);
 107
 108 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 109     description => "Some command save/restore state from this location.",
 110     type => 'string',
 111     maxLength => 128,
 112     optional => 1,
 113 });
 114
 115 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 116         description => "Specifies the Qemu machine type.",
 117         type => 'string',
 118         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 119         maxLength => 40,
 120         optional => 1,
 121 });
 122
 123 #no warnings 'redefine';
 124
 125 my $nodename_cache;
 126 sub nodename {
 127     $nodename_cache //= PVE::INotify::nodename();
 128     return $nodename_cache;
 129 }
 130
 131 my $watchdog_fmt = {
 132     model => {
 133         default_key => 1,
 134         type => 'string',
 135         enum => [qw(i6300esb ib700)],
 136         description => "Watchdog type to emulate.",
 137         default => 'i6300esb',
 138         optional => 1,
 139     },
 140     action => {
 141         type => 'string',
 142         enum => [qw(reset shutdown poweroff pause debug none)],
 143         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 144         optional => 1,
 145     },
 146 };
 147 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 148
 149 my $agent_fmt = {
 150     enabled => {
 151         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 152         type => 'boolean',
 153         default => 0,
 154         default_key => 1,
 155     },
 156     fstrim_cloned_disks => {
 157         description => "Run fstrim after moving a disk or migrating the VM.",
 158         type => 'boolean',
 159         optional => 1,
 160         default => 0
 161     },
 162     type => {
 163         description => "Select the agent type",
 164         type => 'string',
 165         default => 'virtio',
 166         optional => 1,
 167         enum => [qw(virtio isa)],
 168     },
 169 };
 170
 171 my $vga_fmt = {
 172     type => {
 173         description => "Select the VGA type.",
 174         type => 'string',
 175         default => 'std',
 176         optional => 1,
 177         default_key => 1,
 178         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 179     },
 180     memory => {
 181         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 182         type => 'integer',
 183         optional => 1,
 184         minimum => 4,
 185         maximum => 512,
 186     },
 187 };
 188
 189 my $ivshmem_fmt = {
 190     size => {
 191         type => 'integer',
 192         minimum => 1,
 193         description => "The size of the file in MB.",
 194     },
 195     name => {
 196         type => 'string',
 197         pattern => '[a-zA-Z0-9\-]+',
 198         optional => 1,
 199         format_description => 'string',
 200         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 201     },
 202 };
 203
 204 my $audio_fmt = {
 205     device => {
 206         type => 'string',
 207         enum => [qw(ich9-intel-hda intel-hda AC97)],
 208         description =>  "Configure an audio device."
 209     },
 210     driver =>  {
 211         type => 'string',
 212         enum => ['spice', 'none'],
 213         default => 'spice',
 214         optional => 1,
 215         description => "Driver backend for the audio device."
 216     },
 217 };
 218
 219 my $spice_enhancements_fmt = {
 220     foldersharing => {
 221         type => 'boolean',
 222         optional => 1,
 223         default => '0',
 224         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 225     },
 226     videostreaming =>  {
 227         type => 'string',
 228         enum => ['off', 'all', 'filter'],
 229         default => 'off',
 230         optional => 1,
 231         description => "Enable video streaming. Uses compression for detected video streams."
 232     },
 233 };
 234
 235 my $rng_fmt = {
 236     source => {
 237         type => 'string',
 238         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 239         default_key => 1,
 240         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 241             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 242             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 243             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 244             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 245             ." a hardware RNG from the host.",
 246     },
 247     max_bytes => {
 248         type => 'integer',
 249         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 250             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 251             ." `0` to disable limiting (potentially dangerous!).",
 252         optional => 1,
 253
 254         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 255         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 256         # reading from /dev/urandom
 257         default => 1024,
 258     },
 259     period => {
 260         type => 'integer',
 261         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 262             ." the guest to retrieve another 'max_bytes' of entropy.",
 263         optional => 1,
 264         default => 1000,
 265     },
 266 };
 267
 268 my $meta_info_fmt = {
 269     'ctime' => {
 270         type => 'integer',
 271         description => "The guest creation timestamp as UNIX epoch time",
 272         minimum => 0,
 273         optional => 1,
 274     },
 275     'creation-qemu' => {
 276         type => 'string',
 277         description => "The QEMU (machine) version from the time this VM was created.",
 278         pattern => '\d+(\.\d+)+',
 279         optional => 1,
 280     },
 281 };
 282
 283 my $confdesc = {
 284     onboot => {
 285         optional => 1,
 286         type => 'boolean',
 287         description => "Specifies whether a VM will be started during system bootup.",
 288         default => 0,
 289     },
 290     autostart => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Automatic restart after crash (currently ignored).",
 294         default => 0,
 295     },
 296     hotplug => {
 297         optional => 1,
 298         type => 'string', format => 'pve-hotplug-features',
 299         description => "Selectively enable hotplug features. This is a comma separated list of"
 300             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 301             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 302         default => 'network,disk,usb',
 303     },
 304     reboot => {
 305         optional => 1,
 306         type => 'boolean',
 307         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 308         default => 1,
 309     },
 310     lock => {
 311         optional => 1,
 312         type => 'string',
 313         description => "Lock/unlock the VM.",
 314         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 315     },
 316     cpulimit => {
 317         optional => 1,
 318         type => 'number',
 319         description => "Limit of CPU usage.",
 320         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 321             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 322         minimum => 0,
 323         maximum => 128,
 324         default => 0,
 325     },
 326     cpuunits => {
 327         optional => 1,
 328         type => 'integer',
 329         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 330         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 331             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 332             ." weights of all the other running VMs.",
 333         minimum => 1,
 334         maximum => 262144,
 335         default => 'cgroup v1: 1024, cgroup v2: 100',
 336     },
 337     memory => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 341             ." you use the balloon device.",
 342         minimum => 16,
 343         default => 512,
 344     },
 345     balloon => {
 346         optional => 1,
 347         type => 'integer',
 348         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 349         minimum => 0,
 350     },
 351     shares => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 355             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 356             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 357         minimum => 0,
 358         maximum => 50000,
 359         default => 1000,
 360     },
 361     keyboard => {
 362         optional => 1,
 363         type => 'string',
 364         description => "Keyboard layout for VNC server. This option is generally not required and"
 365         ." is often better handled from within the guest OS.",
 366         enum => PVE::Tools::kvmkeymaplist(),
 367         default => undef,
 368     },
 369     name => {
 370         optional => 1,
 371         type => 'string', format => 'dns-name',
 372         description => "Set a name for the VM. Only used on the configuration web interface.",
 373     },
 374     scsihw => {
 375         optional => 1,
 376         type => 'string',
 377         description => "SCSI controller model",
 378         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 379         default => 'lsi',
 380     },
 381     description => {
 382         optional => 1,
 383         type => 'string',
 384         description => "Description for the VM. Shown in the web-interface VM's summary."
 385             ." This is saved as comment inside the configuration file.",
 386         maxLength => 1024 * 8,
 387     },
 388     ostype => {
 389         optional => 1,
 390         type => 'string',
 391         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 392         description => "Specify guest operating system.",
 393         verbose_description => <<EODESC,
 394 Specify guest operating system. This is used to enable special
 395 optimization/features for specific operating systems:
 396
 397 [horizontal]
 398 other;; unspecified OS
 399 wxp;; Microsoft Windows XP
 400 w2k;; Microsoft Windows 2000
 401 w2k3;; Microsoft Windows 2003
 402 w2k8;; Microsoft Windows 2008
 403 wvista;; Microsoft Windows Vista
 404 win7;; Microsoft Windows 7
 405 win8;; Microsoft Windows 8/2012/2012r2
 406 win10;; Microsoft Windows 10/2016/2019
 407 win11;; Microsoft Windows 11/2022
 408 l24;; Linux 2.4 Kernel
 409 l26;; Linux 2.6 - 5.X Kernel
 410 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 411 EODESC
 412     },
 413     boot => {
 414         optional => 1,
 415         type => 'string', format => 'pve-qm-boot',
 416         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 417             ." key or 'legacy=' is deprecated.",
 418     },
 419     bootdisk => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-bootdisk',
 422         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 423         pattern => '(ide|sata|scsi|virtio)\d+',
 424     },
 425     smp => {
 426         optional => 1,
 427         type => 'integer',
 428         description => "The number of CPUs. Please use option -sockets instead.",
 429         minimum => 1,
 430         default => 1,
 431     },
 432     sockets => {
 433         optional => 1,
 434         type => 'integer',
 435         description => "The number of CPU sockets.",
 436         minimum => 1,
 437         default => 1,
 438     },
 439     cores => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "The number of cores per socket.",
 443         minimum => 1,
 444         default => 1,
 445     },
 446     numa => {
 447         optional => 1,
 448         type => 'boolean',
 449         description => "Enable/disable NUMA.",
 450         default => 0,
 451     },
 452     hugepages => {
 453         optional => 1,
 454         type => 'string',
 455         description => "Enable/disable hugepages memory.",
 456         enum => [qw(any 2 1024)],
 457     },
 458     keephugepages => {
 459         optional => 1,
 460         type => 'boolean',
 461         default => 0,
 462         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 463             ." after VM shutdown and can be used for subsequent starts.",
 464     },
 465     vcpus => {
 466         optional => 1,
 467         type => 'integer',
 468         description => "Number of hotplugged vcpus.",
 469         minimum => 1,
 470         default => 0,
 471     },
 472     acpi => {
 473         optional => 1,
 474         type => 'boolean',
 475         description => "Enable/disable ACPI.",
 476         default => 1,
 477     },
 478     agent => {
 479         optional => 1,
 480         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 481         type => 'string',
 482         format => $agent_fmt,
 483     },
 484     kvm => {
 485         optional => 1,
 486         type => 'boolean',
 487         description => "Enable/disable KVM hardware virtualization.",
 488         default => 1,
 489     },
 490     tdf => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable time drift fix.",
 494         default => 0,
 495     },
 496     localtime => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 500             ." the `ostype` indicates a Microsoft Windows OS.",
 501     },
 502     freeze => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 506     },
 507     vga => {
 508         optional => 1,
 509         type => 'string', format => $vga_fmt,
 510         description => "Configure the VGA hardware.",
 511         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 512             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 513             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 514             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 515             ." display server. For win* OS you can select how many independent displays you want,"
 516             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 517             ." using a serial device as terminal.",
 518     },
 519     watchdog => {
 520         optional => 1,
 521         type => 'string', format => 'pve-qm-watchdog',
 522         description => "Create a virtual hardware watchdog device.",
 523         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 524             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 525             ." else the watchdog will reset the guest (or execute the respective action specified)",
 526     },
 527     startdate => {
 528         optional => 1,
 529         type => 'string',
 530         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 531         description => "Set the initial date of the real time clock. Valid format for date are:"
 532             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 533         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 534         default => 'now',
 535     },
 536     startup =>  get_standard_option('pve-startup-order'),
 537     template => {
 538         optional => 1,
 539         type => 'boolean',
 540         description => "Enable/disable Template.",
 541         default => 0,
 542     },
 543     args => {
 544         optional => 1,
 545         type => 'string',
 546         description => "Arbitrary arguments passed to kvm.",
 547         verbose_description => <<EODESCR,
 548 Arbitrary arguments passed to kvm, for example:
 549
 550 args: -no-reboot -no-hpet
 551
 552 NOTE: this option is for experts only.
 553 EODESCR
 554     },
 555     tablet => {
 556         optional => 1,
 557         type => 'boolean',
 558         default => 1,
 559         description => "Enable/disable the USB tablet device.",
 560         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 561             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 562             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 563             ." may consider disabling this to save some context switches. This is turned off by"
 564             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 565     },
 566     migrate_speed => {
 567         optional => 1,
 568         type => 'integer',
 569         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 570         minimum => 0,
 571         default => 0,
 572     },
 573     migrate_downtime => {
 574         optional => 1,
 575         type => 'number',
 576         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 577         minimum => 0,
 578         default => 0.1,
 579     },
 580     cdrom => {
 581         optional => 1,
 582         type => 'string', format => 'pve-qm-ide',
 583         typetext => '<volume>',
 584         description => "This is an alias for option -ide2",
 585     },
 586     cpu => {
 587         optional => 1,
 588         description => "Emulated CPU type.",
 589         type => 'string',
 590         format => 'pve-vm-cpu-conf',
 591     },
 592     parent => get_standard_option('pve-snapshot-name', {
 593         optional => 1,
 594         description => "Parent snapshot name. This is used internally, and should not be modified.",
 595     }),
 596     snaptime => {
 597         optional => 1,
 598         description => "Timestamp for snapshots.",
 599         type => 'integer',
 600         minimum => 0,
 601     },
 602     vmstate => {
 603         optional => 1,
 604         type => 'string', format => 'pve-volume-id',
 605         description => "Reference to a volume which stores the VM state. This is used internally"
 606             ." for snapshots.",
 607     },
 608     vmstatestorage => get_standard_option('pve-storage-id', {
 609         description => "Default storage for VM state volumes/files.",
 610         optional => 1,
 611     }),
 612     runningmachine => get_standard_option('pve-qemu-machine', {
 613         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 614             ." for snapshots.",
 615     }),
 616     runningcpu => {
 617         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 618             ." internally for snapshots.",
 619         optional => 1,
 620         type => 'string',
 621         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 622         format_description => 'QEMU -cpu parameter'
 623     },
 624     machine => get_standard_option('pve-qemu-machine'),
 625     arch => {
 626         description => "Virtual processor architecture. Defaults to the host.",
 627         optional => 1,
 628         type => 'string',
 629         enum => [qw(x86_64 aarch64)],
 630     },
 631     smbios1 => {
 632         description => "Specify SMBIOS type 1 fields.",
 633         type => 'string', format => 'pve-qm-smbios1',
 634         maxLength => 512,
 635         optional => 1,
 636     },
 637     protection => {
 638         optional => 1,
 639         type => 'boolean',
 640         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 641             ." remove disk operations.",
 642         default => 0,
 643     },
 644     bios => {
 645         optional => 1,
 646         type => 'string',
 647         enum => [ qw(seabios ovmf) ],
 648         description => "Select BIOS implementation.",
 649         default => 'seabios',
 650     },
 651     vmgenid => {
 652         type => 'string',
 653         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 654         format_description => 'UUID',
 655         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 656             ." to disable explicitly.",
 657         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 658             ." value identifier to the guest OS. This allows to notify the guest operating system"
 659             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 660             ." execution or creation from a template). The guest operating system notices the"
 661             ." change, and is then able to react as appropriate by marking its copies of"
 662             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 663             ."Note that auto-creation only works when done through API/CLI create or update methods"
 664             .", but not when manually editing the config file.",
 665         default => "1 (autogenerated)",
 666         optional => 1,
 667     },
 668     hookscript => {
 669         type => 'string',
 670         format => 'pve-volume-id',
 671         optional => 1,
 672         description => "Script that will be executed during various steps in the vms lifetime.",
 673     },
 674     ivshmem => {
 675         type => 'string',
 676         format => $ivshmem_fmt,
 677         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 678             ." the host.",
 679         optional => 1,
 680     },
 681     audio0 => {
 682         type => 'string',
 683         format => $audio_fmt,
 684         description => "Configure a audio device, useful in combination with QXL/Spice.",
 685         optional => 1
 686     },
 687     spice_enhancements => {
 688         type => 'string',
 689         format => $spice_enhancements_fmt,
 690         description => "Configure additional enhancements for SPICE.",
 691         optional => 1
 692     },
 693     tags => {
 694         type => 'string', format => 'pve-tag-list',
 695         description => 'Tags of the VM. This is only meta information.',
 696         optional => 1,
 697     },
 698     rng0 => {
 699         type => 'string',
 700         format => $rng_fmt,
 701         description => "Configure a VirtIO-based Random Number Generator.",
 702         optional => 1,
 703     },
 704     meta => {
 705         type => 'string',
 706         format => $meta_info_fmt,
 707         description => "Some (read-only) meta-information about this guest.",
 708         optional => 1,
 709     },
 710 };
 711
 712 my $cicustom_fmt = {
 713     meta => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all meta data passed to the VM via"
 717             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     network => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all network data passed to the VM via'
 725             .' cloud-init.',
 726         format => 'pve-volume-id',
 727         format_description => 'volume',
 728     },
 729     user => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all user data passed to the VM via'
 733             .' cloud-init.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     vendor => {
 738     type => 'string',
 739     optional => 1,
 740     description => 'Specify a custom file containing all vendor data passed to the VM via'
 741      .' cloud-init.',
 742     format => 'pve-volume-id',
 743     format_description => 'volume',
 744     },
 745 };
 746 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 747
 748 my $confdesc_cloudinit = {
 749     citype => {
 750         optional => 1,
 751         type => 'string',
 752         description => 'Specifies the cloud-init configuration format. The default depends on the'
 753             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 754             .' and `configdrive2` for windows.',
 755         enum => ['configdrive2', 'nocloud', 'opennebula'],
 756     },
 757     ciuser => {
 758         optional => 1,
 759         type => 'string',
 760         description => "cloud-init: User name to change ssh keys and password for instead of the"
 761             ." image's configured default user.",
 762     },
 763     cipassword => {
 764         optional => 1,
 765         type => 'string',
 766         description => 'cloud-init: Password to assign the user. Using this is generally not'
 767             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 768             .' support hashed passwords.',
 769     },
 770     cicustom => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'cloud-init: Specify custom files to replace the automatically generated'
 774             .' ones at start.',
 775         format => 'pve-qm-cicustom',
 776     },
 777     searchdomain => {
 778         optional => 1,
 779         type => 'string',
 780         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 781             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 782             .' are set.',
 783     },
 784     nameserver => {
 785         optional => 1,
 786         type => 'string', format => 'address-list',
 787         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 788             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 789             .' are set.',
 790     },
 791     sshkeys => {
 792         optional => 1,
 793         type => 'string',
 794         format => 'urlencoded',
 795         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 796     },
 797 };
 798
 799 # what about other qemu settings ?
 800 #cpu => 'string',
 801 #machine => 'string',
 802 #fda => 'file',
 803 #fdb => 'file',
 804 #mtdblock => 'file',
 805 #sd => 'file',
 806 #pflash => 'file',
 807 #snapshot => 'bool',
 808 #bootp => 'file',
 809 ##tftp => 'dir',
 810 ##smb => 'dir',
 811 #kernel => 'file',
 812 #append => 'string',
 813 #initrd => 'file',
 814 ##soundhw => 'string',
 815
 816 while (my ($k, $v) = each %$confdesc) {
 817     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 818 }
 819
 820 my $MAX_USB_DEVICES = 5;
 821 my $MAX_NETS = 32;
 822 my $MAX_SERIAL_PORTS = 4;
 823 my $MAX_PARALLEL_PORTS = 3;
 824 my $MAX_NUMA = 8;
 825
 826 my $numa_fmt = {
 827     cpus => {
 828         type => "string",
 829         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 830         description => "CPUs accessing this NUMA node.",
 831         format_description => "id[-id];...",
 832     },
 833     memory => {
 834         type => "number",
 835         description => "Amount of memory this NUMA node provides.",
 836         optional => 1,
 837     },
 838     hostnodes => {
 839         type => "string",
 840         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 841         description => "Host NUMA nodes to use.",
 842         format_description => "id[-id];...",
 843         optional => 1,
 844     },
 845     policy => {
 846         type => 'string',
 847         enum => [qw(preferred bind interleave)],
 848         description => "NUMA allocation policy.",
 849         optional => 1,
 850     },
 851 };
 852 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 853 my $numadesc = {
 854     optional => 1,
 855     type => 'string', format => $numa_fmt,
 856     description => "NUMA topology.",
 857 };
 858 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 859
 860 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 861     $confdesc->{"numa$i"} = $numadesc;
 862 }
 863
 864 my $nic_model_list = [
 865     'e1000',
 866     'e1000-82540em',
 867     'e1000-82544gc',
 868     'e1000-82545em',
 869     'e1000e',
 870     'i82551',
 871     'i82557b',
 872     'i82559er',
 873     'ne2k_isa',
 874     'ne2k_pci',
 875     'pcnet',
 876     'rtl8139',
 877     'virtio',
 878     'vmxnet3',
 879 ];
 880 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 881
 882 my $net_fmt_bridge_descr = <<__EOD__;
 883 Bridge to attach the network device to. The Proxmox VE standard bridge
 884 is called 'vmbr0'.
 885
 886 If you do not specify a bridge, we create a kvm user (NATed) network
 887 device, which provides DHCP and DNS services. The following addresses
 888 are used:
 889
 890  10.0.2.2   Gateway
 891  10.0.2.3   DNS Server
 892  10.0.2.4   SMB Server
 893
 894 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 895 __EOD__
 896
 897 my $net_fmt = {
 898     macaddr  => get_standard_option('mac-addr', {
 899         description => "MAC address. That address must be unique withing your network. This is"
 900             ." automatically generated if not specified.",
 901     }),
 902     model => {
 903         type => 'string',
 904         description => "Network Card Model. The 'virtio' model provides the best performance with"
 905             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 906             ." best to use 'e1000'.",
 907         enum => $nic_model_list,
 908         default_key => 1,
 909     },
 910     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 911     bridge => get_standard_option('pve-bridge-id', {
 912         description => $net_fmt_bridge_descr,
 913         optional => 1,
 914     }),
 915     queues => {
 916         type => 'integer',
 917         minimum => 0, maximum => 16,
 918         description => 'Number of packet queues to be used on the device.',
 919         optional => 1,
 920     },
 921     rate => {
 922         type => 'number',
 923         minimum => 0,
 924         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 925         optional => 1,
 926     },
 927     tag => {
 928         type => 'integer',
 929         minimum => 1, maximum => 4094,
 930         description => 'VLAN tag to apply to packets on this interface.',
 931         optional => 1,
 932     },
 933     trunks => {
 934         type => 'string',
 935         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 936         description => 'VLAN trunks to pass through this interface.',
 937         format_description => 'vlanid[;vlanid...]',
 938         optional => 1,
 939     },
 940     firewall => {
 941         type => 'boolean',
 942         description => 'Whether this interface should be protected by the firewall.',
 943         optional => 1,
 944     },
 945     link_down => {
 946         type => 'boolean',
 947         description => 'Whether this interface should be disconnected (like pulling the plug).',
 948         optional => 1,
 949     },
 950     mtu => {
 951         type => 'integer',
 952         minimum => 1, maximum => 65520,
 953         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 954         optional => 1,
 955     },
 956 };
 957
 958 my $netdesc = {
 959     optional => 1,
 960     type => 'string', format => $net_fmt,
 961     description => "Specify network devices.",
 962 };
 963
 964 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 965
 966 my $ipconfig_fmt = {
 967     ip => {
 968         type => 'string',
 969         format => 'pve-ipv4-config',
 970         format_description => 'IPv4Format/CIDR',
 971         description => 'IPv4 address in CIDR format.',
 972         optional => 1,
 973         default => 'dhcp',
 974     },
 975     gw => {
 976         type => 'string',
 977         format => 'ipv4',
 978         format_description => 'GatewayIPv4',
 979         description => 'Default gateway for IPv4 traffic.',
 980         optional => 1,
 981         requires => 'ip',
 982     },
 983     ip6 => {
 984         type => 'string',
 985         format => 'pve-ipv6-config',
 986         format_description => 'IPv6Format/CIDR',
 987         description => 'IPv6 address in CIDR format.',
 988         optional => 1,
 989         default => 'dhcp',
 990     },
 991     gw6 => {
 992         type => 'string',
 993         format => 'ipv6',
 994         format_description => 'GatewayIPv6',
 995         description => 'Default gateway for IPv6 traffic.',
 996         optional => 1,
 997         requires => 'ip6',
 998     },
 999 };
1000 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1001 my $ipconfigdesc = {
1002     optional => 1,
1003     type => 'string', format => 'pve-qm-ipconfig',
1004     description => <<'EODESCR',
1005 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1006
1007 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1008
1009 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1010 gateway should be provided.
1011 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1012 cloud-init 19.4 or newer.
1013
1014 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1015 dhcp on IPv4.
1016 EODESCR
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1019
1020 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1021     $confdesc->{"net$i"} = $netdesc;
1022     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1023 }
1024
1025 foreach my $key (keys %$confdesc_cloudinit) {
1026     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1027 }
1028
1029 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1030 sub verify_volume_id_or_qm_path {
1031     my ($volid, $noerr) = @_;
1032
1033     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1034
1035     return verify_volume_id_or_absolute_path($volid, $noerr);
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1039 sub verify_volume_id_or_absolute_path {
1040     my ($volid, $noerr) = @_;
1041
1042     return $volid if $volid =~ m|^/|;
1043
1044     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1045     if ($@) {
1046         return if $noerr;
1047         die $@;
1048     }
1049     return $volid;
1050 }
1051
1052 my $usb_fmt = {
1053     host => {
1054         default_key => 1,
1055         type => 'string', format => 'pve-qm-usb-device',
1056         format_description => 'HOSTUSBDEVICE|spice',
1057         description => <<EODESCR,
1058 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1059
1060  'bus-port(.port)*' (decimal numbers) or
1061  'vendor_id:product_id' (hexadeciaml numbers) or
1062  'spice'
1063
1064 You can use the 'lsusb -t' command to list existing usb devices.
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 The value 'spice' can be used to add a usb redirection devices for spice.
1070 EODESCR
1071     },
1072     usb3 => {
1073         optional => 1,
1074         type => 'boolean',
1075         description => "Specifies whether if given host option is a USB3 device or port.",
1076         default => 0,
1077     },
1078 };
1079
1080 my $usbdesc = {
1081     optional => 1,
1082     type => 'string', format => $usb_fmt,
1083     description => "Configure an USB device (n is 0 to 4).",
1084 };
1085 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return  $cdrom_path if $cdrom_path;
1265
1266     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1267     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1268     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1269 }
1270
1271 sub get_iso_path {
1272     my ($storecfg, $vmid, $cdrom) = @_;
1273
1274     if ($cdrom eq 'cdrom') {
1275         return get_cdrom_path();
1276     } elsif ($cdrom eq 'none') {
1277         return '';
1278     } elsif ($cdrom =~ m|^/|) {
1279         return $cdrom;
1280     } else {
1281         return PVE::Storage::path($storecfg, $cdrom);
1282     }
1283 }
1284
1285 # try to convert old style file names to volume IDs
1286 sub filename_to_volume_id {
1287     my ($vmid, $file, $media) = @_;
1288
1289      if (!($file eq 'none' || $file eq 'cdrom' ||
1290           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1291
1292         return if $file =~ m|/|;
1293
1294         if ($media && $media eq 'cdrom') {
1295             $file = "local:iso/$file";
1296         } else {
1297             $file = "local:$vmid/$file";
1298         }
1299     }
1300
1301     return $file;
1302 }
1303
1304 sub verify_media_type {
1305     my ($opt, $vtype, $media) = @_;
1306
1307     return if !$media;
1308
1309     my $etype;
1310     if ($media eq 'disk') {
1311         $etype = 'images';
1312     } elsif ($media eq 'cdrom') {
1313         $etype = 'iso';
1314     } else {
1315         die "internal error";
1316     }
1317
1318     return if ($vtype eq $etype);
1319
1320     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1321 }
1322
1323 sub cleanup_drive_path {
1324     my ($opt, $storecfg, $drive) = @_;
1325
1326     # try to convert filesystem paths to volume IDs
1327
1328     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1329         ($drive->{file} !~ m|^/dev/.+|) &&
1330         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1331         ($drive->{file} !~ m/^\d+$/)) {
1332         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1333         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1334             if !$vtype;
1335         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1336         verify_media_type($opt, $vtype, $drive->{media});
1337         $drive->{file} = $volid;
1338     }
1339
1340     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1341 }
1342
1343 sub parse_hotplug_features {
1344     my ($data) = @_;
1345
1346     my $res = {};
1347
1348     return $res if $data eq '0';
1349
1350     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1351
1352     foreach my $feature (PVE::Tools::split_list($data)) {
1353         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1354             $res->{$1} = 1;
1355         } else {
1356             die "invalid hotplug feature '$feature'\n";
1357         }
1358     }
1359     return $res;
1360 }
1361
1362 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1363 sub pve_verify_hotplug_features {
1364     my ($value, $noerr) = @_;
1365
1366     return $value if parse_hotplug_features($value);
1367
1368     return if $noerr;
1369
1370     die "unable to parse hotplug option\n";
1371 }
1372
1373 sub scsi_inquiry {
1374     my($fh, $noerr) = @_;
1375
1376     my $SG_IO = 0x2285;
1377     my $SG_GET_VERSION_NUM = 0x2282;
1378
1379     my $versionbuf = "\x00" x 8;
1380     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1381     if (!$ret) {
1382         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1383         return;
1384     }
1385     my $version = unpack("I", $versionbuf);
1386     if ($version < 30000) {
1387         die "scsi generic interface too old\n"  if !$noerr;
1388         return;
1389     }
1390
1391     my $buf = "\x00" x 36;
1392     my $sensebuf = "\x00" x 8;
1393     my $cmd = pack("C x3 C x1", 0x12, 36);
1394
1395     # see /usr/include/scsi/sg.h
1396     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1397
1398     my $packet = pack(
1399         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1400     );
1401
1402     $ret = ioctl($fh, $SG_IO, $packet);
1403     if (!$ret) {
1404         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1405         return;
1406     }
1407
1408     my @res = unpack($sg_io_hdr_t, $packet);
1409     if ($res[17] || $res[18]) {
1410         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my $res = {};
1415     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1416
1417     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1418     $res->{type} &= 0x1F;
1419
1420     return $res;
1421 }
1422
1423 sub path_is_scsi {
1424     my ($path) = @_;
1425
1426     my $fh = IO::File->new("+<$path") || return;
1427     my $res = scsi_inquiry($fh, 1);
1428     close($fh);
1429
1430     return $res;
1431 }
1432
1433 sub print_tabletdevice_full {
1434     my ($conf, $arch) = @_;
1435
1436     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1437
1438     # we use uhci for old VMs because tablet driver was buggy in older qemu
1439     my $usbbus;
1440     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1441         $usbbus = 'ehci';
1442     } else {
1443         $usbbus = 'uhci';
1444     }
1445
1446     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1447 }
1448
1449 sub print_keyboarddevice_full {
1450     my ($conf, $arch) = @_;
1451
1452     return if $arch ne 'aarch64';
1453
1454     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1455 }
1456
1457 my sub get_drive_id {
1458     my ($drive) = @_;
1459     return "$drive->{interface}$drive->{index}";
1460 }
1461
1462 sub print_drivedevice_full {
1463     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1464
1465     my $device = '';
1466     my $maxdev = 0;
1467
1468     my $drive_id = get_drive_id($drive);
1469     if ($drive->{interface} eq 'virtio') {
1470         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1471         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1472         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1473     } elsif ($drive->{interface} eq 'scsi') {
1474
1475         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1476         my $unit = $drive->{index} % $maxdev;
1477         my $devicetype = 'hd';
1478         my $path = '';
1479         if (drive_is_cdrom($drive)) {
1480             $devicetype = 'cd';
1481         } else {
1482             if ($drive->{file} =~ m|^/|) {
1483                 $path = $drive->{file};
1484                 if (my $info = path_is_scsi($path)) {
1485                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1486                         $devicetype = 'block';
1487                     } elsif ($info->{type} == 1) { # tape
1488                         $devicetype = 'generic';
1489                     }
1490                 }
1491             } else {
1492                  $path = PVE::Storage::path($storecfg, $drive->{file});
1493             }
1494
1495             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1496             my $version = extract_version($machine_type, kvm_user_version());
1497             if ($path =~ m/^iscsi\:\/\// &&
1498                !min_version($version, 4, 1)) {
1499                 $devicetype = 'generic';
1500             }
1501         }
1502
1503         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1504             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1505         } else {
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1507                 .",lun=$drive->{index}";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1512             $device .= ",rotation_rate=1";
1513         }
1514         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1515
1516     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1517         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype";
1523         if ($drive->{interface} eq 'ide') {
1524             $device .= ",bus=ide.$controller,unit=$unit";
1525         } else {
1526             $device .= ",bus=ahci$controller.$unit";
1527         }
1528         $device .= ",drive=drive-$drive_id,id=$drive_id";
1529
1530         if ($devicetype eq 'hd') {
1531             if (my $model = $drive->{model}) {
1532                 $model = URI::Escape::uri_unescape($model);
1533                 $device .= ",model=$model";
1534             }
1535             if ($drive->{ssd}) {
1536                 $device .= ",rotation_rate=1";
1537             }
1538         }
1539         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1540     } elsif ($drive->{interface} eq 'usb') {
1541         die "implement me";
1542         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1543     } else {
1544         die "unsupported interface type";
1545     }
1546
1547     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1548
1549     if (my $serial = $drive->{serial}) {
1550         $serial = URI::Escape::uri_unescape($serial);
1551         $device .= ",serial=$serial";
1552     }
1553
1554
1555     return $device;
1556 }
1557
1558 sub get_initiator_name {
1559     my $initiator;
1560
1561     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1562     while (defined(my $line = <$fh>)) {
1563         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1564         $initiator = $1;
1565         last;
1566     }
1567     $fh->close();
1568
1569     return $initiator;
1570 }
1571
1572 sub print_drive_commandline_full {
1573     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1574
1575     my $path;
1576     my $volid = $drive->{file};
1577     my $format = $drive->{format};
1578     my $drive_id = get_drive_id($drive);
1579
1580     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1581     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1582
1583     if (drive_is_cdrom($drive)) {
1584         $path = get_iso_path($storecfg, $vmid, $volid);
1585         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1586     } else {
1587         if ($storeid) {
1588             $path = PVE::Storage::path($storecfg, $volid);
1589             $format //= qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format //= "raw";
1593         }
1594    }
1595
1596    my $is_rbd = $path =~ m/^rbd:/;
1597
1598     my $opts = '';
1599     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1600     foreach my $o (@qemu_drive_options) {
1601         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1602     }
1603
1604     # snapshot only accepts on|off
1605     if (defined($drive->{snapshot})) {
1606         my $v = $drive->{snapshot} ? 'on' : 'off';
1607         $opts .= ",snapshot=$v";
1608     }
1609
1610     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1611         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1612     }
1613
1614     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1615         my ($dir, $qmpname) = @$type;
1616         if (my $v = $drive->{"mbps$dir"}) {
1617             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1618         }
1619         if (my $v = $drive->{"mbps${dir}_max"}) {
1620             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1621         }
1622         if (my $v = $drive->{"bps${dir}_max_length"}) {
1623             $opts .= ",throttling.bps$qmpname-max-length=$v";
1624         }
1625         if (my $v = $drive->{"iops${dir}"}) {
1626             $opts .= ",throttling.iops$qmpname=$v";
1627         }
1628         if (my $v = $drive->{"iops${dir}_max"}) {
1629             $opts .= ",throttling.iops$qmpname-max=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}_max_length"}) {
1632             $opts .= ",throttling.iops$qmpname-max-length=$v";
1633         }
1634     }
1635
1636     if ($pbs_name) {
1637         $format = "rbd" if $is_rbd;
1638         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1639             if !$format;
1640         $opts .= ",format=alloc-track,file.driver=$format";
1641     } elsif ($format) {
1642         $opts .= ",format=$format";
1643     }
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # io_uring with cache mode writeback or writethrough on krbd will hang...
1655     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1656
1657     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1658     # sometimes, just plain disable...
1659     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1660
1661     # io_uring causes problems when used with CIFS since kernel 5.15
1662     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1663     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1664
1665     if (!$drive->{aio}) {
1666         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1667             # io_uring supports all cache modes
1668             $opts .= ",aio=io_uring";
1669         } else {
1670             # aio native works only with O_DIRECT
1671             if($cache_direct) {
1672                 $opts .= ",aio=native";
1673             } else {
1674                 $opts .= ",aio=threads";
1675             }
1676         }
1677     }
1678
1679     if (!drive_is_cdrom($drive)) {
1680         my $detectzeroes;
1681         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1682             $detectzeroes = 'off';
1683         } elsif ($drive->{discard}) {
1684             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1685         } else {
1686             # This used to be our default with discard not being specified:
1687             $detectzeroes = 'on';
1688         }
1689
1690         # note: 'detect-zeroes' works per blockdev and we want it to persist
1691         # after the alloc-track is removed, so put it on 'file' directly
1692         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1693         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1694     }
1695
1696     if ($pbs_name) {
1697         $opts .= ",backing=$pbs_name";
1698         $opts .= ",auto-remove=on";
1699     }
1700
1701     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1702     my $file_param = "file";
1703     if ($pbs_name) {
1704         # non-rbd drivers require the underlying file to be a seperate block
1705         # node, so add a second .file indirection
1706         $file_param .= ".file" if !$is_rbd;
1707         $file_param .= ".filename";
1708     }
1709     my $pathinfo = $path ? "$file_param=$path," : '';
1710
1711     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1712 }
1713
1714 sub print_pbs_blockdev {
1715     my ($pbs_conf, $pbs_name) = @_;
1716     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1717     $blockdev .= ",repository=$pbs_conf->{repository}";
1718     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1719     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1720     $blockdev .= ",archive=$pbs_conf->{archive}";
1721     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1722     return $blockdev;
1723 }
1724
1725 sub print_netdevice_full {
1726     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1727
1728     my $device = $net->{model};
1729     if ($net->{model} eq 'virtio') {
1730          $device = 'virtio-net-pci';
1731      };
1732
1733     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1734     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1735     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1736         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1737         # and out of each queue plus one config interrupt and control vector queue
1738         my $vectors = $net->{queues} * 2 + 2;
1739         $tmpstr .= ",vectors=$vectors,mq=on";
1740     }
1741     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1742
1743     if (my $mtu = $net->{mtu}) {
1744         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1745             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1746             if ($mtu == 1) {
1747                  $mtu = $bridge_mtu;
1748             } elsif ($mtu < 576) {
1749                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1750             } elsif ($mtu > $bridge_mtu) {
1751                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1752             }
1753             $tmpstr .= ",host_mtu=$mtu";
1754         } else {
1755             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1756         }
1757     }
1758
1759     if ($use_old_bios_files) {
1760         my $romfile;
1761         if ($device eq 'virtio-net-pci') {
1762             $romfile = 'pxe-virtio.rom';
1763         } elsif ($device eq 'e1000') {
1764             $romfile = 'pxe-e1000.rom';
1765         } elsif ($device eq 'e1000e') {
1766             $romfile = 'pxe-e1000e.rom';
1767         } elsif ($device eq 'ne2k') {
1768             $romfile = 'pxe-ne2k_pci.rom';
1769         } elsif ($device eq 'pcnet') {
1770             $romfile = 'pxe-pcnet.rom';
1771         } elsif ($device eq 'rtl8139') {
1772             $romfile = 'pxe-rtl8139.rom';
1773         }
1774         $tmpstr .= ",romfile=$romfile" if $romfile;
1775     }
1776
1777     return $tmpstr;
1778 }
1779
1780 sub print_netdev_full {
1781     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1782
1783     my $i = '';
1784     if ($netid =~ m/^net(\d+)$/) {
1785         $i = int($1);
1786     }
1787
1788     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1789
1790     my $ifname = "tap${vmid}i$i";
1791
1792     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1793     die "interface name '$ifname' is too long (max 15 character)\n"
1794         if length($ifname) >= 16;
1795
1796     my $vhostparam = '';
1797     if (is_native($arch)) {
1798         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1799     }
1800
1801     my $vmname = $conf->{name} || "vm$vmid";
1802
1803     my $netdev = "";
1804     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1805
1806     if ($net->{bridge}) {
1807         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1808             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1809     } else {
1810         $netdev = "type=user,id=$netid,hostname=$vmname";
1811     }
1812
1813     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1814
1815     return $netdev;
1816 }
1817
1818 my $vga_map = {
1819     'cirrus' => 'cirrus-vga',
1820     'std' => 'VGA',
1821     'vmware' => 'vmware-svga',
1822     'virtio' => 'virtio-vga',
1823     'virtio-gl' => 'virtio-vga-gl',
1824 };
1825
1826 sub print_vga_device {
1827     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1828
1829     my $type = $vga_map->{$vga->{type}};
1830     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1831         $type = 'virtio-gpu';
1832     }
1833     my $vgamem_mb = $vga->{memory};
1834
1835     my $max_outputs = '';
1836     if ($qxlnum) {
1837         $type = $id ? 'qxl' : 'qxl-vga';
1838
1839         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1840             # set max outputs so linux can have up to 4 qxl displays with one device
1841             if (min_version($machine_version, 4, 1)) {
1842                 $max_outputs = ",max_outputs=4";
1843             }
1844         }
1845     }
1846
1847     die "no devicetype for $vga->{type}\n" if !$type;
1848
1849     my $memory = "";
1850     if ($vgamem_mb) {
1851         if ($vga->{type} =~ /^virtio/) {
1852             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1853             $memory = ",max_hostmem=$bytes";
1854         } elsif ($qxlnum) {
1855             # from https://www.spice-space.org/multiple-monitors.html
1856             $memory = ",vgamem_mb=$vga->{memory}";
1857             my $ram = $vgamem_mb * 4;
1858             my $vram = $vgamem_mb * 2;
1859             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1860         } else {
1861             $memory = ",vgamem_mb=$vga->{memory}";
1862         }
1863     } elsif ($qxlnum && $id) {
1864         $memory = ",ram_size=67108864,vram_size=33554432";
1865     }
1866
1867     my $edidoff = "";
1868     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1869         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1870     }
1871
1872     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1873     my $vgaid = "vga" . ($id // '');
1874     my $pciaddr;
1875     if ($q35 && $vgaid eq 'vga') {
1876         # the first display uses pcie.0 bus on q35 machines
1877         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1878     } else {
1879         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1880     }
1881
1882     if ($vga->{type} eq 'virtio-gl') {
1883         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1884         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1885             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1886
1887         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1888             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1889     }
1890
1891     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1892 }
1893
1894 sub parse_number_sets {
1895     my ($set) = @_;
1896     my $res = [];
1897     foreach my $part (split(/;/, $set)) {
1898         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1899             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1900             push @$res, [ $1, $2 ];
1901         } else {
1902             die "invalid range: $part\n";
1903         }
1904     }
1905     return $res;
1906 }
1907
1908 sub parse_numa {
1909     my ($data) = @_;
1910
1911     my $res = parse_property_string($numa_fmt, $data);
1912     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1913     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1914     return $res;
1915 }
1916
1917 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1918 sub parse_net {
1919     my ($data) = @_;
1920
1921     my $res = eval { parse_property_string($net_fmt, $data) };
1922     if ($@) {
1923         warn $@;
1924         return;
1925     }
1926     if (!defined($res->{macaddr})) {
1927         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1928         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1929     }
1930     return $res;
1931 }
1932
1933 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1934 sub parse_ipconfig {
1935     my ($data) = @_;
1936
1937     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1938     if ($@) {
1939         warn $@;
1940         return;
1941     }
1942
1943     if ($res->{gw} && !$res->{ip}) {
1944         warn 'gateway specified without specifying an IP address';
1945         return;
1946     }
1947     if ($res->{gw6} && !$res->{ip6}) {
1948         warn 'IPv6 gateway specified without specifying an IPv6 address';
1949         return;
1950     }
1951     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1952         warn 'gateway specified together with DHCP';
1953         return;
1954     }
1955     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1956         # gw6 + auto/dhcp
1957         warn "IPv6 gateway specified together with $res->{ip6} address";
1958         return;
1959     }
1960
1961     if (!$res->{ip} && !$res->{ip6}) {
1962         return { ip => 'dhcp', ip6 => 'dhcp' };
1963     }
1964
1965     return $res;
1966 }
1967
1968 sub print_net {
1969     my $net = shift;
1970
1971     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1972 }
1973
1974 sub add_random_macs {
1975     my ($settings) = @_;
1976
1977     foreach my $opt (keys %$settings) {
1978         next if $opt !~ m/^net(\d+)$/;
1979         my $net = parse_net($settings->{$opt});
1980         next if !$net;
1981         $settings->{$opt} = print_net($net);
1982     }
1983 }
1984
1985 sub vm_is_volid_owner {
1986     my ($storecfg, $vmid, $volid) = @_;
1987
1988     if ($volid !~  m|^/|) {
1989         my ($path, $owner);
1990         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1991         if ($owner && ($owner == $vmid)) {
1992             return 1;
1993         }
1994     }
1995
1996     return;
1997 }
1998
1999 sub vmconfig_register_unused_drive {
2000     my ($storecfg, $vmid, $conf, $drive) = @_;
2001
2002     if (drive_is_cloudinit($drive)) {
2003         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2004         warn $@ if $@;
2005         delete $conf->{cloudinit};
2006     } elsif (!drive_is_cdrom($drive)) {
2007         my $volid = $drive->{file};
2008         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2009             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2010         }
2011     }
2012 }
2013
2014 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2015 my $smbios1_fmt = {
2016     uuid => {
2017         type => 'string',
2018         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2019         format_description => 'UUID',
2020         description => "Set SMBIOS1 UUID.",
2021         optional => 1,
2022     },
2023     version => {
2024         type => 'string',
2025         pattern => '[A-Za-z0-9+\/]+={0,2}',
2026         format_description => 'Base64 encoded string',
2027         description => "Set SMBIOS1 version.",
2028         optional => 1,
2029     },
2030     serial => {
2031         type => 'string',
2032         pattern => '[A-Za-z0-9+\/]+={0,2}',
2033         format_description => 'Base64 encoded string',
2034         description => "Set SMBIOS1 serial number.",
2035         optional => 1,
2036     },
2037     manufacturer => {
2038         type => 'string',
2039         pattern => '[A-Za-z0-9+\/]+={0,2}',
2040         format_description => 'Base64 encoded string',
2041         description => "Set SMBIOS1 manufacturer.",
2042         optional => 1,
2043     },
2044     product => {
2045         type => 'string',
2046         pattern => '[A-Za-z0-9+\/]+={0,2}',
2047         format_description => 'Base64 encoded string',
2048         description => "Set SMBIOS1 product ID.",
2049         optional => 1,
2050     },
2051     sku => {
2052         type => 'string',
2053         pattern => '[A-Za-z0-9+\/]+={0,2}',
2054         format_description => 'Base64 encoded string',
2055         description => "Set SMBIOS1 SKU string.",
2056         optional => 1,
2057     },
2058     family => {
2059         type => 'string',
2060         pattern => '[A-Za-z0-9+\/]+={0,2}',
2061         format_description => 'Base64 encoded string',
2062         description => "Set SMBIOS1 family string.",
2063         optional => 1,
2064     },
2065     base64 => {
2066         type => 'boolean',
2067         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2068         optional => 1,
2069     },
2070 };
2071
2072 sub parse_smbios1 {
2073     my ($data) = @_;
2074
2075     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2076     warn $@ if $@;
2077     return $res;
2078 }
2079
2080 sub print_smbios1 {
2081     my ($smbios1) = @_;
2082     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2083 }
2084
2085 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2086
2087 sub parse_watchdog {
2088     my ($value) = @_;
2089
2090     return if !$value;
2091
2092     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2093     warn $@ if $@;
2094     return $res;
2095 }
2096
2097 sub parse_guest_agent {
2098     my ($conf) = @_;
2099
2100     return {} if !defined($conf->{agent});
2101
2102     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2103     warn $@ if $@;
2104
2105     # if the agent is disabled ignore the other potentially set properties
2106     return {} if !$res->{enabled};
2107     return $res;
2108 }
2109
2110 sub get_qga_key {
2111     my ($conf, $key) = @_;
2112     return undef if !defined($conf->{agent});
2113
2114     my $agent = parse_guest_agent($conf);
2115     return $agent->{$key};
2116 }
2117
2118 sub parse_vga {
2119     my ($value) = @_;
2120
2121     return {} if !$value;
2122     my $res = eval { parse_property_string($vga_fmt, $value) };
2123     warn $@ if $@;
2124     return $res;
2125 }
2126
2127 sub parse_rng {
2128     my ($value) = @_;
2129
2130     return if !$value;
2131
2132     my $res = eval { parse_property_string($rng_fmt, $value) };
2133     warn $@ if $@;
2134     return $res;
2135 }
2136
2137 sub parse_meta_info {
2138     my ($value) = @_;
2139
2140     return if !$value;
2141
2142     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2143     warn $@ if $@;
2144     return $res;
2145 }
2146
2147 sub new_meta_info_string {
2148     my () = @_; # for now do not allow to override any value
2149
2150     return PVE::JSONSchema::print_property_string(
2151         {
2152             'creation-qemu' => kvm_user_version(),
2153             ctime => "". int(time()),
2154         },
2155         $meta_info_fmt
2156     );
2157 }
2158
2159 sub qemu_created_version_fixups {
2160     my ($conf, $forcemachine, $kvmver) = @_;
2161
2162     my $meta = parse_meta_info($conf->{meta}) // {};
2163     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2164
2165     # check if we need to apply some handling for VMs that always use the latest machine version but
2166     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2167     # would be required (we do not want to pin machine version for non-windows OS type)
2168     if (
2169         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2170         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2171         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2172         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2173     ) {
2174         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2175         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2176             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2177             # and thus with the predictable interface naming of systemd
2178             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2179         }
2180     }
2181     return;
2182 }
2183
2184 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2185 sub verify_usb_device {
2186     my ($value, $noerr) = @_;
2187
2188     return $value if parse_usb_device($value);
2189
2190     return if $noerr;
2191
2192     die "unable to parse usb device\n";
2193 }
2194
2195 # add JSON properties for create and set function
2196 sub json_config_properties {
2197     my ($prop, $with_disk_alloc) = @_;
2198
2199     my $skip_json_config_opts = {
2200         parent => 1,
2201         snaptime => 1,
2202         vmstate => 1,
2203         runningmachine => 1,
2204         runningcpu => 1,
2205         meta => 1,
2206     };
2207
2208     foreach my $opt (keys %$confdesc) {
2209         next if $skip_json_config_opts->{$opt};
2210
2211         if ($with_disk_alloc && is_valid_drivename($opt)) {
2212             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2213         } else {
2214             $prop->{$opt} = $confdesc->{$opt};
2215         }
2216     }
2217
2218     return $prop;
2219 }
2220
2221 # Properties that we can read from an OVF file
2222 sub json_ovf_properties {
2223     my $prop = {};
2224
2225     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2226         $prop->{$device} = {
2227             type => 'string',
2228             format => 'pve-volume-id-or-absolute-path',
2229             description => "Disk image that gets imported to $device",
2230             optional => 1,
2231         };
2232     }
2233
2234     $prop->{cores} = {
2235         type => 'integer',
2236         description => "The number of CPU cores.",
2237         optional => 1,
2238     };
2239     $prop->{memory} = {
2240         type => 'integer',
2241         description => "Amount of RAM for the VM in MB.",
2242         optional => 1,
2243     };
2244     $prop->{name} = {
2245         type => 'string',
2246         description => "Name of the VM.",
2247         optional => 1,
2248     };
2249
2250     return $prop;
2251 }
2252
2253 # return copy of $confdesc_cloudinit to generate documentation
2254 sub cloudinit_config_properties {
2255
2256     return dclone($confdesc_cloudinit);
2257 }
2258
2259 sub check_type {
2260     my ($key, $value) = @_;
2261
2262     die "unknown setting '$key'\n" if !$confdesc->{$key};
2263
2264     my $type = $confdesc->{$key}->{type};
2265
2266     if (!defined($value)) {
2267         die "got undefined value\n";
2268     }
2269
2270     if ($value =~ m/[\n\r]/) {
2271         die "property contains a line feed\n";
2272     }
2273
2274     if ($type eq 'boolean') {
2275         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2276         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2277         die "type check ('boolean') failed - got '$value'\n";
2278     } elsif ($type eq 'integer') {
2279         return int($1) if $value =~ m/^(\d+)$/;
2280         die "type check ('integer') failed - got '$value'\n";
2281     } elsif ($type eq 'number') {
2282         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2283         die "type check ('number') failed - got '$value'\n";
2284     } elsif ($type eq 'string') {
2285         if (my $fmt = $confdesc->{$key}->{format}) {
2286             PVE::JSONSchema::check_format($fmt, $value);
2287             return $value;
2288         }
2289         $value =~ s/^\"(.*)\"$/$1/;
2290         return $value;
2291     } else {
2292         die "internal error"
2293     }
2294 }
2295
2296 sub destroy_vm {
2297     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2298
2299     my $conf = PVE::QemuConfig->load_config($vmid);
2300
2301     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2302
2303     if ($conf->{template}) {
2304         # check if any base image is still used by a linked clone
2305         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2306                 my ($ds, $drive) = @_;
2307                 return if drive_is_cdrom($drive);
2308
2309                 my $volid = $drive->{file};
2310                 return if !$volid || $volid =~ m|^/|;
2311
2312                 die "base volume '$volid' is still in use by linked cloned\n"
2313                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2314
2315         });
2316     }
2317
2318     my $volids = {};
2319     my $remove_owned_drive = sub {
2320         my ($ds, $drive) = @_;
2321         return if drive_is_cdrom($drive, 1);
2322
2323         my $volid = $drive->{file};
2324         return if !$volid || $volid =~ m|^/|;
2325         return if $volids->{$volid};
2326
2327         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2328         return if !$path || !$owner || ($owner != $vmid);
2329
2330         $volids->{$volid} = 1;
2331         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2332         warn "Could not remove disk '$volid', check manually: $@" if $@;
2333     };
2334
2335     # only remove disks owned by this VM (referenced in the config)
2336     my $include_opts = {
2337         include_unused => 1,
2338         extra_keys => ['vmstate'],
2339     };
2340     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2341
2342     for my $snap (values %{$conf->{snapshots}}) {
2343         next if !defined($snap->{vmstate});
2344         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2345         next if !defined($drive);
2346         $remove_owned_drive->('vmstate', $drive);
2347     }
2348
2349     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2350
2351     if ($purge_unreferenced) { # also remove unreferenced disk
2352         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2353         PVE::Storage::foreach_volid($vmdisks, sub {
2354             my ($volid, $sid, $volname, $d) = @_;
2355             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2356             warn $@ if $@;
2357         });
2358     }
2359
2360     if (defined $replacement_conf) {
2361         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2362     } else {
2363         PVE::QemuConfig->destroy_config($vmid);
2364     }
2365 }
2366
2367 sub parse_vm_config {
2368     my ($filename, $raw, $strict) = @_;
2369
2370     return if !defined($raw);
2371
2372     my $res = {
2373         digest => Digest::SHA::sha1_hex($raw),
2374         snapshots => {},
2375         pending => {},
2376         cloudinit => {},
2377     };
2378
2379     my $handle_error = sub {
2380         my ($msg) = @_;
2381
2382         if ($strict) {
2383             die $msg;
2384         } else {
2385             warn $msg;
2386         }
2387     };
2388
2389     $filename =~ m|/qemu-server/(\d+)\.conf$|
2390         || die "got strange filename '$filename'";
2391
2392     my $vmid = $1;
2393
2394     my $conf = $res;
2395     my $descr;
2396     my $section = '';
2397
2398     my @lines = split(/\n/, $raw);
2399     foreach my $line (@lines) {
2400         next if $line =~ m/^\s*$/;
2401
2402         if ($line =~ m/^\[PENDING\]\s*$/i) {
2403             $section = 'pending';
2404             if (defined($descr)) {
2405                 $descr =~ s/\s+$//;
2406                 $conf->{description} = $descr;
2407             }
2408             $descr = undef;
2409             $conf = $res->{$section} = {};
2410             next;
2411         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2412             $section = 'cloudinit';
2413             $descr = undef;
2414             $conf = $res->{$section} = {};
2415             next;
2416
2417         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2418             $section = $1;
2419             if (defined($descr)) {
2420                 $descr =~ s/\s+$//;
2421                 $conf->{description} = $descr;
2422             }
2423             $descr = undef;
2424             $conf = $res->{snapshots}->{$section} = {};
2425             next;
2426         }
2427
2428         if ($line =~ m/^\#(.*)$/) {
2429             $descr = '' if !defined($descr);
2430             $descr .= PVE::Tools::decode_text($1) . "\n";
2431             next;
2432         }
2433
2434         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2435             $descr = '' if !defined($descr);
2436             $descr .= PVE::Tools::decode_text($2);
2437         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2438             $conf->{snapstate} = $1;
2439         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2440             my $key = $1;
2441             my $value = $2;
2442             $conf->{$key} = $value;
2443         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2444             my $value = $1;
2445             if ($section eq 'pending') {
2446                 $conf->{delete} = $value; # we parse this later
2447             } else {
2448                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2449             }
2450         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2451             my $key = $1;
2452             my $value = $2;
2453             eval { $value = check_type($key, $value); };
2454             if ($@) {
2455                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2456             } else {
2457                 $key = 'ide2' if $key eq 'cdrom';
2458                 my $fmt = $confdesc->{$key}->{format};
2459                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2460                     my $v = parse_drive($key, $value);
2461                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2462                         $v->{file} = $volid;
2463                         $value = print_drive($v);
2464                     } else {
2465                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2466                         next;
2467                     }
2468                 }
2469
2470                 $conf->{$key} = $value;
2471             }
2472         } else {
2473             $handle_error->("vm $vmid - unable to parse config: $line\n");
2474         }
2475     }
2476
2477     if (defined($descr)) {
2478         $descr =~ s/\s+$//;
2479         $conf->{description} = $descr;
2480     }
2481     delete $res->{snapstate}; # just to be sure
2482
2483     return $res;
2484 }
2485
2486 sub write_vm_config {
2487     my ($filename, $conf) = @_;
2488
2489     delete $conf->{snapstate}; # just to be sure
2490
2491     if ($conf->{cdrom}) {
2492         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2493         $conf->{ide2} = $conf->{cdrom};
2494         delete $conf->{cdrom};
2495     }
2496
2497     # we do not use 'smp' any longer
2498     if ($conf->{sockets}) {
2499         delete $conf->{smp};
2500     } elsif ($conf->{smp}) {
2501         $conf->{sockets} = $conf->{smp};
2502         delete $conf->{cores};
2503         delete $conf->{smp};
2504     }
2505
2506     my $used_volids = {};
2507
2508     my $cleanup_config = sub {
2509         my ($cref, $pending, $snapname) = @_;
2510
2511         foreach my $key (keys %$cref) {
2512             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2513                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2514             my $value = $cref->{$key};
2515             if ($key eq 'delete') {
2516                 die "propertry 'delete' is only allowed in [PENDING]\n"
2517                     if !$pending;
2518                 # fixme: check syntax?
2519                 next;
2520             }
2521             eval { $value = check_type($key, $value); };
2522             die "unable to parse value of '$key' - $@" if $@;
2523
2524             $cref->{$key} = $value;
2525
2526             if (!$snapname && is_valid_drivename($key)) {
2527                 my $drive = parse_drive($key, $value);
2528                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2529             }
2530         }
2531     };
2532
2533     &$cleanup_config($conf);
2534
2535     &$cleanup_config($conf->{pending}, 1);
2536
2537     &$cleanup_config($conf->{cloudinit});
2538
2539     foreach my $snapname (keys %{$conf->{snapshots}}) {
2540         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2541         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2542     }
2543
2544     # remove 'unusedX' settings if we re-add a volume
2545     foreach my $key (keys %$conf) {
2546         my $value = $conf->{$key};
2547         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2548             delete $conf->{$key};
2549         }
2550     }
2551
2552     my $generate_raw_config = sub {
2553         my ($conf, $pending) = @_;
2554
2555         my $raw = '';
2556
2557         # add description as comment to top of file
2558         if (defined(my $descr = $conf->{description})) {
2559             if ($descr) {
2560                 foreach my $cl (split(/\n/, $descr)) {
2561                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2562                 }
2563             } else {
2564                 $raw .= "#\n" if $pending;
2565             }
2566         }
2567
2568         foreach my $key (sort keys %$conf) {
2569             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2570             $raw .= "$key: $conf->{$key}\n";
2571         }
2572         return $raw;
2573     };
2574
2575     my $raw = &$generate_raw_config($conf);
2576
2577     if (scalar(keys %{$conf->{pending}})){
2578         $raw .= "\n[PENDING]\n";
2579         $raw .= &$generate_raw_config($conf->{pending}, 1);
2580     }
2581
2582     if (scalar(keys %{$conf->{cloudinit}})){
2583         $raw .= "\n[special:cloudinit]\n";
2584         $raw .= &$generate_raw_config($conf->{cloudinit});
2585     }
2586
2587     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2588         $raw .= "\n[$snapname]\n";
2589         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2590     }
2591
2592     return $raw;
2593 }
2594
2595 sub load_defaults {
2596
2597     my $res = {};
2598
2599     # we use static defaults from our JSON schema configuration
2600     foreach my $key (keys %$confdesc) {
2601         if (defined(my $default = $confdesc->{$key}->{default})) {
2602             $res->{$key} = $default;
2603         }
2604     }
2605
2606     return $res;
2607 }
2608
2609 sub config_list {
2610     my $vmlist = PVE::Cluster::get_vmlist();
2611     my $res = {};
2612     return $res if !$vmlist || !$vmlist->{ids};
2613     my $ids = $vmlist->{ids};
2614     my $nodename = nodename();
2615
2616     foreach my $vmid (keys %$ids) {
2617         my $d = $ids->{$vmid};
2618         next if !$d->{node} || $d->{node} ne $nodename;
2619         next if !$d->{type} || $d->{type} ne 'qemu';
2620         $res->{$vmid}->{exists} = 1;
2621     }
2622     return $res;
2623 }
2624
2625 # test if VM uses local resources (to prevent migration)
2626 sub check_local_resources {
2627     my ($conf, $noerr) = @_;
2628
2629     my @loc_res = ();
2630
2631     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2632     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2633
2634     push @loc_res, "ivshmem" if $conf->{ivshmem};
2635
2636     foreach my $k (keys %$conf) {
2637         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2638         # sockets are safe: they will recreated be on the target side post-migrate
2639         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2640         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2641     }
2642
2643     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2644
2645     return \@loc_res;
2646 }
2647
2648 # check if used storages are available on all nodes (use by migrate)
2649 sub check_storage_availability {
2650     my ($storecfg, $conf, $node) = @_;
2651
2652     PVE::QemuConfig->foreach_volume($conf, sub {
2653         my ($ds, $drive) = @_;
2654
2655         my $volid = $drive->{file};
2656         return if !$volid;
2657
2658         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2659         return if !$sid;
2660
2661         # check if storage is available on both nodes
2662         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2663         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2664
2665         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2666
2667         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2668             if !$scfg->{content}->{$vtype};
2669    });
2670 }
2671
2672 # list nodes where all VM images are available (used by has_feature API)
2673 sub shared_nodes {
2674     my ($conf, $storecfg) = @_;
2675
2676     my $nodelist = PVE::Cluster::get_nodelist();
2677     my $nodehash = { map { $_ => 1 } @$nodelist };
2678     my $nodename = nodename();
2679
2680     PVE::QemuConfig->foreach_volume($conf, sub {
2681         my ($ds, $drive) = @_;
2682
2683         my $volid = $drive->{file};
2684         return if !$volid;
2685
2686         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2687         if ($storeid) {
2688             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2689             if ($scfg->{disable}) {
2690                 $nodehash = {};
2691             } elsif (my $avail = $scfg->{nodes}) {
2692                 foreach my $node (keys %$nodehash) {
2693                     delete $nodehash->{$node} if !$avail->{$node};
2694                 }
2695             } elsif (!$scfg->{shared}) {
2696                 foreach my $node (keys %$nodehash) {
2697                     delete $nodehash->{$node} if $node ne $nodename
2698                 }
2699             }
2700         }
2701     });
2702
2703     return $nodehash
2704 }
2705
2706 sub check_local_storage_availability {
2707     my ($conf, $storecfg) = @_;
2708
2709     my $nodelist = PVE::Cluster::get_nodelist();
2710     my $nodehash = { map { $_ => {} } @$nodelist };
2711
2712     PVE::QemuConfig->foreach_volume($conf, sub {
2713         my ($ds, $drive) = @_;
2714
2715         my $volid = $drive->{file};
2716         return if !$volid;
2717
2718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2719         if ($storeid) {
2720             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2721
2722             if ($scfg->{disable}) {
2723                 foreach my $node (keys %$nodehash) {
2724                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2725                 }
2726             } elsif (my $avail = $scfg->{nodes}) {
2727                 foreach my $node (keys %$nodehash) {
2728                     if (!$avail->{$node}) {
2729                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2730                     }
2731                 }
2732             }
2733         }
2734     });
2735
2736     foreach my $node (values %$nodehash) {
2737         if (my $unavail = $node->{unavailable_storages}) {
2738             $node->{unavailable_storages} = [ sort keys %$unavail ];
2739         }
2740     }
2741
2742     return $nodehash
2743 }
2744
2745 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2746 sub check_running {
2747     my ($vmid, $nocheck, $node) = @_;
2748
2749     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2750     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2751 }
2752
2753 sub vzlist {
2754
2755     my $vzlist = config_list();
2756
2757     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2758
2759     while (defined(my $de = $fd->read)) {
2760         next if $de !~ m/^(\d+)\.pid$/;
2761         my $vmid = $1;
2762         next if !defined($vzlist->{$vmid});
2763         if (my $pid = check_running($vmid)) {
2764             $vzlist->{$vmid}->{pid} = $pid;
2765         }
2766     }
2767
2768     return $vzlist;
2769 }
2770
2771 our $vmstatus_return_properties = {
2772     vmid => get_standard_option('pve-vmid'),
2773     status => {
2774         description => "Qemu process status.",
2775         type => 'string',
2776         enum => ['stopped', 'running'],
2777     },
2778     maxmem => {
2779         description => "Maximum memory in bytes.",
2780         type => 'integer',
2781         optional => 1,
2782         renderer => 'bytes',
2783     },
2784     maxdisk => {
2785         description => "Root disk size in bytes.",
2786         type => 'integer',
2787         optional => 1,
2788         renderer => 'bytes',
2789     },
2790     name => {
2791         description => "VM name.",
2792         type => 'string',
2793         optional => 1,
2794     },
2795     qmpstatus => {
2796         description => "Qemu QMP agent status.",
2797         type => 'string',
2798         optional => 1,
2799     },
2800     pid => {
2801         description => "PID of running qemu process.",
2802         type => 'integer',
2803         optional => 1,
2804     },
2805     uptime => {
2806         description => "Uptime.",
2807         type => 'integer',
2808         optional => 1,
2809         renderer => 'duration',
2810     },
2811     cpus => {
2812         description => "Maximum usable CPUs.",
2813         type => 'number',
2814         optional => 1,
2815     },
2816     lock => {
2817         description => "The current config lock, if any.",
2818         type => 'string',
2819         optional => 1,
2820     },
2821     tags => {
2822         description  => "The current configured tags, if any",
2823         type => 'string',
2824         optional => 1,
2825     },
2826     'running-machine' => {
2827         description => "The currently running machine type (if running).",
2828         type => 'string',
2829         optional => 1,
2830     },
2831     'running-qemu' => {
2832         description => "The currently running QEMU version (if running).",
2833         type => 'string',
2834         optional => 1,
2835     },
2836 };
2837
2838 my $last_proc_pid_stat;
2839
2840 # get VM status information
2841 # This must be fast and should not block ($full == false)
2842 # We only query KVM using QMP if $full == true (this can be slow)
2843 sub vmstatus {
2844     my ($opt_vmid, $full) = @_;
2845
2846     my $res = {};
2847
2848     my $storecfg = PVE::Storage::config();
2849
2850     my $list = vzlist();
2851     my $defaults = load_defaults();
2852
2853     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2854
2855     my $cpucount = $cpuinfo->{cpus} || 1;
2856
2857     foreach my $vmid (keys %$list) {
2858         next if $opt_vmid && ($vmid ne $opt_vmid);
2859
2860         my $conf = PVE::QemuConfig->load_config($vmid);
2861
2862         my $d = { vmid => int($vmid) };
2863         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2864
2865         # fixme: better status?
2866         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2867
2868         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2869         if (defined($size)) {
2870             $d->{disk} = 0; # no info available
2871             $d->{maxdisk} = $size;
2872         } else {
2873             $d->{disk} = 0;
2874             $d->{maxdisk} = 0;
2875         }
2876
2877         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2878             * ($conf->{cores} || $defaults->{cores});
2879         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2880         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2881
2882         $d->{name} = $conf->{name} || "VM $vmid";
2883         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2884             : $defaults->{memory}*(1024*1024);
2885
2886         if ($conf->{balloon}) {
2887             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2888             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2889                 : $defaults->{shares};
2890         }
2891
2892         $d->{uptime} = 0;
2893         $d->{cpu} = 0;
2894         $d->{mem} = 0;
2895
2896         $d->{netout} = 0;
2897         $d->{netin} = 0;
2898
2899         $d->{diskread} = 0;
2900         $d->{diskwrite} = 0;
2901
2902         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2903
2904         $d->{serial} = 1 if conf_has_serial($conf);
2905         $d->{lock} = $conf->{lock} if $conf->{lock};
2906         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2907
2908         $res->{$vmid} = $d;
2909     }
2910
2911     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2912     foreach my $dev (keys %$netdev) {
2913         next if $dev !~ m/^tap([1-9]\d*)i/;
2914         my $vmid = $1;
2915         my $d = $res->{$vmid};
2916         next if !$d;
2917
2918         $d->{netout} += $netdev->{$dev}->{receive};
2919         $d->{netin} += $netdev->{$dev}->{transmit};
2920
2921         if ($full) {
2922             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2923             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2924         }
2925
2926     }
2927
2928     my $ctime = gettimeofday;
2929
2930     foreach my $vmid (keys %$list) {
2931
2932         my $d = $res->{$vmid};
2933         my $pid = $d->{pid};
2934         next if !$pid;
2935
2936         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2937         next if !$pstat; # not running
2938
2939         my $used = $pstat->{utime} + $pstat->{stime};
2940
2941         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2942
2943         if ($pstat->{vsize}) {
2944             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2945         }
2946
2947         my $old = $last_proc_pid_stat->{$pid};
2948         if (!$old) {
2949             $last_proc_pid_stat->{$pid} = {
2950                 time => $ctime,
2951                 used => $used,
2952                 cpu => 0,
2953             };
2954             next;
2955         }
2956
2957         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2958
2959         if ($dtime > 1000) {
2960             my $dutime = $used -  $old->{used};
2961
2962             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2963             $last_proc_pid_stat->{$pid} = {
2964                 time => $ctime,
2965                 used => $used,
2966                 cpu => $d->{cpu},
2967             };
2968         } else {
2969             $d->{cpu} = $old->{cpu};
2970         }
2971     }
2972
2973     return $res if !$full;
2974
2975     my $qmpclient = PVE::QMPClient->new();
2976
2977     my $ballooncb = sub {
2978         my ($vmid, $resp) = @_;
2979
2980         my $info = $resp->{'return'};
2981         return if !$info->{max_mem};
2982
2983         my $d = $res->{$vmid};
2984
2985         # use memory assigned to VM
2986         $d->{maxmem} = $info->{max_mem};
2987         $d->{balloon} = $info->{actual};
2988
2989         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2990             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2991             $d->{freemem} = $info->{free_mem};
2992         }
2993
2994         $d->{ballooninfo} = $info;
2995     };
2996
2997     my $blockstatscb = sub {
2998         my ($vmid, $resp) = @_;
2999         my $data = $resp->{'return'} || [];
3000         my $totalrdbytes = 0;
3001         my $totalwrbytes = 0;
3002
3003         for my $blockstat (@$data) {
3004             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3005             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3006
3007             $blockstat->{device} =~ s/drive-//;
3008             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3009         }
3010         $res->{$vmid}->{diskread} = $totalrdbytes;
3011         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3012     };
3013
3014     my $machinecb = sub {
3015         my ($vmid, $resp) = @_;
3016         my $data = $resp->{'return'} || [];
3017
3018         $res->{$vmid}->{'running-machine'} =
3019             PVE::QemuServer::Machine::current_from_query_machines($data);
3020     };
3021
3022     my $versioncb = sub {
3023         my ($vmid, $resp) = @_;
3024         my $data = $resp->{'return'} // {};
3025         my $version = 'unknown';
3026
3027         if (my $v = $data->{qemu}) {
3028             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3029         }
3030
3031         $res->{$vmid}->{'running-qemu'} = $version;
3032     };
3033
3034     my $statuscb = sub {
3035         my ($vmid, $resp) = @_;
3036
3037         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3038         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3039         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3040         # this fails if ballon driver is not loaded, so this must be
3041         # the last commnand (following command are aborted if this fails).
3042         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3043
3044         my $status = 'unknown';
3045         if (!defined($status = $resp->{'return'}->{status})) {
3046             warn "unable to get VM status\n";
3047             return;
3048         }
3049
3050         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3051     };
3052
3053     foreach my $vmid (keys %$list) {
3054         next if $opt_vmid && ($vmid ne $opt_vmid);
3055         next if !$res->{$vmid}->{pid}; # not running
3056         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3057     }
3058
3059     $qmpclient->queue_execute(undef, 2);
3060
3061     foreach my $vmid (keys %$list) {
3062         next if $opt_vmid && ($vmid ne $opt_vmid);
3063         next if !$res->{$vmid}->{pid}; #not running
3064
3065         # we can't use the $qmpclient since it might have already aborted on
3066         # 'query-balloon', but this might also fail for older versions...
3067         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3068         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3069     }
3070
3071     foreach my $vmid (keys %$list) {
3072         next if $opt_vmid && ($vmid ne $opt_vmid);
3073         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3074     }
3075
3076     return $res;
3077 }
3078
3079 sub conf_has_serial {
3080     my ($conf) = @_;
3081
3082     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3083         if ($conf->{"serial$i"}) {
3084             return 1;
3085         }
3086     }
3087
3088     return 0;
3089 }
3090
3091 sub conf_has_audio {
3092     my ($conf, $id) = @_;
3093
3094     $id //= 0;
3095     my $audio = $conf->{"audio$id"};
3096     return if !defined($audio);
3097
3098     my $audioproperties = parse_property_string($audio_fmt, $audio);
3099     my $audiodriver = $audioproperties->{driver} // 'spice';
3100
3101     return {
3102         dev => $audioproperties->{device},
3103         dev_id => "audiodev$id",
3104         backend => $audiodriver,
3105         backend_id => "$audiodriver-backend${id}",
3106     };
3107 }
3108
3109 sub audio_devs {
3110     my ($audio, $audiopciaddr, $machine_version) = @_;
3111
3112     my $devs = [];
3113
3114     my $id = $audio->{dev_id};
3115     my $audiodev = "";
3116     if (min_version($machine_version, 4, 2)) {
3117         $audiodev = ",audiodev=$audio->{backend_id}";
3118     }
3119
3120     if ($audio->{dev} eq 'AC97') {
3121         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3122     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3123         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3124         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3125         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3126     } else {
3127         die "unkown audio device '$audio->{dev}', implement me!";
3128     }
3129
3130     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3131
3132     return $devs;
3133 }
3134
3135 sub get_tpm_paths {
3136     my ($vmid) = @_;
3137     return {
3138         socket => "/var/run/qemu-server/$vmid.swtpm",
3139         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3140     };
3141 }
3142
3143 sub add_tpm_device {
3144     my ($vmid, $devices, $conf) = @_;
3145
3146     return if !$conf->{tpmstate0};
3147
3148     my $paths = get_tpm_paths($vmid);
3149
3150     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3151     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3152     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3153 }
3154
3155 sub start_swtpm {
3156     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3157
3158     return if !$tpmdrive;
3159
3160     my $state;
3161     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3162     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3163     if ($storeid) {
3164         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3165     } else {
3166         $state = $tpm->{file};
3167     }
3168
3169     my $paths = get_tpm_paths($vmid);
3170
3171     # during migration, we will get state from remote
3172     #
3173     if (!$migration) {
3174         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3175         my $setup_cmd = [
3176             "swtpm_setup",
3177             "--tpmstate",
3178             "file://$state",
3179             "--createek",
3180             "--create-ek-cert",
3181             "--create-platform-cert",
3182             "--lock-nvram",
3183             "--config",
3184             "/etc/swtpm_setup.conf", # do not use XDG configs
3185             "--runas",
3186             "0", # force creation as root, error if not possible
3187             "--not-overwrite", # ignore existing state, do not modify
3188         ];
3189
3190         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3191         # TPM 2.0 supports ECC crypto, use if possible
3192         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3193
3194         run_command($setup_cmd, outfunc => sub {
3195             print "swtpm_setup: $1\n";
3196         });
3197     }
3198
3199     my $emulator_cmd = [
3200         "swtpm",
3201         "socket",
3202         "--tpmstate",
3203         "backend-uri=file://$state,mode=0600",
3204         "--ctrl",
3205         "type=unixio,path=$paths->{socket},mode=0600",
3206         "--pid",
3207         "file=$paths->{pid}",
3208         "--terminate", # terminate on QEMU disconnect
3209         "--daemon",
3210     ];
3211     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3212     run_command($emulator_cmd, outfunc => sub { print $1; });
3213
3214     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3215     while (! -e $paths->{pid}) {
3216         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3217         usleep(50_000);
3218     }
3219
3220     # return untainted PID of swtpm daemon so it can be killed on error
3221     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3222     return $1;
3223 }
3224
3225 sub vga_conf_has_spice {
3226     my ($vga) = @_;
3227
3228     my $vgaconf = parse_vga($vga);
3229     my $vgatype = $vgaconf->{type};
3230     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3231
3232     return $1 || 1;
3233 }
3234
3235 sub is_native($) {
3236     my ($arch) = @_;
3237     return get_host_arch() eq $arch;
3238 }
3239
3240 sub get_vm_arch {
3241     my ($conf) = @_;
3242     return $conf->{arch} // get_host_arch();
3243 }
3244
3245 my $default_machines = {
3246     x86_64 => 'pc',
3247     aarch64 => 'virt',
3248 };
3249
3250 sub get_installed_machine_version {
3251     my ($kvmversion) = @_;
3252     $kvmversion = kvm_user_version() if !defined($kvmversion);
3253     $kvmversion =~ m/^(\d+\.\d+)/;
3254     return $1;
3255 }
3256
3257 sub windows_get_pinned_machine_version {
3258     my ($machine, $base_version, $kvmversion) = @_;
3259
3260     my $pin_version = $base_version;
3261     if (!defined($base_version) ||
3262         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3263     ) {
3264         $pin_version = get_installed_machine_version($kvmversion);
3265     }
3266     if (!$machine || $machine eq 'pc') {
3267         $machine = "pc-i440fx-$pin_version";
3268     } elsif ($machine eq 'q35') {
3269         $machine = "pc-q35-$pin_version";
3270     } elsif ($machine eq 'virt') {
3271         $machine = "virt-$pin_version";
3272     } else {
3273         warn "unknown machine type '$machine', not touching that!\n";
3274     }
3275
3276     return $machine;
3277 }
3278
3279 sub get_vm_machine {
3280     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3281
3282     my $machine = $forcemachine || $conf->{machine};
3283
3284     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3285         $kvmversion //= kvm_user_version();
3286         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3287         # layout which confuses windows quite a bit and may result in various regressions..
3288         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3289         if (windows_version($conf->{ostype})) {
3290             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3291         }
3292         $arch //= 'x86_64';
3293         $machine ||= $default_machines->{$arch};
3294         if ($add_pve_version) {
3295             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3296             $machine .= "+pve$pvever";
3297         }
3298     }
3299
3300     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3301         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3302         $machine = $1 if $is_pxe;
3303
3304         # for version-pinned machines that do not include a pve-version (e.g.
3305         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3306         $machine .= '+pve0';
3307
3308         $machine .= '.pxe' if $is_pxe;
3309     }
3310
3311     return $machine;
3312 }
3313
3314 sub get_ovmf_files($$$) {
3315     my ($arch, $efidisk, $smm) = @_;
3316
3317     my $types = $OVMF->{$arch}
3318         or die "no OVMF images known for architecture '$arch'\n";
3319
3320     my $type = 'default';
3321     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3322         $type = $smm ? "4m" : "4m-no-smm";
3323         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3324     }
3325
3326     return $types->{$type}->@*;
3327 }
3328
3329 my $Arch2Qemu = {
3330     aarch64 => '/usr/bin/qemu-system-aarch64',
3331     x86_64 => '/usr/bin/qemu-system-x86_64',
3332 };
3333 sub get_command_for_arch($) {
3334     my ($arch) = @_;
3335     return '/usr/bin/kvm' if is_native($arch);
3336
3337     my $cmd = $Arch2Qemu->{$arch}
3338         or die "don't know how to emulate architecture '$arch'\n";
3339     return $cmd;
3340 }
3341
3342 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3343 # to use in a QEMU command line (-cpu element), first array_intersect the result
3344 # of query_supported_ with query_understood_. This is necessary because:
3345 #
3346 # a) query_understood_ returns flags the host cannot use and
3347 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3348 #    flags, but CPU settings - with most of them being flags. Those settings
3349 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3350 #
3351 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3352 # expensive. If you need the value returned from this, you can get it much
3353 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3354 # $accel being 'kvm' or 'tcg'.
3355 #
3356 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3357 # changes, automatically populating pmxcfs.
3358 #
3359 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3360 # since kvm and tcg machines support different flags
3361 #
3362 sub query_supported_cpu_flags {
3363     my ($arch) = @_;
3364
3365     $arch //= get_host_arch();
3366     my $default_machine = $default_machines->{$arch};
3367
3368     my $flags = {};
3369
3370     # FIXME: Once this is merged, the code below should work for ARM as well:
3371     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3372     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3373         $arch eq "aarch64";
3374
3375     my $kvm_supported = defined(kvm_version());
3376     my $qemu_cmd = get_command_for_arch($arch);
3377     my $fakevmid = -1;
3378     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3379
3380     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3381     my $query_supported_run_qemu = sub {
3382         my ($kvm) = @_;
3383
3384         my $flags = {};
3385         my $cmd = [
3386             $qemu_cmd,
3387             '-machine', $default_machine,
3388             '-display', 'none',
3389             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3390             '-mon', 'chardev=qmp,mode=control',
3391             '-pidfile', $pidfile,
3392             '-S', '-daemonize'
3393         ];
3394
3395         if (!$kvm) {
3396             push @$cmd, '-accel', 'tcg';
3397         }
3398
3399         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3400         die "QEMU flag querying VM exited with code " . $rc if $rc;
3401
3402         eval {
3403             my $cmd_result = mon_cmd(
3404                 $fakevmid,
3405                 'query-cpu-model-expansion',
3406                 type => 'full',
3407                 model => { name => 'host' }
3408             );
3409
3410             my $props = $cmd_result->{model}->{props};
3411             foreach my $prop (keys %$props) {
3412                 next if $props->{$prop} ne '1';
3413                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3414                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3415                 # We only keep those with underscores, to match /proc/cpuinfo
3416                 $prop =~ s/\.|-/_/g;
3417                 $flags->{$prop} = 1;
3418             }
3419         };
3420         my $err = $@;
3421
3422         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3423         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3424
3425         die $err if $err;
3426
3427         return [ sort keys %$flags ];
3428     };
3429
3430     # We need to query QEMU twice, since KVM and TCG have different supported flags
3431     PVE::QemuConfig->lock_config($fakevmid, sub {
3432         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3433         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3434
3435         if ($kvm_supported) {
3436             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3437             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3438         }
3439     });
3440
3441     return $flags;
3442 }
3443
3444 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3445 my $understood_cpu_flag_dir = "/usr/share/kvm";
3446 sub query_understood_cpu_flags {
3447     my $arch = get_host_arch();
3448     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3449
3450     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3451         if ! -e $filepath;
3452
3453     my $raw = file_get_contents($filepath);
3454     $raw =~ s/^\s+|\s+$//g;
3455     my @flags = split(/\s+/, $raw);
3456
3457     return \@flags;
3458 }
3459
3460 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3461 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3462 my sub should_disable_smm {
3463     my ($conf, $vga) = @_;
3464
3465     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3466         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3467 }
3468
3469 sub config_to_command {
3470     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3471         $pbs_backing) = @_;
3472
3473     my $cmd = [];
3474     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3475     my $devices = [];
3476     my $bridges = {};
3477     my $ostype = $conf->{ostype};
3478     my $winversion = windows_version($ostype);
3479     my $kvm = $conf->{kvm};
3480     my $nodename = nodename();
3481
3482     my $arch = get_vm_arch($conf);
3483     my $kvm_binary = get_command_for_arch($arch);
3484     my $kvmver = kvm_user_version($kvm_binary);
3485
3486     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3487         $kvmver //= "undefined";
3488         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3489     }
3490
3491     my $add_pve_version = min_version($kvmver, 4, 1);
3492
3493     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3494     my $machine_version = extract_version($machine_type, $kvmver);
3495     $kvm //= 1 if is_native($arch);
3496
3497     $machine_version =~ m/(\d+)\.(\d+)/;
3498     my ($machine_major, $machine_minor) = ($1, $2);
3499
3500     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3501         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3502     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3503         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3504             ." please upgrade node '$nodename'\n"
3505     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3506         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3507         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3508             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3509             ." node '$nodename'\n";
3510     }
3511
3512     # if a specific +pve version is required for a feature, use $version_guard
3513     # instead of min_version to allow machines to be run with the minimum
3514     # required version
3515     my $required_pve_version = 0;
3516     my $version_guard = sub {
3517         my ($major, $minor, $pve) = @_;
3518         return 0 if !min_version($machine_version, $major, $minor, $pve);
3519         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3520         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3521         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3522         return 1;
3523     };
3524
3525     if ($kvm && !defined kvm_version()) {
3526         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3527             ." or enable in BIOS.\n";
3528     }
3529
3530     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3531     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3532     my $use_old_bios_files = undef;
3533     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3534
3535     push @$cmd, $kvm_binary;
3536
3537     push @$cmd, '-id', $vmid;
3538
3539     my $vmname = $conf->{name} || "vm$vmid";
3540
3541     push @$cmd, '-name', "$vmname,debug-threads=on";
3542
3543     push @$cmd, '-no-shutdown';
3544
3545     my $use_virtio = 0;
3546
3547     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3548     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3549     push @$cmd, '-mon', "chardev=qmp,mode=control";
3550
3551     if (min_version($machine_version, 2, 12)) {
3552         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3553         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3554     }
3555
3556     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3557
3558     push @$cmd, '-daemonize';
3559
3560     if ($conf->{smbios1}) {
3561         my $smbios_conf = parse_smbios1($conf->{smbios1});
3562         if ($smbios_conf->{base64}) {
3563             # Do not pass base64 flag to qemu
3564             delete $smbios_conf->{base64};
3565             my $smbios_string = "";
3566             foreach my $key (keys %$smbios_conf) {
3567                 my $value;
3568                 if ($key eq "uuid") {
3569                     $value = $smbios_conf->{uuid}
3570                 } else {
3571                     $value = decode_base64($smbios_conf->{$key});
3572                 }
3573                 # qemu accepts any binary data, only commas need escaping by double comma
3574                 $value =~ s/,/,,/g;
3575                 $smbios_string .= "," . $key . "=" . $value if $value;
3576             }
3577             push @$cmd, '-smbios', "type=1" . $smbios_string;
3578         } else {
3579             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3580         }
3581     }
3582
3583     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3584         my $d;
3585         if (my $efidisk = $conf->{efidisk0}) {
3586             $d = parse_drive('efidisk0', $efidisk);
3587         }
3588
3589         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3590         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3591
3592         my ($path, $format);
3593         my $read_only_str = '';
3594         if ($d) {
3595             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3596             $format = $d->{format};
3597             if ($storeid) {
3598                 $path = PVE::Storage::path($storecfg, $d->{file});
3599                 if (!defined($format)) {
3600                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3601                     $format = qemu_img_format($scfg, $volname);
3602                 }
3603             } else {
3604                 $path = $d->{file};
3605                 die "efidisk format must be specified\n"
3606                     if !defined($format);
3607             }
3608
3609             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3610         } else {
3611             log_warn("no efidisk configured! Using temporary efivars disk.");
3612             $path = "/tmp/$vmid-ovmf.fd";
3613             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3614             $format = 'raw';
3615         }
3616
3617         my $size_str = "";
3618
3619         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3620             $size_str = ",size=" . (-s $ovmf_vars);
3621         }
3622
3623         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3624         my $cache = "";
3625         if ($path =~ m/^rbd:/) {
3626                 $cache = ',cache=writeback';
3627                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3628         }
3629
3630         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3631         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3632     }
3633
3634     if ($q35) { # tell QEMU to load q35 config early
3635         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3636         if (min_version($machine_version, 4, 0)) {
3637             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3638         } else {
3639             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3640         }
3641     }
3642
3643     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3644         push @$cmd, $fixups->@*;
3645     }
3646
3647     if ($conf->{vmgenid}) {
3648         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3649     }
3650
3651     # add usb controllers
3652     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3653         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3654     push @$devices, @usbcontrollers if @usbcontrollers;
3655     my $vga = parse_vga($conf->{vga});
3656
3657     my $qxlnum = vga_conf_has_spice($conf->{vga});
3658     $vga->{type} = 'qxl' if $qxlnum;
3659
3660     if (!$vga->{type}) {
3661         if ($arch eq 'aarch64') {
3662             $vga->{type} = 'virtio';
3663         } elsif (min_version($machine_version, 2, 9)) {
3664             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3665         } else {
3666             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3667         }
3668     }
3669
3670     # enable absolute mouse coordinates (needed by vnc)
3671     my $tablet = $conf->{tablet};
3672     if (!defined($tablet)) {
3673         $tablet = $defaults->{tablet};
3674         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3675         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3676     }
3677
3678     if ($tablet) {
3679         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3680         my $kbd = print_keyboarddevice_full($conf, $arch);
3681         push @$devices, '-device', $kbd if defined($kbd);
3682     }
3683
3684     my $bootorder = device_bootorder($conf);
3685
3686     # host pci device passthrough
3687     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3688         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3689
3690     # usb devices
3691     my $usb_dev_features = {};
3692     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3693
3694     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3695         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3696     push @$devices, @usbdevices if @usbdevices;
3697
3698     # serial devices
3699     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3700         my $path = $conf->{"serial$i"} or next;
3701         if ($path eq 'socket') {
3702             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3703             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3704             # On aarch64, serial0 is the UART device. Qemu only allows
3705             # connecting UART devices via the '-serial' command line, as
3706             # the device has a fixed slot on the hardware...
3707             if ($arch eq 'aarch64' && $i == 0) {
3708                 push @$devices, '-serial', "chardev:serial$i";
3709             } else {
3710                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3711             }
3712         } else {
3713             die "no such serial device\n" if ! -c $path;
3714             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3715             push @$devices, '-device', "isa-serial,chardev=serial$i";
3716         }
3717     }
3718
3719     # parallel devices
3720     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3721         if (my $path = $conf->{"parallel$i"}) {
3722             die "no such parallel device\n" if ! -c $path;
3723             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3724             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3725             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3726         }
3727     }
3728
3729     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3730         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3731         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3732         push @$devices, @$audio_devs;
3733     }
3734
3735     add_tpm_device($vmid, $devices, $conf);
3736
3737     my $sockets = 1;
3738     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3739     $sockets = $conf->{sockets} if  $conf->{sockets};
3740
3741     my $cores = $conf->{cores} || 1;
3742
3743     my $maxcpus = $sockets * $cores;
3744
3745     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3746
3747     my $allowed_vcpus = $cpuinfo->{cpus};
3748
3749     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3750
3751     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3752         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753         for (my $i = 2; $i <= $vcpus; $i++)  {
3754             my $cpustr = print_cpu_device($conf,$i);
3755             push @$cmd, '-device', $cpustr;
3756         }
3757
3758     } else {
3759
3760         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3761     }
3762     push @$cmd, '-nodefaults';
3763
3764     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3765
3766     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3767
3768     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3769
3770     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3771         push @$devices, '-device', print_vga_device(
3772             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3773
3774         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3775
3776         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3777         push @$cmd,  '-vnc', "unix:$socket,password=on";
3778     } else {
3779         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3780         push @$cmd, '-nographic';
3781     }
3782
3783     # time drift fix
3784     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3785     my $useLocaltime = $conf->{localtime};
3786
3787     if ($winversion >= 5) { # windows
3788         $useLocaltime = 1 if !defined($conf->{localtime});
3789
3790         # use time drift fix when acpi is enabled
3791         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3792             $tdf = 1 if !defined($conf->{tdf});
3793         }
3794     }
3795
3796     if ($winversion >= 6) {
3797         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3798         push @$cmd, '-no-hpet';
3799     }
3800
3801     push @$rtcFlags, 'driftfix=slew' if $tdf;
3802
3803     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3804         push @$rtcFlags, "base=$conf->{startdate}";
3805     } elsif ($useLocaltime) {
3806         push @$rtcFlags, 'base=localtime';
3807     }
3808
3809     if ($forcecpu) {
3810         push @$cmd, '-cpu', $forcecpu;
3811     } else {
3812         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3813     }
3814
3815     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3816
3817     push @$cmd, '-S' if $conf->{freeze};
3818
3819     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3820
3821     my $guest_agent = parse_guest_agent($conf);
3822
3823     if ($guest_agent->{enabled}) {
3824         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3825         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3826
3827         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3828             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3829             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3830             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3831         } elsif ($guest_agent->{type} eq 'isa') {
3832             push @$devices, '-device', "isa-serial,chardev=qga0";
3833         }
3834     }
3835
3836     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3837     if ($rng && $version_guard->(4, 1, 2)) {
3838         check_rng_source($rng->{source});
3839
3840         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3841         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3842         my $limiter_str = "";
3843         if ($max_bytes) {
3844             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3845         }
3846
3847         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3848         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3849         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3850     }
3851
3852     my $spice_port;
3853
3854     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3855         if ($qxlnum > 1) {
3856             if ($winversion){
3857                 for (my $i = 1; $i < $qxlnum; $i++){
3858                     push @$devices, '-device', print_vga_device(
3859                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3860                 }
3861             } else {
3862                 # assume other OS works like Linux
3863                 my ($ram, $vram) = ("134217728", "67108864");
3864                 if ($vga->{memory}) {
3865                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3866                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3867                 }
3868                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3869                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3870             }
3871         }
3872
3873         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3874
3875         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3876         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3877         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3878
3879         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3880         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3881         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3882
3883         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3884         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3885
3886         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3887         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3888         if ($spice_enhancement->{foldersharing}) {
3889             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3890             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3891         }
3892
3893         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3894         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3895             if $spice_enhancement->{videostreaming};
3896
3897         push @$devices, '-spice', "$spice_opts";
3898     }
3899
3900     # enable balloon by default, unless explicitly disabled
3901     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3902         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3903         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3904         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3905         push @$devices, '-device', $ballooncmd;
3906     }
3907
3908     if ($conf->{watchdog}) {
3909         my $wdopts = parse_watchdog($conf->{watchdog});
3910         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3911         my $watchdog = $wdopts->{model} || 'i6300esb';
3912         push @$devices, '-device', "$watchdog$pciaddr";
3913         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3914     }
3915
3916     my $vollist = [];
3917     my $scsicontroller = {};
3918     my $ahcicontroller = {};
3919     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3920
3921     # Add iscsi initiator name if available
3922     if (my $initiator = get_initiator_name()) {
3923         push @$devices, '-iscsi', "initiator-name=$initiator";
3924     }
3925
3926     PVE::QemuConfig->foreach_volume($conf, sub {
3927         my ($ds, $drive) = @_;
3928
3929         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3930             check_volume_storage_type($storecfg, $drive->{file});
3931             push @$vollist, $drive->{file};
3932         }
3933
3934         # ignore efidisk here, already added in bios/fw handling code above
3935         return if $drive->{interface} eq 'efidisk';
3936         # similar for TPM
3937         return if $drive->{interface} eq 'tpmstate';
3938
3939         $use_virtio = 1 if $ds =~ m/^virtio/;
3940
3941         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3942
3943         if ($drive->{interface} eq 'virtio'){
3944            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3945         }
3946
3947         if ($drive->{interface} eq 'scsi') {
3948
3949             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3950
3951             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3952                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3953
3954             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3955             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3956
3957             my $iothread = '';
3958             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3959                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3960                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3961             } elsif ($drive->{iothread}) {
3962                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3963             }
3964
3965             my $queues = '';
3966             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3967                 $queues = ",num_queues=$drive->{queues}";
3968             }
3969
3970             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3971                 if !$scsicontroller->{$controller};
3972             $scsicontroller->{$controller}=1;
3973         }
3974
3975         if ($drive->{interface} eq 'sata') {
3976             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3977             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3978             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3979                 if !$ahcicontroller->{$controller};
3980             $ahcicontroller->{$controller}=1;
3981         }
3982
3983         my $pbs_conf = $pbs_backing->{$ds};
3984         my $pbs_name = undef;
3985         if ($pbs_conf) {
3986             $pbs_name = "drive-$ds-pbs";
3987             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3988         }
3989
3990         my $drive_cmd = print_drive_commandline_full(
3991             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3992
3993         # extra protection for templates, but SATA and IDE don't support it..
3994         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3995
3996         push @$devices, '-drive',$drive_cmd;
3997         push @$devices, '-device', print_drivedevice_full(
3998             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3999     });
4000
4001     for (my $i = 0; $i < $MAX_NETS; $i++) {
4002         my $netname = "net$i";
4003
4004         next if !$conf->{$netname};
4005         my $d = parse_net($conf->{$netname});
4006         next if !$d;
4007
4008         $use_virtio = 1 if $d->{model} eq 'virtio';
4009
4010         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4011
4012         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4013         push @$devices, '-netdev', $netdevfull;
4014
4015         my $netdevicefull = print_netdevice_full(
4016             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4017
4018         push @$devices, '-device', $netdevicefull;
4019     }
4020
4021     if ($conf->{ivshmem}) {
4022         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4023
4024         my $bus;
4025         if ($q35) {
4026             $bus = print_pcie_addr("ivshmem");
4027         } else {
4028             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4029         }
4030
4031         my $ivshmem_name = $ivshmem->{name} // $vmid;
4032         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4033
4034         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4035         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4036             .",size=$ivshmem->{size}M";
4037     }
4038
4039     # pci.4 is nested in pci.1
4040     $bridges->{1} = 1 if $bridges->{4};
4041
4042     if (!$q35) { # add pci bridges
4043         if (min_version($machine_version, 2, 3)) {
4044            $bridges->{1} = 1;
4045            $bridges->{2} = 1;
4046         }
4047         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4048     }
4049
4050     for my $k (sort {$b cmp $a} keys %$bridges) {
4051         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4052
4053         my $k_name = $k;
4054         if ($k == 2 && $legacy_igd) {
4055             $k_name = "$k-igd";
4056         }
4057         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4058         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4059
4060         if ($q35) { # add after -readconfig pve-q35.cfg
4061             splice @$devices, 2, 0, '-device', $devstr;
4062         } else {
4063             unshift @$devices, '-device', $devstr if $k > 0;
4064         }
4065     }
4066
4067     if (!$kvm) {
4068         push @$machineFlags, 'accel=tcg';
4069     }
4070
4071     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4072
4073     my $machine_type_min = $machine_type;
4074     if ($add_pve_version) {
4075         $machine_type_min =~ s/\+pve\d+$//;
4076         $machine_type_min .= "+pve$required_pve_version";
4077     }
4078     push @$machineFlags, "type=${machine_type_min}";
4079
4080     push @$cmd, @$devices;
4081     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4082     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4083     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4084
4085     if (my $vmstate = $conf->{vmstate}) {
4086         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4087         push @$vollist, $vmstate;
4088         push @$cmd, '-loadstate', $statepath;
4089         print "activating and using '$vmstate' as vmstate\n";
4090     }
4091
4092     if (PVE::QemuConfig->is_template($conf)) {
4093         # needed to workaround base volumes being read-only
4094         push @$cmd, '-snapshot';
4095     }
4096
4097     # add custom args
4098     if ($conf->{args}) {
4099         my $aa = PVE::Tools::split_args($conf->{args});
4100         push @$cmd, @$aa;
4101     }
4102
4103     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4104 }
4105
4106 sub check_rng_source {
4107     my ($source) = @_;
4108
4109     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4110     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4111         if ! -e $source;
4112
4113     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4114     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4115         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4116         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4117         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4118             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4119             ." to the host.\n";
4120     }
4121 }
4122
4123 sub spice_port {
4124     my ($vmid) = @_;
4125
4126     my $res = mon_cmd($vmid, 'query-spice');
4127
4128     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4129 }
4130
4131 sub vm_devices_list {
4132     my ($vmid) = @_;
4133
4134     my $res = mon_cmd($vmid, 'query-pci');
4135     my $devices_to_check = [];
4136     my $devices = {};
4137     foreach my $pcibus (@$res) {
4138         push @$devices_to_check, @{$pcibus->{devices}},
4139     }
4140
4141     while (@$devices_to_check) {
4142         my $to_check = [];
4143         for my $d (@$devices_to_check) {
4144             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4145             next if !$d->{'pci_bridge'};
4146
4147             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4148             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4149         }
4150         $devices_to_check = $to_check;
4151     }
4152
4153     my $resblock = mon_cmd($vmid, 'query-block');
4154     foreach my $block (@$resblock) {
4155         if($block->{device} =~ m/^drive-(\S+)/){
4156                 $devices->{$1} = 1;
4157         }
4158     }
4159
4160     my $resmice = mon_cmd($vmid, 'query-mice');
4161     foreach my $mice (@$resmice) {
4162         if ($mice->{name} eq 'QEMU HID Tablet') {
4163             $devices->{tablet} = 1;
4164             last;
4165         }
4166     }
4167
4168     # for usb devices there is no query-usb
4169     # but we can iterate over the entries in
4170     # qom-list path=/machine/peripheral
4171     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4172     foreach my $per (@$resperipheral) {
4173         if ($per->{name} =~ m/^usb\d+$/) {
4174             $devices->{$per->{name}} = 1;
4175         }
4176     }
4177
4178     return $devices;
4179 }
4180
4181 sub vm_deviceplug {
4182     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4183
4184     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4185
4186     my $devices_list = vm_devices_list($vmid);
4187     return 1 if defined($devices_list->{$deviceid});
4188
4189     # add PCI bridge if we need it for the device
4190     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4191
4192     if ($deviceid eq 'tablet') {
4193         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4194     } elsif ($deviceid eq 'keyboard') {
4195         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4196     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4197         die "usb hotplug currently not reliable\n";
4198         # since we can't reliably hot unplug all added usb devices and usb
4199         # passthrough breaks live migration we disable usb hotplugging for now
4200         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4201     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4202         qemu_iothread_add($vmid, $deviceid, $device);
4203
4204         qemu_driveadd($storecfg, $vmid, $device);
4205         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4206
4207         qemu_deviceadd($vmid, $devicefull);
4208         eval { qemu_deviceaddverify($vmid, $deviceid); };
4209         if (my $err = $@) {
4210             eval { qemu_drivedel($vmid, $deviceid); };
4211             warn $@ if $@;
4212             die $err;
4213         }
4214     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4215         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4216         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4217         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4218
4219         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4220
4221         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4222             qemu_iothread_add($vmid, $deviceid, $device);
4223             $devicefull .= ",iothread=iothread-$deviceid";
4224         }
4225
4226         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4227             $devicefull .= ",num_queues=$device->{queues}";
4228         }
4229
4230         qemu_deviceadd($vmid, $devicefull);
4231         qemu_deviceaddverify($vmid, $deviceid);
4232     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4233         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4234         qemu_driveadd($storecfg, $vmid, $device);
4235
4236         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4237         eval { qemu_deviceadd($vmid, $devicefull); };
4238         if (my $err = $@) {
4239             eval { qemu_drivedel($vmid, $deviceid); };
4240             warn $@ if $@;
4241             die $err;
4242         }
4243     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4244         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4245
4246         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4247         my $use_old_bios_files = undef;
4248         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4249
4250         my $netdevicefull = print_netdevice_full(
4251             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4252         qemu_deviceadd($vmid, $netdevicefull);
4253         eval {
4254             qemu_deviceaddverify($vmid, $deviceid);
4255             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4256         };
4257         if (my $err = $@) {
4258             eval { qemu_netdevdel($vmid, $deviceid); };
4259             warn $@ if $@;
4260             die $err;
4261         }
4262     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4263         my $bridgeid = $2;
4264         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4265         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4266
4267         qemu_deviceadd($vmid, $devicefull);
4268         qemu_deviceaddverify($vmid, $deviceid);
4269     } else {
4270         die "can't hotplug device '$deviceid'\n";
4271     }
4272
4273     return 1;
4274 }
4275
4276 # fixme: this should raise exceptions on error!
4277 sub vm_deviceunplug {
4278     my ($vmid, $conf, $deviceid) = @_;
4279
4280     my $devices_list = vm_devices_list($vmid);
4281     return 1 if !defined($devices_list->{$deviceid});
4282
4283     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4284     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4285
4286     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4287         qemu_devicedel($vmid, $deviceid);
4288     } elsif ($deviceid =~ m/^usb\d+$/) {
4289         die "usb hotplug currently not reliable\n";
4290         # when unplugging usb devices this way, there may be remaining usb
4291         # controllers/hubs so we disable it for now
4292         #qemu_devicedel($vmid, $deviceid);
4293         #qemu_devicedelverify($vmid, $deviceid);
4294     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4295         my $device = parse_drive($deviceid, $conf->{$deviceid});
4296
4297         qemu_devicedel($vmid, $deviceid);
4298         qemu_devicedelverify($vmid, $deviceid);
4299         qemu_drivedel($vmid, $deviceid);
4300         qemu_iothread_del($vmid, $deviceid, $device);
4301     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4302         qemu_devicedel($vmid, $deviceid);
4303         qemu_devicedelverify($vmid, $deviceid);
4304     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4305         my $device = parse_drive($deviceid, $conf->{$deviceid});
4306
4307         qemu_devicedel($vmid, $deviceid);
4308         qemu_devicedelverify($vmid, $deviceid);
4309         qemu_drivedel($vmid, $deviceid);
4310         qemu_deletescsihw($conf, $vmid, $deviceid);
4311
4312         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4313             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4314     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4315         qemu_devicedel($vmid, $deviceid);
4316         qemu_devicedelverify($vmid, $deviceid);
4317         qemu_netdevdel($vmid, $deviceid);
4318     } else {
4319         die "can't unplug device '$deviceid'\n";
4320     }
4321
4322     return 1;
4323 }
4324
4325 sub qemu_deviceadd {
4326     my ($vmid, $devicefull) = @_;
4327
4328     $devicefull = "driver=".$devicefull;
4329     my %options =  split(/[=,]/, $devicefull);
4330
4331     mon_cmd($vmid, "device_add" , %options);
4332 }
4333
4334 sub qemu_devicedel {
4335     my ($vmid, $deviceid) = @_;
4336
4337     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4338 }
4339
4340 sub qemu_iothread_add {
4341     my ($vmid, $deviceid, $device) = @_;
4342
4343     if ($device->{iothread}) {
4344         my $iothreads = vm_iothreads_list($vmid);
4345         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4346     }
4347 }
4348
4349 sub qemu_iothread_del {
4350     my ($vmid, $deviceid, $device) = @_;
4351
4352     if ($device->{iothread}) {
4353         my $iothreads = vm_iothreads_list($vmid);
4354         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4355     }
4356 }
4357
4358 sub qemu_objectadd {
4359     my ($vmid, $objectid, $qomtype) = @_;
4360
4361     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4362
4363     return 1;
4364 }
4365
4366 sub qemu_objectdel {
4367     my ($vmid, $objectid) = @_;
4368
4369     mon_cmd($vmid, "object-del", id => $objectid);
4370
4371     return 1;
4372 }
4373
4374 sub qemu_driveadd {
4375     my ($storecfg, $vmid, $device) = @_;
4376
4377     my $kvmver = get_running_qemu_version($vmid);
4378     my $io_uring = min_version($kvmver, 6, 0);
4379     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4380     $drive =~ s/\\/\\\\/g;
4381     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4382
4383     # If the command succeeds qemu prints: "OK"
4384     return 1 if $ret =~ m/OK/s;
4385
4386     die "adding drive failed: $ret\n";
4387 }
4388
4389 sub qemu_drivedel {
4390     my ($vmid, $deviceid) = @_;
4391
4392     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4393     $ret =~ s/^\s+//;
4394
4395     return 1 if $ret eq "";
4396
4397     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4398     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4399
4400     die "deleting drive $deviceid failed : $ret\n";
4401 }
4402
4403 sub qemu_deviceaddverify {
4404     my ($vmid, $deviceid) = @_;
4405
4406     for (my $i = 0; $i <= 5; $i++) {
4407          my $devices_list = vm_devices_list($vmid);
4408          return 1 if defined($devices_list->{$deviceid});
4409          sleep 1;
4410     }
4411
4412     die "error on hotplug device '$deviceid'\n";
4413 }
4414
4415
4416 sub qemu_devicedelverify {
4417     my ($vmid, $deviceid) = @_;
4418
4419     # need to verify that the device is correctly removed as device_del
4420     # is async and empty return is not reliable
4421
4422     for (my $i = 0; $i <= 5; $i++) {
4423          my $devices_list = vm_devices_list($vmid);
4424          return 1 if !defined($devices_list->{$deviceid});
4425          sleep 1;
4426     }
4427
4428     die "error on hot-unplugging device '$deviceid'\n";
4429 }
4430
4431 sub qemu_findorcreatescsihw {
4432     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4433
4434     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4435
4436     my $scsihwid="$controller_prefix$controller";
4437     my $devices_list = vm_devices_list($vmid);
4438
4439     if (!defined($devices_list->{$scsihwid})) {
4440         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4441     }
4442
4443     return 1;
4444 }
4445
4446 sub qemu_deletescsihw {
4447     my ($conf, $vmid, $opt) = @_;
4448
4449     my $device = parse_drive($opt, $conf->{$opt});
4450
4451     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4452         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4453         return 1;
4454     }
4455
4456     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4457
4458     my $devices_list = vm_devices_list($vmid);
4459     foreach my $opt (keys %{$devices_list}) {
4460         if (is_valid_drivename($opt)) {
4461             my $drive = parse_drive($opt, $conf->{$opt});
4462             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4463                 return 1;
4464             }
4465         }
4466     }
4467
4468     my $scsihwid="scsihw$controller";
4469
4470     vm_deviceunplug($vmid, $conf, $scsihwid);
4471
4472     return 1;
4473 }
4474
4475 sub qemu_add_pci_bridge {
4476     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4477
4478     my $bridges = {};
4479
4480     my $bridgeid;
4481
4482     print_pci_addr($device, $bridges, $arch, $machine_type);
4483
4484     while (my ($k, $v) = each %$bridges) {
4485         $bridgeid = $k;
4486     }
4487     return 1 if !defined($bridgeid) || $bridgeid < 1;
4488
4489     my $bridge = "pci.$bridgeid";
4490     my $devices_list = vm_devices_list($vmid);
4491
4492     if (!defined($devices_list->{$bridge})) {
4493         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4494     }
4495
4496     return 1;
4497 }
4498
4499 sub qemu_set_link_status {
4500     my ($vmid, $device, $up) = @_;
4501
4502     mon_cmd($vmid, "set_link", name => $device,
4503                up => $up ? JSON::true : JSON::false);
4504 }
4505
4506 sub qemu_netdevadd {
4507     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4508
4509     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4510     my %options =  split(/[=,]/, $netdev);
4511
4512     if (defined(my $vhost = $options{vhost})) {
4513         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4514     }
4515
4516     if (defined(my $queues = $options{queues})) {
4517         $options{queues} = $queues + 0;
4518     }
4519
4520     mon_cmd($vmid, "netdev_add",  %options);
4521     return 1;
4522 }
4523
4524 sub qemu_netdevdel {
4525     my ($vmid, $deviceid) = @_;
4526
4527     mon_cmd($vmid, "netdev_del", id => $deviceid);
4528 }
4529
4530 sub qemu_usb_hotplug {
4531     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4532
4533     return if !$device;
4534
4535     # remove the old one first
4536     vm_deviceunplug($vmid, $conf, $deviceid);
4537
4538     # check if xhci controller is necessary and available
4539     if ($device->{usb3}) {
4540
4541         my $devicelist = vm_devices_list($vmid);
4542
4543         if (!$devicelist->{xhci}) {
4544             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4545             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4546         }
4547     }
4548     my $d = parse_usb_device($device->{host});
4549     $d->{usb3} = $device->{usb3};
4550
4551     # add the new one
4552     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4553 }
4554
4555 sub qemu_cpu_hotplug {
4556     my ($vmid, $conf, $vcpus) = @_;
4557
4558     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4559
4560     my $sockets = 1;
4561     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4562     $sockets = $conf->{sockets} if  $conf->{sockets};
4563     my $cores = $conf->{cores} || 1;
4564     my $maxcpus = $sockets * $cores;
4565
4566     $vcpus = $maxcpus if !$vcpus;
4567
4568     die "you can't add more vcpus than maxcpus\n"
4569         if $vcpus > $maxcpus;
4570
4571     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4572
4573     if ($vcpus < $currentvcpus) {
4574
4575         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4576
4577             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4578                 qemu_devicedel($vmid, "cpu$i");
4579                 my $retry = 0;
4580                 my $currentrunningvcpus = undef;
4581                 while (1) {
4582                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4583                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4584                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4585                     $retry++;
4586                     sleep 1;
4587                 }
4588                 #update conf after each succesfull cpu unplug
4589                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4590                 PVE::QemuConfig->write_config($vmid, $conf);
4591             }
4592         } else {
4593             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4594         }
4595
4596         return;
4597     }
4598
4599     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600     die "vcpus in running vm does not match its configuration\n"
4601         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4602
4603     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4604
4605         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4606             my $cpustr = print_cpu_device($conf, $i);
4607             qemu_deviceadd($vmid, $cpustr);
4608
4609             my $retry = 0;
4610             my $currentrunningvcpus = undef;
4611             while (1) {
4612                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4613                 last if scalar(@{$currentrunningvcpus}) == $i;
4614                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4615                 sleep 1;
4616                 $retry++;
4617             }
4618             #update conf after each succesfull cpu hotplug
4619             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4620             PVE::QemuConfig->write_config($vmid, $conf);
4621         }
4622     } else {
4623
4624         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4625             mon_cmd($vmid, "cpu-add", id => int($i));
4626         }
4627     }
4628 }
4629
4630 sub qemu_block_set_io_throttle {
4631     my ($vmid, $deviceid,
4632         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4633         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4634         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4635         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4636
4637     return if !check_running($vmid) ;
4638
4639     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4640         bps => int($bps),
4641         bps_rd => int($bps_rd),
4642         bps_wr => int($bps_wr),
4643         iops => int($iops),
4644         iops_rd => int($iops_rd),
4645         iops_wr => int($iops_wr),
4646         bps_max => int($bps_max),
4647         bps_rd_max => int($bps_rd_max),
4648         bps_wr_max => int($bps_wr_max),
4649         iops_max => int($iops_max),
4650         iops_rd_max => int($iops_rd_max),
4651         iops_wr_max => int($iops_wr_max),
4652         bps_max_length => int($bps_max_length),
4653         bps_rd_max_length => int($bps_rd_max_length),
4654         bps_wr_max_length => int($bps_wr_max_length),
4655         iops_max_length => int($iops_max_length),
4656         iops_rd_max_length => int($iops_rd_max_length),
4657         iops_wr_max_length => int($iops_wr_max_length),
4658     );
4659
4660 }
4661
4662 sub qemu_block_resize {
4663     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4664
4665     my $running = check_running($vmid);
4666
4667     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4668
4669     return if !$running;
4670
4671     my $padding = (1024 - $size % 1024) % 1024;
4672     $size = $size + $padding;
4673
4674     mon_cmd(
4675         $vmid,
4676         "block_resize",
4677         device => $deviceid,
4678         size => int($size),
4679         timeout => 60,
4680     );
4681 }
4682
4683 sub qemu_volume_snapshot {
4684     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4685
4686     my $running = check_running($vmid);
4687
4688     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4689         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4690     } else {
4691         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4692     }
4693 }
4694
4695 sub qemu_volume_snapshot_delete {
4696     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4697
4698     my $running = check_running($vmid);
4699
4700     if($running) {
4701
4702         $running = undef;
4703         my $conf = PVE::QemuConfig->load_config($vmid);
4704         PVE::QemuConfig->foreach_volume($conf, sub {
4705             my ($ds, $drive) = @_;
4706             $running = 1 if $drive->{file} eq $volid;
4707         });
4708     }
4709
4710     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4711         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4712     } else {
4713         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4714     }
4715 }
4716
4717 sub set_migration_caps {
4718     my ($vmid, $savevm) = @_;
4719
4720     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4721
4722     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4723     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4724
4725     my $cap_ref = [];
4726
4727     my $enabled_cap = {
4728         "auto-converge" => 1,
4729         "xbzrle" => 1,
4730         "x-rdma-pin-all" => 0,
4731         "zero-blocks" => 0,
4732         "compress" => 0,
4733         "dirty-bitmaps" => $dirty_bitmaps,
4734     };
4735
4736     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4737
4738     for my $supported_capability (@$supported_capabilities) {
4739         push @$cap_ref, {
4740             capability => $supported_capability->{capability},
4741             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4742         };
4743     }
4744
4745     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4746 }
4747
4748 sub foreach_volid {
4749     my ($conf, $func, @param) = @_;
4750
4751     my $volhash = {};
4752
4753     my $test_volid = sub {
4754         my ($key, $drive, $snapname) = @_;
4755
4756         my $volid = $drive->{file};
4757         return if !$volid;
4758
4759         $volhash->{$volid}->{cdrom} //= 1;
4760         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4761
4762         my $replicate = $drive->{replicate} // 1;
4763         $volhash->{$volid}->{replicate} //= 0;
4764         $volhash->{$volid}->{replicate} = 1 if $replicate;
4765
4766         $volhash->{$volid}->{shared} //= 0;
4767         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4768
4769         $volhash->{$volid}->{referenced_in_config} //= 0;
4770         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4771
4772         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4773             if defined($snapname);
4774
4775         my $size = $drive->{size};
4776         $volhash->{$volid}->{size} //= $size if $size;
4777
4778         $volhash->{$volid}->{is_vmstate} //= 0;
4779         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4780
4781         $volhash->{$volid}->{is_tpmstate} //= 0;
4782         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4783
4784         $volhash->{$volid}->{is_unused} //= 0;
4785         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4786
4787         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4788     };
4789
4790     my $include_opts = {
4791         extra_keys => ['vmstate'],
4792         include_unused => 1,
4793     };
4794
4795     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4796     foreach my $snapname (keys %{$conf->{snapshots}}) {
4797         my $snap = $conf->{snapshots}->{$snapname};
4798         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4799     }
4800
4801     foreach my $volid (keys %$volhash) {
4802         &$func($volid, $volhash->{$volid}, @param);
4803     }
4804 }
4805
4806 my $fast_plug_option = {
4807     'lock' => 1,
4808     'name' => 1,
4809     'onboot' => 1,
4810     'shares' => 1,
4811     'startup' => 1,
4812     'description' => 1,
4813     'protection' => 1,
4814     'vmstatestorage' => 1,
4815     'hookscript' => 1,
4816     'tags' => 1,
4817 };
4818
4819 for my $opt (keys %$confdesc_cloudinit) {
4820     $fast_plug_option->{$opt} = 1;
4821 };
4822
4823 # hotplug changes in [PENDING]
4824 # $selection hash can be used to only apply specified options, for
4825 # example: { cores => 1 } (only apply changed 'cores')
4826 # $errors ref is used to return error messages
4827 sub vmconfig_hotplug_pending {
4828     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4829
4830     my $defaults = load_defaults();
4831     my $arch = get_vm_arch($conf);
4832     my $machine_type = get_vm_machine($conf, undef, $arch);
4833
4834     # commit values which do not have any impact on running VM first
4835     # Note: those option cannot raise errors, we we do not care about
4836     # $selection and always apply them.
4837
4838     my $add_error = sub {
4839         my ($opt, $msg) = @_;
4840         $errors->{$opt} = "hotplug problem - $msg";
4841     };
4842
4843     my $changes = 0;
4844     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4845         if ($fast_plug_option->{$opt}) {
4846             $conf->{$opt} = $conf->{pending}->{$opt};
4847             delete $conf->{pending}->{$opt};
4848             $changes = 1;
4849         }
4850     }
4851
4852     if ($changes) {
4853         PVE::QemuConfig->write_config($vmid, $conf);
4854     }
4855
4856     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4857
4858     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4859     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4860     foreach my $opt (sort keys %$pending_delete_hash) {
4861         next if $selection && !$selection->{$opt};
4862         my $force = $pending_delete_hash->{$opt}->{force};
4863         eval {
4864             if ($opt eq 'hotplug') {
4865                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4866             } elsif ($opt eq 'tablet') {
4867                 die "skip\n" if !$hotplug_features->{usb};
4868                 if ($defaults->{tablet}) {
4869                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4870                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4871                         if $arch eq 'aarch64';
4872                 } else {
4873                     vm_deviceunplug($vmid, $conf, 'tablet');
4874                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4875                 }
4876             } elsif ($opt =~ m/^usb\d+/) {
4877                 die "skip\n";
4878                 # since we cannot reliably hot unplug usb devices we are disabling it
4879                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4880                 #vm_deviceunplug($vmid, $conf, $opt);
4881             } elsif ($opt eq 'vcpus') {
4882                 die "skip\n" if !$hotplug_features->{cpu};
4883                 qemu_cpu_hotplug($vmid, $conf, undef);
4884             } elsif ($opt eq 'balloon') {
4885                 # enable balloon device is not hotpluggable
4886                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4887                 # here we reset the ballooning value to memory
4888                 my $balloon = $conf->{memory} || $defaults->{memory};
4889                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4890             } elsif ($fast_plug_option->{$opt}) {
4891                 # do nothing
4892             } elsif ($opt =~ m/^net(\d+)$/) {
4893                 die "skip\n" if !$hotplug_features->{network};
4894                 vm_deviceunplug($vmid, $conf, $opt);
4895             } elsif (is_valid_drivename($opt)) {
4896                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4897                 vm_deviceunplug($vmid, $conf, $opt);
4898                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4899             } elsif ($opt =~ m/^memory$/) {
4900                 die "skip\n" if !$hotplug_features->{memory};
4901                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4902             } elsif ($opt eq 'cpuunits') {
4903                 $cgroup->change_cpu_shares(undef);
4904             } elsif ($opt eq 'cpulimit') {
4905                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4906             } else {
4907                 die "skip\n";
4908             }
4909         };
4910         if (my $err = $@) {
4911             &$add_error($opt, $err) if $err ne "skip\n";
4912         } else {
4913             delete $conf->{$opt};
4914             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4915         }
4916     }
4917
4918     foreach my $opt (keys %{$conf->{pending}}) {
4919         next if $selection && !$selection->{$opt};
4920         my $value = $conf->{pending}->{$opt};
4921         eval {
4922             if ($opt eq 'hotplug') {
4923                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4924             } elsif ($opt eq 'tablet') {
4925                 die "skip\n" if !$hotplug_features->{usb};
4926                 if ($value == 1) {
4927                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4928                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4929                         if $arch eq 'aarch64';
4930                 } elsif ($value == 0) {
4931                     vm_deviceunplug($vmid, $conf, 'tablet');
4932                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4933                 }
4934             } elsif ($opt =~ m/^usb\d+$/) {
4935                 die "skip\n";
4936                 # since we cannot reliably hot unplug usb devices we disable it for now
4937                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4938                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4939                 #die "skip\n" if !$d;
4940                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4941             } elsif ($opt eq 'vcpus') {
4942                 die "skip\n" if !$hotplug_features->{cpu};
4943                 qemu_cpu_hotplug($vmid, $conf, $value);
4944             } elsif ($opt eq 'balloon') {
4945                 # enable/disable balloning device is not hotpluggable
4946                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4947                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4948                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4949
4950                 # allow manual ballooning if shares is set to zero
4951                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4952                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4953                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4954                 }
4955             } elsif ($opt =~ m/^net(\d+)$/) {
4956                 # some changes can be done without hotplug
4957                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4958                                     $vmid, $opt, $value, $arch, $machine_type);
4959             } elsif (is_valid_drivename($opt)) {
4960                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4961                 # some changes can be done without hotplug
4962                 my $drive = parse_drive($opt, $value);
4963                 if (drive_is_cloudinit($drive)) {
4964                     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4965                 }
4966                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4967                                      $vmid, $opt, $value, $arch, $machine_type);
4968             } elsif ($opt =~ m/^memory$/) { #dimms
4969                 die "skip\n" if !$hotplug_features->{memory};
4970                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4971             } elsif ($opt eq 'cpuunits') {
4972                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
4973                 $cgroup->change_cpu_shares($new_cpuunits);
4974             } elsif ($opt eq 'cpulimit') {
4975                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4976                 $cgroup->change_cpu_quota($cpulimit, 100000);
4977             } elsif ($opt eq 'agent') {
4978                 vmconfig_update_agent($conf, $opt, $value);
4979             } else {
4980                 die "skip\n";  # skip non-hot-pluggable options
4981             }
4982         };
4983         if (my $err = $@) {
4984             &$add_error($opt, $err) if $err ne "skip\n";
4985         } else {
4986             $conf->{$opt} = $value;
4987             delete $conf->{pending}->{$opt};
4988         }
4989     }
4990
4991     PVE::QemuConfig->write_config($vmid, $conf);
4992 }
4993
4994 sub try_deallocate_drive {
4995     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4996
4997     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4998         my $volid = $drive->{file};
4999         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5000             my $sid = PVE::Storage::parse_volume_id($volid);
5001             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5002
5003             # check if the disk is really unused
5004             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5005                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5006             PVE::Storage::vdisk_free($storecfg, $volid);
5007             return 1;
5008         } else {
5009             # If vm is not owner of this disk remove from config
5010             return 1;
5011         }
5012     }
5013
5014     return;
5015 }
5016
5017 sub vmconfig_delete_or_detach_drive {
5018     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5019
5020     my $drive = parse_drive($opt, $conf->{$opt});
5021
5022     my $rpcenv = PVE::RPCEnvironment::get();
5023     my $authuser = $rpcenv->get_user();
5024
5025     if ($force) {
5026         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5027         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5028     } else {
5029         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5030     }
5031 }
5032
5033
5034
5035 sub vmconfig_apply_pending {
5036     my ($vmid, $conf, $storecfg, $errors) = @_;
5037
5038     return if !scalar(keys %{$conf->{pending}});
5039
5040     my $add_apply_error = sub {
5041         my ($opt, $msg) = @_;
5042         my $err_msg = "unable to apply pending change $opt : $msg";
5043         $errors->{$opt} = $err_msg;
5044         warn $err_msg;
5045     };
5046
5047     # cold plug
5048
5049     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5050     foreach my $opt (sort keys %$pending_delete_hash) {
5051         my $force = $pending_delete_hash->{$opt}->{force};
5052         eval {
5053             if ($opt =~ m/^unused/) {
5054                 die "internal error";
5055             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5056                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5057             }
5058         };
5059         if (my $err = $@) {
5060             $add_apply_error->($opt, $err);
5061         } else {
5062             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5063             delete $conf->{$opt};
5064         }
5065     }
5066
5067     PVE::QemuConfig->cleanup_pending($conf);
5068
5069     my $generate_cloudnit = undef;
5070
5071     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5072         next if $opt eq 'delete'; # just to be sure
5073         eval {
5074             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5075                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5076             }
5077         };
5078         if (my $err = $@) {
5079             $add_apply_error->($opt, $err);
5080         } else {
5081
5082             if (is_valid_drivename($opt)) {
5083                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5084                 $generate_cloudnit = 1 if drive_is_cloudinit($drive);
5085             }
5086
5087             $conf->{$opt} = delete $conf->{pending}->{$opt};
5088         }
5089     }
5090
5091     # write all changes at once to avoid unnecessary i/o
5092     PVE::QemuConfig->write_config($vmid, $conf);
5093     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if $generate_cloudnit;
5094 }
5095
5096 sub vmconfig_update_net {
5097     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5098
5099     my $newnet = parse_net($value);
5100
5101     if ($conf->{$opt}) {
5102         my $oldnet = parse_net($conf->{$opt});
5103
5104         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5105             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5106             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5107             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5108
5109             # for non online change, we try to hot-unplug
5110             die "skip\n" if !$hotplug;
5111             vm_deviceunplug($vmid, $conf, $opt);
5112         } else {
5113
5114             die "internal error" if $opt !~ m/net(\d+)/;
5115             my $iface = "tap${vmid}i$1";
5116
5117             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5118                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5119                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5120                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5121                 PVE::Network::tap_unplug($iface);
5122
5123                 if ($have_sdn) {
5124                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5125                 } else {
5126                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5127                 }
5128             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5129                 # Rate can be applied on its own but any change above needs to
5130                 # include the rate in tap_plug since OVS resets everything.
5131                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5132             }
5133
5134             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5135                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5136             }
5137
5138             return 1;
5139         }
5140     }
5141
5142     if ($hotplug) {
5143         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5144     } else {
5145         die "skip\n";
5146     }
5147 }
5148
5149 sub vmconfig_update_agent {
5150     my ($conf, $opt, $value) = @_;
5151
5152     die "skip\n" if !$conf->{$opt};
5153
5154     my $hotplug_options = { fstrim_cloned_disks => 1 };
5155
5156     my $old_agent = parse_guest_agent($conf);
5157     my $agent = parse_guest_agent({$opt => $value});
5158
5159     for my $option (keys %$agent) { # added/changed options
5160         next if defined($hotplug_options->{$option});
5161         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5162     }
5163
5164     for my $option (keys %$old_agent) { # removed options
5165         next if defined($hotplug_options->{$option});
5166         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5167     }
5168
5169     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5170 }
5171
5172 sub vmconfig_update_disk {
5173     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5174
5175     my $drive = parse_drive($opt, $value);
5176
5177     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5178         my $media = $drive->{media} || 'disk';
5179         my $oldmedia = $old_drive->{media} || 'disk';
5180         die "unable to change media type\n" if $media ne $oldmedia;
5181
5182         if (!drive_is_cdrom($old_drive)) {
5183
5184             if ($drive->{file} ne $old_drive->{file}) {
5185
5186                 die "skip\n" if !$hotplug;
5187
5188                 # unplug and register as unused
5189                 vm_deviceunplug($vmid, $conf, $opt);
5190                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5191
5192             } else {
5193                 # update existing disk
5194
5195                 # skip non hotpluggable value
5196                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5197                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5198                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5199                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5200                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5201                     die "skip\n";
5202                 }
5203
5204                 # apply throttle
5205                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5206                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5207                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5208                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5209                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5210                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5211                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5212                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5213                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5214                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5215                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5216                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5217                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5218                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5219                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5220                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5221                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5222                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5223
5224                     qemu_block_set_io_throttle(
5225                         $vmid,"drive-$opt",
5226                         ($drive->{mbps} || 0)*1024*1024,
5227                         ($drive->{mbps_rd} || 0)*1024*1024,
5228                         ($drive->{mbps_wr} || 0)*1024*1024,
5229                          $drive->{iops} || 0,
5230                          $drive->{iops_rd} || 0,
5231                          $drive->{iops_wr} || 0,
5232                         ($drive->{mbps_max} || 0)*1024*1024,
5233                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5234                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5235                          $drive->{iops_max} || 0,
5236                          $drive->{iops_rd_max} || 0,
5237                          $drive->{iops_wr_max} || 0,
5238                          $drive->{bps_max_length} || 1,
5239                          $drive->{bps_rd_max_length} || 1,
5240                          $drive->{bps_wr_max_length} || 1,
5241                          $drive->{iops_max_length} || 1,
5242                          $drive->{iops_rd_max_length} || 1,
5243                          $drive->{iops_wr_max_length} || 1,
5244                     );
5245
5246                 }
5247
5248                 return 1;
5249             }
5250
5251         } else { # cdrom
5252
5253             if ($drive->{file} eq 'none') {
5254                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5255                 if (drive_is_cloudinit($old_drive)) {
5256                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5257                 }
5258             } else {
5259                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5260
5261                 # force eject if locked
5262                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5263
5264                 if ($path) {
5265                     mon_cmd($vmid, "blockdev-change-medium",
5266                         id => "$opt", filename => "$path");
5267                 }
5268             }
5269
5270             return 1;
5271         }
5272     }
5273
5274     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5275     # hotplug new disks
5276     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5277     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5278 }
5279
5280 sub vmconfig_update_cloudinit_drive {
5281     my ($storecfg, $conf, $vmid) = @_;
5282
5283     my $cloudinit_ds = undef;
5284     my $cloudinit_drive = undef;
5285
5286     PVE::QemuConfig->foreach_volume($conf, sub {
5287         my ($ds, $drive) = @_;
5288         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5289             $cloudinit_ds = $ds;
5290             $cloudinit_drive = $drive;
5291         }
5292     });
5293
5294     return if !$cloudinit_drive;
5295
5296     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5297     my $running = PVE::QemuServer::check_running($vmid);
5298
5299     if ($running) {
5300         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5301         if ($path) {
5302             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5303             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5304         }
5305     }
5306 }
5307
5308 # called in locked context by incoming migration
5309 sub vm_migrate_get_nbd_disks {
5310     my ($storecfg, $conf, $replicated_volumes) = @_;
5311
5312     my $local_volumes = {};
5313     PVE::QemuConfig->foreach_volume($conf, sub {
5314         my ($ds, $drive) = @_;
5315
5316         return if drive_is_cdrom($drive);
5317         return if $ds eq 'tpmstate0';
5318
5319         my $volid = $drive->{file};
5320
5321         return if !$volid;
5322
5323         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5324
5325         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5326         return if $scfg->{shared};
5327
5328         # replicated disks re-use existing state via bitmap
5329         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5330         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5331     });
5332     return $local_volumes;
5333 }
5334
5335 # called in locked context by incoming migration
5336 sub vm_migrate_alloc_nbd_disks {
5337     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5338
5339     my $nbd = {};
5340     foreach my $opt (sort keys %$source_volumes) {
5341         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5342
5343         if ($use_existing) {
5344             $nbd->{$opt}->{drivestr} = print_drive($drive);
5345             $nbd->{$opt}->{volid} = $volid;
5346             $nbd->{$opt}->{replicated} = 1;
5347             next;
5348         }
5349
5350         # storage mapping + volname = regular migration
5351         # storage mapping + format = remote migration
5352         # order of precedence, filtered by whether storage supports it:
5353         # 1. explicit requested format
5354         # 2. format of current volume
5355         # 3. default format of storage
5356         if (!$storagemap->{identity}) {
5357             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5358             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5359             if (!$format || !grep { $format eq $_ } @$validFormats) {
5360                 if ($volname) {
5361                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5362                     my $fileFormat = qemu_img_format($scfg, $volname);
5363                     $format = $fileFormat
5364                         if grep { $fileFormat eq $_ } @$validFormats;
5365                 }
5366                 $format //= $defFormat;
5367             }
5368         } else {
5369             # can't happen for remote migration, so $volname is always defined
5370             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5371             $format = qemu_img_format($scfg, $volname);
5372         }
5373
5374         my $size = $drive->{size} / 1024;
5375         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5376         my $newdrive = $drive;
5377         $newdrive->{format} = $format;
5378         $newdrive->{file} = $newvolid;
5379         my $drivestr = print_drive($newdrive);
5380         $nbd->{$opt}->{drivestr} = $drivestr;
5381         $nbd->{$opt}->{volid} = $newvolid;
5382     }
5383
5384     return $nbd;
5385 }
5386
5387 # see vm_start_nolock for parameters, additionally:
5388 # migrate_opts:
5389 #   storagemap = parsed storage map for allocating NBD disks
5390 sub vm_start {
5391     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5392
5393     return PVE::QemuConfig->lock_config($vmid, sub {
5394         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5395
5396         die "you can't start a vm if it's a template\n"
5397             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5398
5399         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5400         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5401
5402         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5403
5404         if ($has_backup_lock && $running) {
5405             # a backup is currently running, attempt to start the guest in the
5406             # existing QEMU instance
5407             return vm_resume($vmid);
5408         }
5409
5410         PVE::QemuConfig->check_lock($conf)
5411             if !($params->{skiplock} || $has_suspended_lock);
5412
5413         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5414
5415         die "VM $vmid already running\n" if $running;
5416
5417         if (my $storagemap = $migrate_opts->{storagemap}) {
5418             my $replicated = $migrate_opts->{replicated_volumes};
5419             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5420             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5421
5422             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5423                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5424             }
5425         }
5426
5427         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5428     });
5429 }
5430
5431
5432 # params:
5433 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5434 #   skiplock => 0/1, skip checking for config lock
5435 #   skiptemplate => 0/1, skip checking whether VM is template
5436 #   forcemachine => to force Qemu machine (rollback/migration)
5437 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5438 #   timeout => in seconds
5439 #   paused => start VM in paused state (backup)
5440 #   resume => resume from hibernation
5441 #   pbs-backing => {
5442 #      sata0 => {
5443 #         repository
5444 #         snapshot
5445 #         keyfile
5446 #         archive
5447 #      },
5448 #      virtio2 => ...
5449 #   }
5450 # migrate_opts:
5451 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5452 #   migratedfrom => source node
5453 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5454 #   network => CIDR of migration network
5455 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5456 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5457 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5458 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5459 #       contained in config
5460 sub vm_start_nolock {
5461     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5462
5463     my $statefile = $params->{statefile};
5464     my $resume = $params->{resume};
5465
5466     my $migratedfrom = $migrate_opts->{migratedfrom};
5467     my $migration_type = $migrate_opts->{type};
5468
5469     my $res = {};
5470
5471     # clean up leftover reboot request files
5472     eval { clear_reboot_request($vmid); };
5473     warn $@ if $@;
5474
5475     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5476         vmconfig_apply_pending($vmid, $conf, $storecfg);
5477         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5478     }
5479
5480     # don't regenerate the ISO if the VM is started as part of a live migration
5481     # this way we can reuse the old ISO with the correct config
5482     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5483
5484     # override offline migrated volumes, conf is out of date still
5485     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5486         for my $key (sort keys $offline_volumes->%*) {
5487             my $parsed = parse_drive($key, $conf->{$key});
5488             $parsed->{file} = $offline_volumes->{$key};
5489             $conf->{$key} = print_drive($parsed);
5490         }
5491     }
5492
5493     my $defaults = load_defaults();
5494
5495     # set environment variable useful inside network script
5496     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5497
5498     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5499
5500     my $forcemachine = $params->{forcemachine};
5501     my $forcecpu = $params->{forcecpu};
5502     if ($resume) {
5503         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5504         $forcemachine = $conf->{runningmachine};
5505         $forcecpu = $conf->{runningcpu};
5506         print "Resuming suspended VM\n";
5507     }
5508
5509     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5510         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5511
5512     my $migration_ip;
5513     my $get_migration_ip = sub {
5514         my ($nodename) = @_;
5515
5516         return $migration_ip if defined($migration_ip);
5517
5518         my $cidr = $migrate_opts->{network};
5519
5520         if (!defined($cidr)) {
5521             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5522             $cidr = $dc_conf->{migration}->{network};
5523         }
5524
5525         if (defined($cidr)) {
5526             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5527
5528             die "could not get IP: no address configured on local " .
5529                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5530
5531             die "could not get IP: multiple addresses configured on local " .
5532                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5533
5534             $migration_ip = @$ips[0];
5535         }
5536
5537         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5538             if !defined($migration_ip);
5539
5540         return $migration_ip;
5541     };
5542
5543     my $migrate_uri;
5544     if ($statefile) {
5545         if ($statefile eq 'tcp') {
5546             my $localip = "localhost";
5547             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5548             my $nodename = nodename();
5549
5550             if (!defined($migration_type)) {
5551                 if (defined($datacenterconf->{migration}->{type})) {
5552                     $migration_type = $datacenterconf->{migration}->{type};
5553                 } else {
5554                     $migration_type = 'secure';
5555                 }
5556             }
5557
5558             if ($migration_type eq 'insecure') {
5559                 $localip = $get_migration_ip->($nodename);
5560                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5561             }
5562
5563             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5564             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5565             $migrate_uri = "tcp:${localip}:${migrate_port}";
5566             push @$cmd, '-incoming', $migrate_uri;
5567             push @$cmd, '-S';
5568
5569         } elsif ($statefile eq 'unix') {
5570             # should be default for secure migrations as a ssh TCP forward
5571             # tunnel is not deterministic reliable ready and fails regurarly
5572             # to set up in time, so use UNIX socket forwards
5573             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5574             unlink $socket_addr;
5575
5576             $migrate_uri = "unix:$socket_addr";
5577
5578             push @$cmd, '-incoming', $migrate_uri;
5579             push @$cmd, '-S';
5580
5581         } elsif (-e $statefile) {
5582             push @$cmd, '-loadstate', $statefile;
5583         } else {
5584             my $statepath = PVE::Storage::path($storecfg, $statefile);
5585             push @$vollist, $statefile;
5586             push @$cmd, '-loadstate', $statepath;
5587         }
5588     } elsif ($params->{paused}) {
5589         push @$cmd, '-S';
5590     }
5591
5592     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5593
5594     my $pci_devices = {}; # host pci devices
5595     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5596         my $dev = $conf->{"hostpci$i"} or next;
5597         $pci_devices->{$i} = parse_hostpci($dev);
5598     }
5599
5600     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5601     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5602
5603     # map to a flat list of pci ids
5604     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5605
5606     # reserve all PCI IDs before actually doing anything with them
5607     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5608
5609     eval {
5610         my $uuid;
5611         for my $id (sort keys %$pci_devices) {
5612             my $d = $pci_devices->{$id};
5613             for my $dev ($d->{pciid}->@*) {
5614                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5615
5616                 # nvidia grid needs the uuid of the mdev as qemu parameter
5617                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5618                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5619                 }
5620             }
5621         }
5622         push @$cmd, '-uuid', $uuid if defined($uuid);
5623     };
5624     if (my $err = $@) {
5625         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5626         warn $@ if $@;
5627         die $err;
5628     }
5629
5630     PVE::Storage::activate_volumes($storecfg, $vollist);
5631
5632     eval {
5633         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5634     };
5635     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5636     # timeout should be more than enough here...
5637     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5638
5639     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5640
5641     my %run_params = (
5642         timeout => $statefile ? undef : $start_timeout,
5643         umask => 0077,
5644         noerr => 1,
5645     );
5646
5647     # when migrating, prefix QEMU output so other side can pick up any
5648     # errors that might occur and show the user
5649     if ($migratedfrom) {
5650         $run_params{quiet} = 1;
5651         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5652     }
5653
5654     my %systemd_properties = (
5655         Slice => 'qemu.slice',
5656         KillMode => 'process',
5657         SendSIGKILL => 0,
5658         TimeoutStopUSec => ULONG_MAX, # infinity
5659     );
5660
5661     if (PVE::CGroup::cgroup_mode() == 2) {
5662         $systemd_properties{CPUWeight} = $cpuunits;
5663     } else {
5664         $systemd_properties{CPUShares} = $cpuunits;
5665     }
5666
5667     if (my $cpulimit = $conf->{cpulimit}) {
5668         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5669     }
5670     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5671
5672     my $run_qemu = sub {
5673         PVE::Tools::run_fork sub {
5674             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5675
5676             my $tpmpid;
5677             if (my $tpm = $conf->{tpmstate0}) {
5678                 # start the TPM emulator so QEMU can connect on start
5679                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5680             }
5681
5682             my $exitcode = run_command($cmd, %run_params);
5683             if ($exitcode) {
5684                 if ($tpmpid) {
5685                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5686                     kill 'TERM', $tpmpid;
5687                 }
5688                 die "QEMU exited with code $exitcode\n";
5689             }
5690         };
5691     };
5692
5693     if ($conf->{hugepages}) {
5694
5695         my $code = sub {
5696             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5697             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5698
5699             PVE::QemuServer::Memory::hugepages_mount();
5700             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5701
5702             eval { $run_qemu->() };
5703             if (my $err = $@) {
5704                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5705                     if !$conf->{keephugepages};
5706                 die $err;
5707             }
5708
5709             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5710                 if !$conf->{keephugepages};
5711         };
5712         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5713
5714     } else {
5715         eval { $run_qemu->() };
5716     }
5717
5718     if (my $err = $@) {
5719         # deactivate volumes if start fails
5720         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5721         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5722
5723         die "start failed: $err";
5724     }
5725
5726     # re-reserve all PCI IDs now that we can know the actual VM PID
5727     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5728     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5729     warn $@ if $@;
5730
5731     print "migration listens on $migrate_uri\n" if $migrate_uri;
5732     $res->{migrate_uri} = $migrate_uri;
5733
5734     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5735         eval { mon_cmd($vmid, "cont"); };
5736         warn $@ if $@;
5737     }
5738
5739     #start nbd server for storage migration
5740     if (my $nbd = $migrate_opts->{nbd}) {
5741         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5742
5743         my $migrate_storage_uri;
5744         # nbd_protocol_version > 0 for unix socket support
5745         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5746             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5747             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5748             $migrate_storage_uri = "nbd:unix:$socket_path";
5749         } else {
5750             my $nodename = nodename();
5751             my $localip = $get_migration_ip->($nodename);
5752             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5753             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5754
5755             mon_cmd($vmid, "nbd-server-start", addr => {
5756                 type => 'inet',
5757                 data => {
5758                     host => "${localip}",
5759                     port => "${storage_migrate_port}",
5760                 },
5761             });
5762             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5763             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5764         }
5765
5766         $res->{migrate_storage_uri} = $migrate_storage_uri;
5767
5768         foreach my $opt (sort keys %$nbd) {
5769             my $drivestr = $nbd->{$opt}->{drivestr};
5770             my $volid = $nbd->{$opt}->{volid};
5771             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5772             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5773             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5774             print "re-using replicated volume: $opt - $volid\n"
5775                 if $nbd->{$opt}->{replicated};
5776
5777             $res->{drives}->{$opt} = $nbd->{$opt};
5778             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5779         }
5780     }
5781
5782     if ($migratedfrom) {
5783         eval {
5784             set_migration_caps($vmid);
5785         };
5786         warn $@ if $@;
5787
5788         if ($spice_port) {
5789             print "spice listens on port $spice_port\n";
5790             $res->{spice_port} = $spice_port;
5791             if ($migrate_opts->{spice_ticket}) {
5792                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5793                     $migrate_opts->{spice_ticket});
5794                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5795             }
5796         }
5797
5798     } else {
5799         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5800             if !$statefile && $conf->{balloon};
5801
5802         foreach my $opt (keys %$conf) {
5803             next if $opt !~  m/^net\d+$/;
5804             my $nicconf = parse_net($conf->{$opt});
5805             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5806         }
5807     }
5808
5809     mon_cmd($vmid, 'qom-set',
5810                 path => "machine/peripheral/balloon0",
5811                 property => "guest-stats-polling-interval",
5812                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5813
5814     if ($resume) {
5815         print "Resumed VM, removing state\n";
5816         if (my $vmstate = $conf->{vmstate}) {
5817             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5818             PVE::Storage::vdisk_free($storecfg, $vmstate);
5819         }
5820         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5821         PVE::QemuConfig->write_config($vmid, $conf);
5822     }
5823
5824     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5825
5826     return $res;
5827 }
5828
5829 sub vm_commandline {
5830     my ($storecfg, $vmid, $snapname) = @_;
5831
5832     my $conf = PVE::QemuConfig->load_config($vmid);
5833
5834     my ($forcemachine, $forcecpu);
5835     if ($snapname) {
5836         my $snapshot = $conf->{snapshots}->{$snapname};
5837         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5838
5839         # check for machine or CPU overrides in snapshot
5840         $forcemachine = $snapshot->{runningmachine};
5841         $forcecpu = $snapshot->{runningcpu};
5842
5843         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5844
5845         $conf = $snapshot;
5846     }
5847
5848     my $defaults = load_defaults();
5849
5850     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5851
5852     return PVE::Tools::cmd2string($cmd);
5853 }
5854
5855 sub vm_reset {
5856     my ($vmid, $skiplock) = @_;
5857
5858     PVE::QemuConfig->lock_config($vmid, sub {
5859
5860         my $conf = PVE::QemuConfig->load_config($vmid);
5861
5862         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5863
5864         mon_cmd($vmid, "system_reset");
5865     });
5866 }
5867
5868 sub get_vm_volumes {
5869     my ($conf) = @_;
5870
5871     my $vollist = [];
5872     foreach_volid($conf, sub {
5873         my ($volid, $attr) = @_;
5874
5875         return if $volid =~ m|^/|;
5876
5877         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5878         return if !$sid;
5879
5880         push @$vollist, $volid;
5881     });
5882
5883     return $vollist;
5884 }
5885
5886 sub vm_stop_cleanup {
5887     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5888
5889     eval {
5890
5891         if (!$keepActive) {
5892             my $vollist = get_vm_volumes($conf);
5893             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5894
5895             if (my $tpmdrive = $conf->{tpmstate0}) {
5896                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5897                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5898                 if ($storeid) {
5899                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5900                 }
5901             }
5902         }
5903
5904         foreach my $ext (qw(mon qmp pid vnc qga)) {
5905             unlink "/var/run/qemu-server/${vmid}.$ext";
5906         }
5907
5908         if ($conf->{ivshmem}) {
5909             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5910             # just delete it for now, VMs which have this already open do not
5911             # are affected, but new VMs will get a separated one. If this
5912             # becomes an issue we either add some sort of ref-counting or just
5913             # add a "don't delete on stop" flag to the ivshmem format.
5914             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5915         }
5916
5917         my $ids = [];
5918         foreach my $key (keys %$conf) {
5919             next if $key !~ m/^hostpci(\d+)$/;
5920             my $hostpciindex = $1;
5921             my $d = parse_hostpci($conf->{$key});
5922             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5923
5924             foreach my $pci (@{$d->{pciid}}) {
5925                 my $pciid = $pci->{id};
5926                 push @$ids, $pci->{id};
5927                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5928             }
5929         }
5930         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5931
5932         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5933     };
5934     warn $@ if $@; # avoid errors - just warn
5935 }
5936
5937 # call only in locked context
5938 sub _do_vm_stop {
5939     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5940
5941     my $pid = check_running($vmid, $nocheck);
5942     return if !$pid;
5943
5944     my $conf;
5945     if (!$nocheck) {
5946         $conf = PVE::QemuConfig->load_config($vmid);
5947         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5948         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5949             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5950             $timeout = $opts->{down} if $opts->{down};
5951         }
5952         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5953     }
5954
5955     eval {
5956         if ($shutdown) {
5957             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5958                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5959             } else {
5960                 mon_cmd($vmid, "system_powerdown");
5961             }
5962         } else {
5963             mon_cmd($vmid, "quit");
5964         }
5965     };
5966     my $err = $@;
5967
5968     if (!$err) {
5969         $timeout = 60 if !defined($timeout);
5970
5971         my $count = 0;
5972         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5973             $count++;
5974             sleep 1;
5975         }
5976
5977         if ($count >= $timeout) {
5978             if ($force) {
5979                 warn "VM still running - terminating now with SIGTERM\n";
5980                 kill 15, $pid;
5981             } else {
5982                 die "VM quit/powerdown failed - got timeout\n";
5983             }
5984         } else {
5985             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5986             return;
5987         }
5988     } else {
5989        if (!check_running($vmid, $nocheck)) {
5990             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5991             return;
5992        }
5993        if ($force) {
5994             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5995             kill 15, $pid;
5996         } else {
5997             die "VM quit/powerdown failed\n";
5998         }
5999     }
6000
6001     # wait again
6002     $timeout = 10;
6003
6004     my $count = 0;
6005     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6006         $count++;
6007         sleep 1;
6008     }
6009
6010     if ($count >= $timeout) {
6011         warn "VM still running - terminating now with SIGKILL\n";
6012         kill 9, $pid;
6013         sleep 1;
6014     }
6015
6016     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6017 }
6018
6019 # Note: use $nocheck to skip tests if VM configuration file exists.
6020 # We need that when migration VMs to other nodes (files already moved)
6021 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6022 sub vm_stop {
6023     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6024
6025     $force = 1 if !defined($force) && !$shutdown;
6026
6027     if ($migratedfrom){
6028         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6029         kill 15, $pid if $pid;
6030         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6031         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6032         return;
6033     }
6034
6035     PVE::QemuConfig->lock_config($vmid, sub {
6036         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6037    });
6038 }
6039
6040 sub vm_reboot {
6041     my ($vmid, $timeout) = @_;
6042
6043     PVE::QemuConfig->lock_config($vmid, sub {
6044         eval {
6045
6046             # only reboot if running, as qmeventd starts it again on a stop event
6047             return if !check_running($vmid);
6048
6049             create_reboot_request($vmid);
6050
6051             my $storecfg = PVE::Storage::config();
6052             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6053
6054         };
6055         if (my $err = $@) {
6056             # avoid that the next normal shutdown will be confused for a reboot
6057             clear_reboot_request($vmid);
6058             die $err;
6059         }
6060    });
6061 }
6062
6063 # note: if using the statestorage parameter, the caller has to check privileges
6064 sub vm_suspend {
6065     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6066
6067     my $conf;
6068     my $path;
6069     my $storecfg;
6070     my $vmstate;
6071
6072     PVE::QemuConfig->lock_config($vmid, sub {
6073
6074         $conf = PVE::QemuConfig->load_config($vmid);
6075
6076         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6077         PVE::QemuConfig->check_lock($conf)
6078             if !($skiplock || $is_backing_up);
6079
6080         die "cannot suspend to disk during backup\n"
6081             if $is_backing_up && $includestate;
6082
6083         if ($includestate) {
6084             $conf->{lock} = 'suspending';
6085             my $date = strftime("%Y-%m-%d", localtime(time()));
6086             $storecfg = PVE::Storage::config();
6087             if (!$statestorage) {
6088                 $statestorage = find_vmstate_storage($conf, $storecfg);
6089                 # check permissions for the storage
6090                 my $rpcenv = PVE::RPCEnvironment::get();
6091                 if ($rpcenv->{type} ne 'cli') {
6092                     my $authuser = $rpcenv->get_user();
6093                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6094                 }
6095             }
6096
6097
6098             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6099                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6100             $path = PVE::Storage::path($storecfg, $vmstate);
6101             PVE::QemuConfig->write_config($vmid, $conf);
6102         } else {
6103             mon_cmd($vmid, "stop");
6104         }
6105     });
6106
6107     if ($includestate) {
6108         # save vm state
6109         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6110
6111         eval {
6112             set_migration_caps($vmid, 1);
6113             mon_cmd($vmid, "savevm-start", statefile => $path);
6114             for(;;) {
6115                 my $state = mon_cmd($vmid, "query-savevm");
6116                 if (!$state->{status}) {
6117                     die "savevm not active\n";
6118                 } elsif ($state->{status} eq 'active') {
6119                     sleep(1);
6120                     next;
6121                 } elsif ($state->{status} eq 'completed') {
6122                     print "State saved, quitting\n";
6123                     last;
6124                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6125                     die "query-savevm failed with error '$state->{error}'\n"
6126                 } else {
6127                     die "query-savevm returned status '$state->{status}'\n";
6128                 }
6129             }
6130         };
6131         my $err = $@;
6132
6133         PVE::QemuConfig->lock_config($vmid, sub {
6134             $conf = PVE::QemuConfig->load_config($vmid);
6135             if ($err) {
6136                 # cleanup, but leave suspending lock, to indicate something went wrong
6137                 eval {
6138                     mon_cmd($vmid, "savevm-end");
6139                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6140                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6141                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6142                     PVE::QemuConfig->write_config($vmid, $conf);
6143                 };
6144                 warn $@ if $@;
6145                 die $err;
6146             }
6147
6148             die "lock changed unexpectedly\n"
6149                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6150
6151             mon_cmd($vmid, "quit");
6152             $conf->{lock} = 'suspended';
6153             PVE::QemuConfig->write_config($vmid, $conf);
6154         });
6155     }
6156 }
6157
6158 sub vm_resume {
6159     my ($vmid, $skiplock, $nocheck) = @_;
6160
6161     PVE::QemuConfig->lock_config($vmid, sub {
6162         my $res = mon_cmd($vmid, 'query-status');
6163         my $resume_cmd = 'cont';
6164         my $reset = 0;
6165
6166         if ($res->{status}) {
6167             return if $res->{status} eq 'running'; # job done, go home
6168             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6169             $reset = 1 if $res->{status} eq 'shutdown';
6170         }
6171
6172         if (!$nocheck) {
6173
6174             my $conf = PVE::QemuConfig->load_config($vmid);
6175
6176             PVE::QemuConfig->check_lock($conf)
6177                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6178         }
6179
6180         if ($reset) {
6181             # required if a VM shuts down during a backup and we get a resume
6182             # request before the backup finishes for example
6183             mon_cmd($vmid, "system_reset");
6184         }
6185         mon_cmd($vmid, $resume_cmd);
6186     });
6187 }
6188
6189 sub vm_sendkey {
6190     my ($vmid, $skiplock, $key) = @_;
6191
6192     PVE::QemuConfig->lock_config($vmid, sub {
6193
6194         my $conf = PVE::QemuConfig->load_config($vmid);
6195
6196         # there is no qmp command, so we use the human monitor command
6197         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6198         die $res if $res ne '';
6199     });
6200 }
6201
6202 # vzdump restore implementaion
6203
6204 sub tar_archive_read_firstfile {
6205     my $archive = shift;
6206
6207     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6208
6209     # try to detect archive type first
6210     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6211         die "unable to open file '$archive'\n";
6212     my $firstfile = <$fh>;
6213     kill 15, $pid;
6214     close $fh;
6215
6216     die "ERROR: archive contaions no data\n" if !$firstfile;
6217     chomp $firstfile;
6218
6219     return $firstfile;
6220 }
6221
6222 sub tar_restore_cleanup {
6223     my ($storecfg, $statfile) = @_;
6224
6225     print STDERR "starting cleanup\n";
6226
6227     if (my $fd = IO::File->new($statfile, "r")) {
6228         while (defined(my $line = <$fd>)) {
6229             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6230                 my $volid = $2;
6231                 eval {
6232                     if ($volid =~ m|^/|) {
6233                         unlink $volid || die 'unlink failed\n';
6234                     } else {
6235                         PVE::Storage::vdisk_free($storecfg, $volid);
6236                     }
6237                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6238                 };
6239                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6240             } else {
6241                 print STDERR "unable to parse line in statfile - $line";
6242             }
6243         }
6244         $fd->close();
6245     }
6246 }
6247
6248 sub restore_file_archive {
6249     my ($archive, $vmid, $user, $opts) = @_;
6250
6251     return restore_vma_archive($archive, $vmid, $user, $opts)
6252         if $archive eq '-';
6253
6254     my $info = PVE::Storage::archive_info($archive);
6255     my $format = $opts->{format} // $info->{format};
6256     my $comp = $info->{compression};
6257
6258     # try to detect archive format
6259     if ($format eq 'tar') {
6260         return restore_tar_archive($archive, $vmid, $user, $opts);
6261     } else {
6262         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6263     }
6264 }
6265
6266 # hepler to remove disks that will not be used after restore
6267 my $restore_cleanup_oldconf = sub {
6268     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6269
6270     my $kept_disks = {};
6271
6272     PVE::QemuConfig->foreach_volume($oldconf, sub {
6273         my ($ds, $drive) = @_;
6274
6275         return if drive_is_cdrom($drive, 1);
6276
6277         my $volid = $drive->{file};
6278         return if !$volid || $volid =~ m|^/|;
6279
6280         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6281         return if !$path || !$owner || ($owner != $vmid);
6282
6283         # Note: only delete disk we want to restore
6284         # other volumes will become unused
6285         if ($virtdev_hash->{$ds}) {
6286             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6287             if (my $err = $@) {
6288                 warn $err;
6289             }
6290         } else {
6291             $kept_disks->{$volid} = 1;
6292         }
6293     });
6294
6295     # after the restore we have no snapshots anymore
6296     for my $snapname (keys $oldconf->{snapshots}->%*) {
6297         my $snap = $oldconf->{snapshots}->{$snapname};
6298         if ($snap->{vmstate}) {
6299             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6300             if (my $err = $@) {
6301                 warn $err;
6302             }
6303         }
6304
6305         for my $volid (keys $kept_disks->%*) {
6306             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6307             warn $@ if $@;
6308         }
6309     }
6310 };
6311
6312 # Helper to parse vzdump backup device hints
6313 #
6314 # $rpcenv: Environment, used to ckeck storage permissions
6315 # $user: User ID, to check storage permissions
6316 # $storecfg: Storage configuration
6317 # $fh: the file handle for reading the configuration
6318 # $devinfo: should contain device sizes for all backu-up'ed devices
6319 # $options: backup options (pool, default storage)
6320 #
6321 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6322 my $parse_backup_hints = sub {
6323     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6324
6325     my $check_storage = sub { # assert if an image can be allocate
6326         my ($storeid, $scfg) = @_;
6327         die "Content type 'images' is not available on storage '$storeid'\n"
6328             if !$scfg->{content}->{images};
6329         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6330             if $user ne 'root@pam';
6331     };
6332
6333     my $virtdev_hash = {};
6334     while (defined(my $line = <$fh>)) {
6335         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6336             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6337             die "archive does not contain data for drive '$virtdev'\n"
6338                 if !$devinfo->{$devname};
6339
6340             if (defined($options->{storage})) {
6341                 $storeid = $options->{storage} || 'local';
6342             } elsif (!$storeid) {
6343                 $storeid = 'local';
6344             }
6345             $format = 'raw' if !$format;
6346             $devinfo->{$devname}->{devname} = $devname;
6347             $devinfo->{$devname}->{virtdev} = $virtdev;
6348             $devinfo->{$devname}->{format} = $format;
6349             $devinfo->{$devname}->{storeid} = $storeid;
6350
6351             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6352             $check_storage->($storeid, $scfg); # permission and content type check
6353
6354             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6355         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6356             my $virtdev = $1;
6357             my $drive = parse_drive($virtdev, $2);
6358
6359             if (drive_is_cloudinit($drive)) {
6360                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6361                 $storeid = $options->{storage} if defined ($options->{storage});
6362                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6363                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6364
6365                 $check_storage->($storeid, $scfg); # permission and content type check
6366
6367                 $virtdev_hash->{$virtdev} = {
6368                     format => $format,
6369                     storeid => $storeid,
6370                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6371                     is_cloudinit => 1,
6372                 };
6373             }
6374         }
6375     }
6376
6377     return $virtdev_hash;
6378 };
6379
6380 # Helper to allocate and activate all volumes required for a restore
6381 #
6382 # $storecfg: Storage configuration
6383 # $virtdev_hash: as returned by parse_backup_hints()
6384 #
6385 # Returns: { $virtdev => $volid }
6386 my $restore_allocate_devices = sub {
6387     my ($storecfg, $virtdev_hash, $vmid) = @_;
6388
6389     my $map = {};
6390     foreach my $virtdev (sort keys %$virtdev_hash) {
6391         my $d = $virtdev_hash->{$virtdev};
6392         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6393         my $storeid = $d->{storeid};
6394         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6395
6396         # test if requested format is supported
6397         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6398         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6399         $d->{format} = $defFormat if !$supported;
6400
6401         my $name;
6402         if ($d->{is_cloudinit}) {
6403             $name = "vm-$vmid-cloudinit";
6404             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6405             if ($scfg->{path}) {
6406                 $name .= ".$d->{format}";
6407             }
6408         }
6409
6410         my $volid = PVE::Storage::vdisk_alloc(
6411             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6412
6413         print STDERR "new volume ID is '$volid'\n";
6414         $d->{volid} = $volid;
6415
6416         PVE::Storage::activate_volumes($storecfg, [$volid]);
6417
6418         $map->{$virtdev} = $volid;
6419     }
6420
6421     return $map;
6422 };
6423
6424 sub restore_update_config_line {
6425     my ($cookie, $map, $line, $unique) = @_;
6426
6427     return '' if $line =~ m/^\#qmdump\#/;
6428     return '' if $line =~ m/^\#vzdump\#/;
6429     return '' if $line =~ m/^lock:/;
6430     return '' if $line =~ m/^unused\d+:/;
6431     return '' if $line =~ m/^parent:/;
6432
6433     my $res = '';
6434
6435     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6436     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6437         # try to convert old 1.X settings
6438         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6439         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6440             my ($model, $macaddr) = split(/\=/, $devconfig);
6441             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6442             my $net = {
6443                 model => $model,
6444                 bridge => "vmbr$ind",
6445                 macaddr => $macaddr,
6446             };
6447             my $netstr = print_net($net);
6448
6449             $res .= "net$cookie->{netcount}: $netstr\n";
6450             $cookie->{netcount}++;
6451         }
6452     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6453         my ($id, $netstr) = ($1, $2);
6454         my $net = parse_net($netstr);
6455         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6456         $netstr = print_net($net);
6457         $res .= "$id: $netstr\n";
6458     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6459         my $virtdev = $1;
6460         my $value = $3;
6461         my $di = parse_drive($virtdev, $value);
6462         if (defined($di->{backup}) && !$di->{backup}) {
6463             $res .= "#$line";
6464         } elsif ($map->{$virtdev}) {
6465             delete $di->{format}; # format can change on restore
6466             $di->{file} = $map->{$virtdev};
6467             $value = print_drive($di);
6468             $res .= "$virtdev: $value\n";
6469         } else {
6470             $res .= $line;
6471         }
6472     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6473         my $vmgenid = $1;
6474         if ($vmgenid ne '0') {
6475             # always generate a new vmgenid if there was a valid one setup
6476             $vmgenid = generate_uuid();
6477         }
6478         $res .= "vmgenid: $vmgenid\n";
6479     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6480         my ($uuid, $uuid_str);
6481         UUID::generate($uuid);
6482         UUID::unparse($uuid, $uuid_str);
6483         my $smbios1 = parse_smbios1($2);
6484         $smbios1->{uuid} = $uuid_str;
6485         $res .= $1.print_smbios1($smbios1)."\n";
6486     } else {
6487         $res .= $line;
6488     }
6489
6490     return $res;
6491 }
6492
6493 my $restore_deactivate_volumes = sub {
6494     my ($storecfg, $virtdev_hash) = @_;
6495
6496     my $vollist = [];
6497     for my $dev (values $virtdev_hash->%*) {
6498         push $vollist->@*, $dev->{volid} if $dev->{volid};
6499     }
6500
6501     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6502     print STDERR $@ if $@;
6503 };
6504
6505 my $restore_destroy_volumes = sub {
6506     my ($storecfg, $virtdev_hash) = @_;
6507
6508     for my $dev (values $virtdev_hash->%*) {
6509         my $volid = $dev->{volid} or next;
6510         eval {
6511             PVE::Storage::vdisk_free($storecfg, $volid);
6512             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6513         };
6514         print STDERR "unable to cleanup '$volid' - $@" if $@;
6515     }
6516 };
6517
6518 my $restore_merge_config = sub {
6519     my ($filename, $backup_conf_raw, $override_conf) = @_;
6520
6521     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6522     for my $key (keys $override_conf->%*) {
6523         $backup_conf->{$key} = $override_conf->{$key};
6524     }
6525
6526     return $backup_conf;
6527 };
6528
6529 sub scan_volids {
6530     my ($cfg, $vmid) = @_;
6531
6532     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6533
6534     my $volid_hash = {};
6535     foreach my $storeid (keys %$info) {
6536         foreach my $item (@{$info->{$storeid}}) {
6537             next if !($item->{volid} && $item->{size});
6538             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6539             $volid_hash->{$item->{volid}} = $item;
6540         }
6541     }
6542
6543     return $volid_hash;
6544 }
6545
6546 sub update_disk_config {
6547     my ($vmid, $conf, $volid_hash) = @_;
6548
6549     my $changes;
6550     my $prefix = "VM $vmid";
6551
6552     # used and unused disks
6553     my $referenced = {};
6554
6555     # Note: it is allowed to define multiple storages with same path (alias), so
6556     # we need to check both 'volid' and real 'path' (two different volid can point
6557     # to the same path).
6558
6559     my $referencedpath = {};
6560
6561     # update size info
6562     PVE::QemuConfig->foreach_volume($conf, sub {
6563         my ($opt, $drive) = @_;
6564
6565         my $volid = $drive->{file};
6566         return if !$volid;
6567         my $volume = $volid_hash->{$volid};
6568
6569         # mark volid as "in-use" for next step
6570         $referenced->{$volid} = 1;
6571         if ($volume && (my $path = $volume->{path})) {
6572             $referencedpath->{$path} = 1;
6573         }
6574
6575         return if drive_is_cdrom($drive);
6576         return if !$volume;
6577
6578         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6579         if (defined($updated)) {
6580             $changes = 1;
6581             $conf->{$opt} = print_drive($updated);
6582             print "$prefix ($opt): $msg\n";
6583         }
6584     });
6585
6586     # remove 'unusedX' entry if volume is used
6587     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6588         my ($opt, $drive) = @_;
6589
6590         my $volid = $drive->{file};
6591         return if !$volid;
6592
6593         my $path;
6594         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6595         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6596             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6597             $changes = 1;
6598             delete $conf->{$opt};
6599         }
6600
6601         $referenced->{$volid} = 1;
6602         $referencedpath->{$path} = 1 if $path;
6603     });
6604
6605     foreach my $volid (sort keys %$volid_hash) {
6606         next if $volid =~ m/vm-$vmid-state-/;
6607         next if $referenced->{$volid};
6608         my $path = $volid_hash->{$volid}->{path};
6609         next if !$path; # just to be sure
6610         next if $referencedpath->{$path};
6611         $changes = 1;
6612         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6613         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6614         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6615     }
6616
6617     return $changes;
6618 }
6619
6620 sub rescan {
6621     my ($vmid, $nolock, $dryrun) = @_;
6622
6623     my $cfg = PVE::Storage::config();
6624
6625     print "rescan volumes...\n";
6626     my $volid_hash = scan_volids($cfg, $vmid);
6627
6628     my $updatefn =  sub {
6629         my ($vmid) = @_;
6630
6631         my $conf = PVE::QemuConfig->load_config($vmid);
6632
6633         PVE::QemuConfig->check_lock($conf);
6634
6635         my $vm_volids = {};
6636         foreach my $volid (keys %$volid_hash) {
6637             my $info = $volid_hash->{$volid};
6638             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6639         }
6640
6641         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6642
6643         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6644     };
6645
6646     if (defined($vmid)) {
6647         if ($nolock) {
6648             &$updatefn($vmid);
6649         } else {
6650             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6651         }
6652     } else {
6653         my $vmlist = config_list();
6654         foreach my $vmid (keys %$vmlist) {
6655             if ($nolock) {
6656                 &$updatefn($vmid);
6657             } else {
6658                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6659             }
6660         }
6661     }
6662 }
6663
6664 sub restore_proxmox_backup_archive {
6665     my ($archive, $vmid, $user, $options) = @_;
6666
6667     my $storecfg = PVE::Storage::config();
6668
6669     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6670     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6671
6672     my $fingerprint = $scfg->{fingerprint};
6673     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6674
6675     my $repo = PVE::PBSClient::get_repository($scfg);
6676     my $namespace = $scfg->{namespace};
6677
6678     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6679     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6680     local $ENV{PBS_PASSWORD} = $password;
6681     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6682
6683     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6684         PVE::Storage::parse_volname($storecfg, $archive);
6685
6686     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6687
6688     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6689
6690     my $tmpdir = "/var/tmp/vzdumptmp$$";
6691     rmtree $tmpdir;
6692     mkpath $tmpdir;
6693
6694     my $conffile = PVE::QemuConfig->config_file($vmid);
6695      # disable interrupts (always do cleanups)
6696     local $SIG{INT} =
6697         local $SIG{TERM} =
6698         local $SIG{QUIT} =
6699         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6700
6701     # Note: $oldconf is undef if VM does not exists
6702     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6703     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6704     my $new_conf_raw = '';
6705
6706     my $rpcenv = PVE::RPCEnvironment::get();
6707     my $devinfo = {}; # info about drives included in backup
6708     my $virtdev_hash = {}; # info about allocated drives
6709
6710     eval {
6711         # enable interrupts
6712         local $SIG{INT} =
6713             local $SIG{TERM} =
6714             local $SIG{QUIT} =
6715             local $SIG{HUP} =
6716             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6717
6718         my $cfgfn = "$tmpdir/qemu-server.conf";
6719         my $firewall_config_fn = "$tmpdir/fw.conf";
6720         my $index_fn = "$tmpdir/index.json";
6721
6722         my $cmd = "restore";
6723
6724         my $param = [$pbs_backup_name, "index.json", $index_fn];
6725         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6726         my $index = PVE::Tools::file_get_contents($index_fn);
6727         $index = decode_json($index);
6728
6729         foreach my $info (@{$index->{files}}) {
6730             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6731                 my $devname = $1;
6732                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6733                     $devinfo->{$devname}->{size} = $1;
6734                 } else {
6735                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6736                 }
6737             }
6738         }
6739
6740         my $is_qemu_server_backup = scalar(
6741             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6742         );
6743         if (!$is_qemu_server_backup) {
6744             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6745         }
6746         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6747
6748         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6749         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6750
6751         if ($has_firewall_config) {
6752             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6753             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6754
6755             my $pve_firewall_dir = '/etc/pve/firewall';
6756             mkdir $pve_firewall_dir; # make sure the dir exists
6757             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6758         }
6759
6760         my $fh = IO::File->new($cfgfn, "r") ||
6761             die "unable to read qemu-server.conf - $!\n";
6762
6763         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6764
6765         # fixme: rate limit?
6766
6767         # create empty/temp config
6768         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6769
6770         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6771
6772         # allocate volumes
6773         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6774
6775         foreach my $virtdev (sort keys %$virtdev_hash) {
6776             my $d = $virtdev_hash->{$virtdev};
6777             next if $d->{is_cloudinit}; # no need to restore cloudinit
6778
6779             # this fails if storage is unavailable
6780             my $volid = $d->{volid};
6781             my $path = PVE::Storage::path($storecfg, $volid);
6782
6783             # for live-restore we only want to preload the efidisk and TPM state
6784             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6785
6786             my @ns_arg;
6787             if (defined(my $ns = $scfg->{namespace})) {
6788                 @ns_arg = ('--ns', $ns);
6789             }
6790
6791             my $pbs_restore_cmd = [
6792                 '/usr/bin/pbs-restore',
6793                 '--repository', $repo,
6794                 @ns_arg,
6795                 $pbs_backup_name,
6796                 "$d->{devname}.img.fidx",
6797                 $path,
6798                 '--verbose',
6799                 ];
6800
6801             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6802             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6803
6804             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6805                 push @$pbs_restore_cmd, '--skip-zero';
6806             }
6807
6808             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6809             print "restore proxmox backup image: $dbg_cmdstring\n";
6810             run_command($pbs_restore_cmd);
6811         }
6812
6813         $fh->seek(0, 0) || die "seek failed - $!\n";
6814
6815         my $cookie = { netcount => 0 };
6816         while (defined(my $line = <$fh>)) {
6817             $new_conf_raw .= restore_update_config_line(
6818                 $cookie,
6819                 $map,
6820                 $line,
6821                 $options->{unique},
6822             );
6823         }
6824
6825         $fh->close();
6826     };
6827     my $err = $@;
6828
6829     if ($err || !$options->{live}) {
6830         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6831     }
6832
6833     rmtree $tmpdir;
6834
6835     if ($err) {
6836         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6837         die $err;
6838     }
6839
6840     if ($options->{live}) {
6841         # keep lock during live-restore
6842         $new_conf_raw .= "\nlock: create";
6843     }
6844
6845     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6846     PVE::QemuConfig->write_config($vmid, $new_conf);
6847
6848     eval { rescan($vmid, 1); };
6849     warn $@ if $@;
6850
6851     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6852
6853     if ($options->{live}) {
6854         # enable interrupts
6855         local $SIG{INT} =
6856             local $SIG{TERM} =
6857             local $SIG{QUIT} =
6858             local $SIG{HUP} =
6859             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6860
6861         my $conf = PVE::QemuConfig->load_config($vmid);
6862         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6863
6864         # these special drives are already restored before start
6865         delete $devinfo->{'drive-efidisk0'};
6866         delete $devinfo->{'drive-tpmstate0-backup'};
6867
6868         my $pbs_opts = {
6869             repo => $repo,
6870             keyfile => $keyfile,
6871             snapshot => $pbs_backup_name,
6872             namespace => $namespace,
6873         };
6874         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6875
6876         PVE::QemuConfig->remove_lock($vmid, "create");
6877     }
6878 }
6879
6880 sub pbs_live_restore {
6881     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6882
6883     print "starting VM for live-restore\n";
6884     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6885
6886     my $pbs_backing = {};
6887     for my $ds (keys %$restored_disks) {
6888         $ds =~ m/^drive-(.*)$/;
6889         my $confname = $1;
6890         $pbs_backing->{$confname} = {
6891             repository => $opts->{repo},
6892             snapshot => $opts->{snapshot},
6893             archive => "$ds.img.fidx",
6894         };
6895         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6896         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6897
6898         my $drive = parse_drive($confname, $conf->{$confname});
6899         print "restoring '$ds' to '$drive->{file}'\n";
6900     }
6901
6902     my $drives_streamed = 0;
6903     eval {
6904         # make sure HA doesn't interrupt our restore by stopping the VM
6905         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6906             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6907         }
6908
6909         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6910         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6911         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6912
6913         my $qmeventd_fd = register_qmeventd_handle($vmid);
6914
6915         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6916         # this will effectively collapse the backing image chain consisting of
6917         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6918         # removes itself once all backing images vanish with 'auto-remove=on')
6919         my $jobs = {};
6920         for my $ds (sort keys %$restored_disks) {
6921             my $job_id = "restore-$ds";
6922             mon_cmd($vmid, 'block-stream',
6923                 'job-id' => $job_id,
6924                 device => "$ds",
6925             );
6926             $jobs->{$job_id} = {};
6927         }
6928
6929         mon_cmd($vmid, 'cont');
6930         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6931
6932         print "restore-drive jobs finished successfully, removing all tracking block devices"
6933             ." to disconnect from Proxmox Backup Server\n";
6934
6935         for my $ds (sort keys %$restored_disks) {
6936             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6937         }
6938
6939         close($qmeventd_fd);
6940     };
6941
6942     my $err = $@;
6943
6944     if ($err) {
6945         warn "An error occurred during live-restore: $err\n";
6946         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6947         die "live-restore failed\n";
6948     }
6949 }
6950
6951 sub restore_vma_archive {
6952     my ($archive, $vmid, $user, $opts, $comp) = @_;
6953
6954     my $readfrom = $archive;
6955
6956     my $cfg = PVE::Storage::config();
6957     my $commands = [];
6958     my $bwlimit = $opts->{bwlimit};
6959
6960     my $dbg_cmdstring = '';
6961     my $add_pipe = sub {
6962         my ($cmd) = @_;
6963         push @$commands, $cmd;
6964         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6965         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6966         $readfrom = '-';
6967     };
6968
6969     my $input = undef;
6970     if ($archive eq '-') {
6971         $input = '<&STDIN';
6972     } else {
6973         # If we use a backup from a PVE defined storage we also consider that
6974         # storage's rate limit:
6975         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6976         if (defined($volid)) {
6977             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6978             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6979             if ($readlimit) {
6980                 print STDERR "applying read rate limit: $readlimit\n";
6981                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6982                 $add_pipe->($cstream);
6983             }
6984         }
6985     }
6986
6987     if ($comp) {
6988         my $info = PVE::Storage::decompressor_info('vma', $comp);
6989         my $cmd = $info->{decompressor};
6990         push @$cmd, $readfrom;
6991         $add_pipe->($cmd);
6992     }
6993
6994     my $tmpdir = "/var/tmp/vzdumptmp$$";
6995     rmtree $tmpdir;
6996
6997     # disable interrupts (always do cleanups)
6998     local $SIG{INT} =
6999         local $SIG{TERM} =
7000         local $SIG{QUIT} =
7001         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7002
7003     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7004     POSIX::mkfifo($mapfifo, 0600);
7005     my $fifofh;
7006     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7007
7008     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7009
7010     my $oldtimeout;
7011     my $timeout = 5;
7012
7013     my $devinfo = {}; # info about drives included in backup
7014     my $virtdev_hash = {}; # info about allocated drives
7015
7016     my $rpcenv = PVE::RPCEnvironment::get();
7017
7018     my $conffile = PVE::QemuConfig->config_file($vmid);
7019
7020     # Note: $oldconf is undef if VM does not exist
7021     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7022     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7023     my $new_conf_raw = '';
7024
7025     my %storage_limits;
7026
7027     my $print_devmap = sub {
7028         my $cfgfn = "$tmpdir/qemu-server.conf";
7029
7030         # we can read the config - that is already extracted
7031         my $fh = IO::File->new($cfgfn, "r") ||
7032             die "unable to read qemu-server.conf - $!\n";
7033
7034         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7035         if (-f $fwcfgfn) {
7036             my $pve_firewall_dir = '/etc/pve/firewall';
7037             mkdir $pve_firewall_dir; # make sure the dir exists
7038             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7039         }
7040
7041         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7042
7043         foreach my $info (values %{$virtdev_hash}) {
7044             my $storeid = $info->{storeid};
7045             next if defined($storage_limits{$storeid});
7046
7047             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7048             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7049             $storage_limits{$storeid} = $limit * 1024;
7050         }
7051
7052         foreach my $devname (keys %$devinfo) {
7053             die "found no device mapping information for device '$devname'\n"
7054                 if !$devinfo->{$devname}->{virtdev};
7055         }
7056
7057         # create empty/temp config
7058         if ($oldconf) {
7059             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7060             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7061         }
7062
7063         # allocate volumes
7064         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7065
7066         # print restore information to $fifofh
7067         foreach my $virtdev (sort keys %$virtdev_hash) {
7068             my $d = $virtdev_hash->{$virtdev};
7069             next if $d->{is_cloudinit}; # no need to restore cloudinit
7070
7071             my $storeid = $d->{storeid};
7072             my $volid = $d->{volid};
7073
7074             my $map_opts = '';
7075             if (my $limit = $storage_limits{$storeid}) {
7076                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7077             }
7078
7079             my $write_zeros = 1;
7080             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7081                 $write_zeros = 0;
7082             }
7083
7084             my $path = PVE::Storage::path($cfg, $volid);
7085
7086             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7087
7088             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7089         }
7090
7091         $fh->seek(0, 0) || die "seek failed - $!\n";
7092
7093         my $cookie = { netcount => 0 };
7094         while (defined(my $line = <$fh>)) {
7095             $new_conf_raw .= restore_update_config_line(
7096                 $cookie,
7097                 $map,
7098                 $line,
7099                 $opts->{unique},
7100             );
7101         }
7102
7103         $fh->close();
7104     };
7105
7106     eval {
7107         # enable interrupts
7108         local $SIG{INT} =
7109             local $SIG{TERM} =
7110             local $SIG{QUIT} =
7111             local $SIG{HUP} =
7112             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7113         local $SIG{ALRM} = sub { die "got timeout\n"; };
7114
7115         $oldtimeout = alarm($timeout);
7116
7117         my $parser = sub {
7118             my $line = shift;
7119
7120             print "$line\n";
7121
7122             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7123                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7124                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7125             } elsif ($line =~ m/^CTIME: /) {
7126                 # we correctly received the vma config, so we can disable
7127                 # the timeout now for disk allocation (set to 10 minutes, so
7128                 # that we always timeout if something goes wrong)
7129                 alarm(600);
7130                 &$print_devmap();
7131                 print $fifofh "done\n";
7132                 my $tmp = $oldtimeout || 0;
7133                 $oldtimeout = undef;
7134                 alarm($tmp);
7135                 close($fifofh);
7136                 $fifofh = undef;
7137             }
7138         };
7139
7140         print "restore vma archive: $dbg_cmdstring\n";
7141         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7142     };
7143     my $err = $@;
7144
7145     alarm($oldtimeout) if $oldtimeout;
7146
7147     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7148
7149     close($fifofh) if $fifofh;
7150     unlink $mapfifo;
7151     rmtree $tmpdir;
7152
7153     if ($err) {
7154         $restore_destroy_volumes->($cfg, $virtdev_hash);
7155         die $err;
7156     }
7157
7158     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7159     PVE::QemuConfig->write_config($vmid, $new_conf);
7160
7161     eval { rescan($vmid, 1); };
7162     warn $@ if $@;
7163
7164     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7165 }
7166
7167 sub restore_tar_archive {
7168     my ($archive, $vmid, $user, $opts) = @_;
7169
7170     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7171         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7172         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7173     }
7174
7175     if ($archive ne '-') {
7176         my $firstfile = tar_archive_read_firstfile($archive);
7177         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7178             if $firstfile ne 'qemu-server.conf';
7179     }
7180
7181     my $storecfg = PVE::Storage::config();
7182
7183     # avoid zombie disks when restoring over an existing VM -> cleanup first
7184     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7185     # skiplock=1 because qmrestore has set the 'create' lock itself already
7186     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7187     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7188
7189     my $tocmd = "/usr/lib/qemu-server/qmextract";
7190
7191     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7192     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7193     $tocmd .= ' --prealloc' if $opts->{prealloc};
7194     $tocmd .= ' --info' if $opts->{info};
7195
7196     # tar option "xf" does not autodetect compression when read from STDIN,
7197     # so we pipe to zcat
7198     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7199         PVE::Tools::shellquote("--to-command=$tocmd");
7200
7201     my $tmpdir = "/var/tmp/vzdumptmp$$";
7202     mkpath $tmpdir;
7203
7204     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7205     local $ENV{VZDUMP_VMID} = $vmid;
7206     local $ENV{VZDUMP_USER} = $user;
7207
7208     my $conffile = PVE::QemuConfig->config_file($vmid);
7209     my $new_conf_raw = '';
7210
7211     # disable interrupts (always do cleanups)
7212     local $SIG{INT} =
7213         local $SIG{TERM} =
7214         local $SIG{QUIT} =
7215         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7216
7217     eval {
7218         # enable interrupts
7219         local $SIG{INT} =
7220             local $SIG{TERM} =
7221             local $SIG{QUIT} =
7222             local $SIG{HUP} =
7223             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7224
7225         if ($archive eq '-') {
7226             print "extracting archive from STDIN\n";
7227             run_command($cmd, input => "<&STDIN");
7228         } else {
7229             print "extracting archive '$archive'\n";
7230             run_command($cmd);
7231         }
7232
7233         return if $opts->{info};
7234
7235         # read new mapping
7236         my $map = {};
7237         my $statfile = "$tmpdir/qmrestore.stat";
7238         if (my $fd = IO::File->new($statfile, "r")) {
7239             while (defined (my $line = <$fd>)) {
7240                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7241                     $map->{$1} = $2 if $1;
7242                 } else {
7243                     print STDERR "unable to parse line in statfile - $line\n";
7244                 }
7245             }
7246             $fd->close();
7247         }
7248
7249         my $confsrc = "$tmpdir/qemu-server.conf";
7250
7251         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7252
7253         my $cookie = { netcount => 0 };
7254         while (defined (my $line = <$srcfd>)) {
7255             $new_conf_raw .= restore_update_config_line(
7256                 $cookie,
7257                 $map,
7258                 $line,
7259                 $opts->{unique},
7260             );
7261         }
7262
7263         $srcfd->close();
7264     };
7265     if (my $err = $@) {
7266         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7267         die $err;
7268     }
7269
7270     rmtree $tmpdir;
7271
7272     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7273
7274     PVE::Cluster::cfs_update(); # make sure we read new file
7275
7276     eval { rescan($vmid, 1); };
7277     warn $@ if $@;
7278 };
7279
7280 sub foreach_storage_used_by_vm {
7281     my ($conf, $func) = @_;
7282
7283     my $sidhash = {};
7284
7285     PVE::QemuConfig->foreach_volume($conf, sub {
7286         my ($ds, $drive) = @_;
7287         return if drive_is_cdrom($drive);
7288
7289         my $volid = $drive->{file};
7290
7291         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7292         $sidhash->{$sid} = $sid if $sid;
7293     });
7294
7295     foreach my $sid (sort keys %$sidhash) {
7296         &$func($sid);
7297     }
7298 }
7299
7300 my $qemu_snap_storage = {
7301     rbd => 1,
7302 };
7303 sub do_snapshots_with_qemu {
7304     my ($storecfg, $volid, $deviceid) = @_;
7305
7306     return if $deviceid =~ m/tpmstate0/;
7307
7308     my $storage_name = PVE::Storage::parse_volume_id($volid);
7309     my $scfg = $storecfg->{ids}->{$storage_name};
7310     die "could not find storage '$storage_name'\n" if !defined($scfg);
7311
7312     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7313         return 1;
7314     }
7315
7316     if ($volid =~ m/\.(qcow2|qed)$/){
7317         return 1;
7318     }
7319
7320     return;
7321 }
7322
7323 sub qga_check_running {
7324     my ($vmid, $nowarn) = @_;
7325
7326     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7327     if ($@) {
7328         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7329         return 0;
7330     }
7331     return 1;
7332 }
7333
7334 sub template_create {
7335     my ($vmid, $conf, $disk) = @_;
7336
7337     my $storecfg = PVE::Storage::config();
7338
7339     PVE::QemuConfig->foreach_volume($conf, sub {
7340         my ($ds, $drive) = @_;
7341
7342         return if drive_is_cdrom($drive);
7343         return if $disk && $ds ne $disk;
7344
7345         my $volid = $drive->{file};
7346         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7347
7348         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7349         $drive->{file} = $voliddst;
7350         $conf->{$ds} = print_drive($drive);
7351         PVE::QemuConfig->write_config($vmid, $conf);
7352     });
7353 }
7354
7355 sub convert_iscsi_path {
7356     my ($path) = @_;
7357
7358     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7359         my $portal = $1;
7360         my $target = $2;
7361         my $lun = $3;
7362
7363         my $initiator_name = get_initiator_name();
7364
7365         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7366                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7367     }
7368
7369     die "cannot convert iscsi path '$path', unkown format\n";
7370 }
7371
7372 sub qemu_img_convert {
7373     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7374
7375     my $storecfg = PVE::Storage::config();
7376     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7377     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7378
7379     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7380
7381     my $cachemode;
7382     my $src_path;
7383     my $src_is_iscsi = 0;
7384     my $src_format;
7385
7386     if ($src_storeid) {
7387         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7388         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7389         $src_format = qemu_img_format($src_scfg, $src_volname);
7390         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7391         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7392         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7393     } elsif (-f $src_volid || -b $src_volid) {
7394         $src_path = $src_volid;
7395         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7396             $src_format = $1;
7397         }
7398     }
7399
7400     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7401
7402     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7403     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7404     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7405     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7406
7407     my $cmd = [];
7408     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7409     push @$cmd, '-l', "snapshot.name=$snapname"
7410         if $snapname && $src_format && $src_format eq "qcow2";
7411     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7412     push @$cmd, '-T', $cachemode if defined($cachemode);
7413
7414     if ($src_is_iscsi) {
7415         push @$cmd, '--image-opts';
7416         $src_path = convert_iscsi_path($src_path);
7417     } elsif ($src_format) {
7418         push @$cmd, '-f', $src_format;
7419     }
7420
7421     if ($dst_is_iscsi) {
7422         push @$cmd, '--target-image-opts';
7423         $dst_path = convert_iscsi_path($dst_path);
7424     } else {
7425         push @$cmd, '-O', $dst_format;
7426     }
7427
7428     push @$cmd, $src_path;
7429
7430     if (!$dst_is_iscsi && $is_zero_initialized) {
7431         push @$cmd, "zeroinit:$dst_path";
7432     } else {
7433         push @$cmd, $dst_path;
7434     }
7435
7436     my $parser = sub {
7437         my $line = shift;
7438         if($line =~ m/\((\S+)\/100\%\)/){
7439             my $percent = $1;
7440             my $transferred = int($size * $percent / 100);
7441             my $total_h = render_bytes($size, 1);
7442             my $transferred_h = render_bytes($transferred, 1);
7443
7444             print "transferred $transferred_h of $total_h ($percent%)\n";
7445         }
7446
7447     };
7448
7449     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7450     my $err = $@;
7451     die "copy failed: $err" if $err;
7452 }
7453
7454 sub qemu_img_format {
7455     my ($scfg, $volname) = @_;
7456
7457     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7458         return $1;
7459     } else {
7460         return "raw";
7461     }
7462 }
7463
7464 sub qemu_drive_mirror {
7465     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7466
7467     $jobs = {} if !$jobs;
7468
7469     my $qemu_target;
7470     my $format;
7471     $jobs->{"drive-$drive"} = {};
7472
7473     if ($dst_volid =~ /^nbd:/) {
7474         $qemu_target = $dst_volid;
7475         $format = "nbd";
7476     } else {
7477         my $storecfg = PVE::Storage::config();
7478         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7479
7480         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7481
7482         $format = qemu_img_format($dst_scfg, $dst_volname);
7483
7484         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7485
7486         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7487     }
7488
7489     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7490     $opts->{format} = $format if $format;
7491
7492     if (defined($src_bitmap)) {
7493         $opts->{sync} = 'incremental';
7494         $opts->{bitmap} = $src_bitmap;
7495         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7496     }
7497
7498     if (defined($bwlimit)) {
7499         $opts->{speed} = $bwlimit * 1024;
7500         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7501     } else {
7502         print "drive mirror is starting for drive-$drive\n";
7503     }
7504
7505     # if a job already runs for this device we get an error, catch it for cleanup
7506     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7507     if (my $err = $@) {
7508         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7509         warn "$@\n" if $@;
7510         die "mirroring error: $err\n";
7511     }
7512
7513     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7514 }
7515
7516 # $completion can be either
7517 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7518 # 'cancel': wait until all jobs are ready, block-job-cancel them
7519 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7520 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7521 sub qemu_drive_mirror_monitor {
7522     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7523
7524     $completion //= 'complete';
7525     $op //= "mirror";
7526
7527     eval {
7528         my $err_complete = 0;
7529
7530         my $starttime = time ();
7531         while (1) {
7532             die "block job ('$op') timed out\n" if $err_complete > 300;
7533
7534             my $stats = mon_cmd($vmid, "query-block-jobs");
7535             my $ctime = time();
7536
7537             my $running_jobs = {};
7538             for my $stat (@$stats) {
7539                 next if $stat->{type} ne $op;
7540                 $running_jobs->{$stat->{device}} = $stat;
7541             }
7542
7543             my $readycounter = 0;
7544
7545             for my $job_id (sort keys %$jobs) {
7546                 my $job = $running_jobs->{$job_id};
7547
7548                 my $vanished = !defined($job);
7549                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7550                 if($complete || ($vanished && $completion eq 'auto')) {
7551                     print "$job_id: $op-job finished\n";
7552                     delete $jobs->{$job_id};
7553                     next;
7554                 }
7555
7556                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7557
7558                 my $busy = $job->{busy};
7559                 my $ready = $job->{ready};
7560                 if (my $total = $job->{len}) {
7561                     my $transferred = $job->{offset} || 0;
7562                     my $remaining = $total - $transferred;
7563                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7564
7565                     my $duration = $ctime - $starttime;
7566                     my $total_h = render_bytes($total, 1);
7567                     my $transferred_h = render_bytes($transferred, 1);
7568
7569                     my $status = sprintf(
7570                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7571                         render_duration($duration),
7572                     );
7573
7574                     if ($ready) {
7575                         if ($busy) {
7576                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7577                         } else {
7578                             $status .= ", ready";
7579                         }
7580                     }
7581                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7582                     $jobs->{$job_id}->{ready} = $ready;
7583                 }
7584
7585                 $readycounter++ if $job->{ready};
7586             }
7587
7588             last if scalar(keys %$jobs) == 0;
7589
7590             if ($readycounter == scalar(keys %$jobs)) {
7591                 print "all '$op' jobs are ready\n";
7592
7593                 # do the complete later (or has already been done)
7594                 last if $completion eq 'skip' || $completion eq 'auto';
7595
7596                 if ($vmiddst && $vmiddst != $vmid) {
7597                     my $agent_running = $qga && qga_check_running($vmid);
7598                     if ($agent_running) {
7599                         print "freeze filesystem\n";
7600                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7601                         warn $@ if $@;
7602                     } else {
7603                         print "suspend vm\n";
7604                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7605                         warn $@ if $@;
7606                     }
7607
7608                     # if we clone a disk for a new target vm, we don't switch the disk
7609                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7610
7611                     if ($agent_running) {
7612                         print "unfreeze filesystem\n";
7613                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7614                         warn $@ if $@;
7615                     } else {
7616                         print "resume vm\n";
7617                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7618                         warn $@ if $@;
7619                     }
7620
7621                     last;
7622                 } else {
7623
7624                     for my $job_id (sort keys %$jobs) {
7625                         # try to switch the disk if source and destination are on the same guest
7626                         print "$job_id: Completing block job_id...\n";
7627
7628                         my $op;
7629                         if ($completion eq 'complete') {
7630                             $op = 'block-job-complete';
7631                         } elsif ($completion eq 'cancel') {
7632                             $op = 'block-job-cancel';
7633                         } else {
7634                             die "invalid completion value: $completion\n";
7635                         }
7636                         eval { mon_cmd($vmid, $op, device => $job_id) };
7637                         if ($@ =~ m/cannot be completed/) {
7638                             print "$job_id: block job cannot be completed, trying again.\n";
7639                             $err_complete++;
7640                         }else {
7641                             print "$job_id: Completed successfully.\n";
7642                             $jobs->{$job_id}->{complete} = 1;
7643                         }
7644                     }
7645                 }
7646             }
7647             sleep 1;
7648         }
7649     };
7650     my $err = $@;
7651
7652     if ($err) {
7653         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7654         die "block job ($op) error: $err";
7655     }
7656 }
7657
7658 sub qemu_blockjobs_cancel {
7659     my ($vmid, $jobs) = @_;
7660
7661     foreach my $job (keys %$jobs) {
7662         print "$job: Cancelling block job\n";
7663         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7664         $jobs->{$job}->{cancel} = 1;
7665     }
7666
7667     while (1) {
7668         my $stats = mon_cmd($vmid, "query-block-jobs");
7669
7670         my $running_jobs = {};
7671         foreach my $stat (@$stats) {
7672             $running_jobs->{$stat->{device}} = $stat;
7673         }
7674
7675         foreach my $job (keys %$jobs) {
7676
7677             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7678                 print "$job: Done.\n";
7679                 delete $jobs->{$job};
7680             }
7681         }
7682
7683         last if scalar(keys %$jobs) == 0;
7684
7685         sleep 1;
7686     }
7687 }
7688
7689 sub clone_disk {
7690     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7691
7692     my ($vmid, $running) = $source->@{qw(vmid running)};
7693     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7694
7695     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7696     my ($storage, $format) = $dest->@{qw(storage format)};
7697
7698     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7699
7700     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7701         die "cloning from/to EFI disk requires EFI disk\n"
7702             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7703         die "cloning from/to TPM state requires TPM state\n"
7704             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7705
7706         # This would lead to two device nodes in QEMU pointing to the same backing image!
7707         die "cannot change drive name when cloning disk from/to the same VM\n"
7708             if $use_drive_mirror && $vmid == $newvmid;
7709     }
7710
7711     die "cannot move TPM state while VM is running\n"
7712         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7713
7714     my $newvolid;
7715
7716     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7717     print "$src_drivename " if $src_drivename;
7718     print "($drive->{file})\n";
7719
7720     if (!$full) {
7721         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7722         push @$newvollist, $newvolid;
7723     } else {
7724
7725         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7726         $storeid = $storage if $storage;
7727
7728         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7729
7730         my $name = undef;
7731         my $size = undef;
7732         if (drive_is_cloudinit($drive)) {
7733             $name = "vm-$newvmid-cloudinit";
7734             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7735             if ($scfg->{path}) {
7736                 $name .= ".$dst_format";
7737             }
7738             $snapname = undef;
7739             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7740         } elsif ($dst_drivename eq 'efidisk0') {
7741             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7742         } elsif ($dst_drivename eq 'tpmstate0') {
7743             $dst_format = 'raw';
7744             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7745         } else {
7746             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7747         }
7748         $newvolid = PVE::Storage::vdisk_alloc(
7749             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7750         );
7751         push @$newvollist, $newvolid;
7752
7753         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7754
7755         if (drive_is_cloudinit($drive)) {
7756             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7757             # if this is the case, we have to complete any block-jobs still there from
7758             # previous drive-mirrors
7759             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7760                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7761             }
7762             goto no_data_clone;
7763         }
7764
7765         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7766         if ($use_drive_mirror) {
7767             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7768                 $completion, $qga, $bwlimit);
7769         } else {
7770             # TODO: handle bwlimits
7771             if ($dst_drivename eq 'efidisk0') {
7772                 # the relevant data on the efidisk may be smaller than the source
7773                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7774                 # that is given by the OVMF_VARS.fd
7775                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7776                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7777
7778                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7779
7780                 # better for Ceph if block size is not too small, see bug #3324
7781                 my $bs = 1024*1024;
7782
7783                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7784
7785                 if ($src_format eq 'qcow2' && $snapname) {
7786                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7787                         if !min_version(kvm_user_version(), 6, 2);
7788                     push $cmd->@*, '-l', $snapname;
7789                 }
7790                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7791                 run_command($cmd);
7792             } else {
7793                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7794             }
7795         }
7796     }
7797
7798 no_data_clone:
7799     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7800
7801     my $disk = dclone($drive);
7802     delete $disk->{format};
7803     $disk->{file} = $newvolid;
7804     $disk->{size} = $size if defined($size);
7805
7806     return $disk;
7807 }
7808
7809 sub get_running_qemu_version {
7810     my ($vmid) = @_;
7811     my $res = mon_cmd($vmid, "query-version");
7812     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7813 }
7814
7815 sub qemu_use_old_bios_files {
7816     my ($machine_type) = @_;
7817
7818     return if !$machine_type;
7819
7820     my $use_old_bios_files = undef;
7821
7822     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7823         $machine_type = $1;
7824         $use_old_bios_files = 1;
7825     } else {
7826         my $version = extract_version($machine_type, kvm_user_version());
7827         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7828         # load new efi bios files on migration. So this hack is required to allow
7829         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7830         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7831         $use_old_bios_files = !min_version($version, 2, 4);
7832     }
7833
7834     return ($use_old_bios_files, $machine_type);
7835 }
7836
7837 sub get_efivars_size {
7838     my ($conf, $efidisk) = @_;
7839
7840     my $arch = get_vm_arch($conf);
7841     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7842     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7843     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7844     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7845     return -s $ovmf_vars;
7846 }
7847
7848 sub update_efidisk_size {
7849     my ($conf) = @_;
7850
7851     return if !defined($conf->{efidisk0});
7852
7853     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7854     $disk->{size} = get_efivars_size($conf);
7855     $conf->{efidisk0} = print_drive($disk);
7856
7857     return;
7858 }
7859
7860 sub update_tpmstate_size {
7861     my ($conf) = @_;
7862
7863     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7864     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7865     $conf->{tpmstate0} = print_drive($disk);
7866 }
7867
7868 sub create_efidisk($$$$$$$) {
7869     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7870
7871     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7872     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7873
7874     my $vars_size_b = -s $ovmf_vars;
7875     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7876     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7877     PVE::Storage::activate_volumes($storecfg, [$volid]);
7878
7879     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7880     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7881
7882     return ($volid, $size/1024);
7883 }
7884
7885 sub vm_iothreads_list {
7886     my ($vmid) = @_;
7887
7888     my $res = mon_cmd($vmid, 'query-iothreads');
7889
7890     my $iothreads = {};
7891     foreach my $iothread (@$res) {
7892         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7893     }
7894
7895     return $iothreads;
7896 }
7897
7898 sub scsihw_infos {
7899     my ($conf, $drive) = @_;
7900
7901     my $maxdev = 0;
7902
7903     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7904         $maxdev = 7;
7905     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7906         $maxdev = 1;
7907     } else {
7908         $maxdev = 256;
7909     }
7910
7911     my $controller = int($drive->{index} / $maxdev);
7912     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7913         ? "virtioscsi"
7914         : "scsihw";
7915
7916     return ($maxdev, $controller, $controller_prefix);
7917 }
7918
7919 sub windows_version {
7920     my ($ostype) = @_;
7921
7922     return 0 if !$ostype;
7923
7924     my $winversion = 0;
7925
7926     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7927         $winversion = 5;
7928     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7929         $winversion = 6;
7930     } elsif ($ostype =~ m/^win(\d+)$/) {
7931         $winversion = $1;
7932     }
7933
7934     return $winversion;
7935 }
7936
7937 sub resolve_dst_disk_format {
7938         my ($storecfg, $storeid, $src_volname, $format) = @_;
7939         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7940
7941         if (!$format) {
7942             # if no target format is specified, use the source disk format as hint
7943             if ($src_volname) {
7944                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7945                 $format = qemu_img_format($scfg, $src_volname);
7946             } else {
7947                 return $defFormat;
7948             }
7949         }
7950
7951         # test if requested format is supported - else use default
7952         my $supported = grep { $_ eq $format } @$validFormats;
7953         $format = $defFormat if !$supported;
7954         return $format;
7955 }
7956
7957 # NOTE: if this logic changes, please update docs & possibly gui logic
7958 sub find_vmstate_storage {
7959     my ($conf, $storecfg) = @_;
7960
7961     # first, return storage from conf if set
7962     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7963
7964     my ($target, $shared, $local);
7965
7966     foreach_storage_used_by_vm($conf, sub {
7967         my ($sid) = @_;
7968         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7969         my $dst = $scfg->{shared} ? \$shared : \$local;
7970         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7971     });
7972
7973     # second, use shared storage where VM has at least one disk
7974     # third, use local storage where VM has at least one disk
7975     # fall back to local storage
7976     $target = $shared // $local // 'local';
7977
7978     return $target;
7979 }
7980
7981 sub generate_uuid {
7982     my ($uuid, $uuid_str);
7983     UUID::generate($uuid);
7984     UUID::unparse($uuid, $uuid_str);
7985     return $uuid_str;
7986 }
7987
7988 sub generate_smbios1_uuid {
7989     return "uuid=".generate_uuid();
7990 }
7991
7992 sub nbd_stop {
7993     my ($vmid) = @_;
7994
7995     mon_cmd($vmid, 'nbd-server-stop');
7996 }
7997
7998 sub create_reboot_request {
7999     my ($vmid) = @_;
8000     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8001         or die "failed to create reboot trigger file: $!\n";
8002     close($fh);
8003 }
8004
8005 sub clear_reboot_request {
8006     my ($vmid) = @_;
8007     my $path = "/run/qemu-server/$vmid.reboot";
8008     my $res = 0;
8009
8010     $res = unlink($path);
8011     die "could not remove reboot request for $vmid: $!"
8012         if !$res && $! != POSIX::ENOENT;
8013
8014     return $res;
8015 }
8016
8017 sub bootorder_from_legacy {
8018     my ($conf, $bootcfg) = @_;
8019
8020     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8021     my $bootindex_hash = {};
8022     my $i = 1;
8023     foreach my $o (split(//, $boot)) {
8024         $bootindex_hash->{$o} = $i*100;
8025         $i++;
8026     }
8027
8028     my $bootorder = {};
8029
8030     PVE::QemuConfig->foreach_volume($conf, sub {
8031         my ($ds, $drive) = @_;
8032
8033         if (drive_is_cdrom ($drive, 1)) {
8034             if ($bootindex_hash->{d}) {
8035                 $bootorder->{$ds} = $bootindex_hash->{d};
8036                 $bootindex_hash->{d} += 1;
8037             }
8038         } elsif ($bootindex_hash->{c}) {
8039             $bootorder->{$ds} = $bootindex_hash->{c}
8040                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8041             $bootindex_hash->{c} += 1;
8042         }
8043     });
8044
8045     if ($bootindex_hash->{n}) {
8046         for (my $i = 0; $i < $MAX_NETS; $i++) {
8047             my $netname = "net$i";
8048             next if !$conf->{$netname};
8049             $bootorder->{$netname} = $bootindex_hash->{n};
8050             $bootindex_hash->{n} += 1;
8051         }
8052     }
8053
8054     return $bootorder;
8055 }
8056
8057 # Generate default device list for 'boot: order=' property. Matches legacy
8058 # default boot order, but with explicit device names. This is important, since
8059 # the fallback for when neither 'order' nor the old format is specified relies
8060 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8061 sub get_default_bootdevices {
8062     my ($conf) = @_;
8063
8064     my @ret = ();
8065
8066     # harddisk
8067     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8068     push @ret, $first if $first;
8069
8070     # cdrom
8071     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8072     push @ret, $first if $first;
8073
8074     # network
8075     for (my $i = 0; $i < $MAX_NETS; $i++) {
8076         my $netname = "net$i";
8077         next if !$conf->{$netname};
8078         push @ret, $netname;
8079         last;
8080     }
8081
8082     return \@ret;
8083 }
8084
8085 sub device_bootorder {
8086     my ($conf) = @_;
8087
8088     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8089
8090     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8091
8092     my $bootorder = {};
8093     if (!defined($boot) || $boot->{legacy}) {
8094         $bootorder = bootorder_from_legacy($conf, $boot);
8095     } elsif ($boot->{order}) {
8096         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8097         for my $dev (PVE::Tools::split_list($boot->{order})) {
8098             $bootorder->{$dev} = $i++;
8099         }
8100     }
8101
8102     return $bootorder;
8103 }
8104
8105 sub register_qmeventd_handle {
8106     my ($vmid) = @_;
8107
8108     my $fh;
8109     my $peer = "/var/run/qmeventd.sock";
8110     my $count = 0;
8111
8112     for (;;) {
8113         $count++;
8114         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8115         last if $fh;
8116         if ($! != EINTR && $! != EAGAIN) {
8117             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8118         }
8119         if ($count > 4) {
8120             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8121               . "after $count retries\n";
8122         }
8123         usleep(25000);
8124     }
8125
8126     # send handshake to mark VM as backing up
8127     print $fh to_json({vzdump => {vmid => "$vmid"}});
8128
8129     # return handle to be closed later when inhibit is no longer required
8130     return $fh;
8131 }
8132
8133 # bash completion helper
8134
8135 sub complete_backup_archives {
8136     my ($cmdname, $pname, $cvalue) = @_;
8137
8138     my $cfg = PVE::Storage::config();
8139
8140     my $storeid;
8141
8142     if ($cvalue =~ m/^([^:]+):/) {
8143         $storeid = $1;
8144     }
8145
8146     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8147
8148     my $res = [];
8149     foreach my $id (keys %$data) {
8150         foreach my $item (@{$data->{$id}}) {
8151             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8152             push @$res, $item->{volid} if defined($item->{volid});
8153         }
8154     }
8155
8156     return $res;
8157 }
8158
8159 my $complete_vmid_full = sub {
8160     my ($running) = @_;
8161
8162     my $idlist = vmstatus();
8163
8164     my $res = [];
8165
8166     foreach my $id (keys %$idlist) {
8167         my $d = $idlist->{$id};
8168         if (defined($running)) {
8169             next if $d->{template};
8170             next if $running && $d->{status} ne 'running';
8171             next if !$running && $d->{status} eq 'running';
8172         }
8173         push @$res, $id;
8174
8175     }
8176     return $res;
8177 };
8178
8179 sub complete_vmid {
8180     return &$complete_vmid_full();
8181 }
8182
8183 sub complete_vmid_stopped {
8184     return &$complete_vmid_full(0);
8185 }
8186
8187 sub complete_vmid_running {
8188     return &$complete_vmid_full(1);
8189 }
8190
8191 sub complete_storage {
8192
8193     my $cfg = PVE::Storage::config();
8194     my $ids = $cfg->{ids};
8195
8196     my $res = [];
8197     foreach my $sid (keys %$ids) {
8198         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8199         next if !$ids->{$sid}->{content}->{images};
8200         push @$res, $sid;
8201     }
8202
8203     return $res;
8204 }
8205
8206 sub complete_migration_storage {
8207     my ($cmd, $param, $current_value, $all_args) = @_;
8208
8209     my $targetnode = @$all_args[1];
8210
8211     my $cfg = PVE::Storage::config();
8212     my $ids = $cfg->{ids};
8213
8214     my $res = [];
8215     foreach my $sid (keys %$ids) {
8216         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8217         next if !$ids->{$sid}->{content}->{images};
8218         push @$res, $sid;
8219     }
8220
8221     return $res;
8222 }
8223
8224 sub vm_is_paused {
8225     my ($vmid) = @_;
8226     my $qmpstatus = eval {
8227         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8228         mon_cmd($vmid, "query-status");
8229     };
8230     warn "$@\n" if $@;
8231     return $qmpstatus && $qmpstatus->{status} eq "paused";
8232 }
8233
8234 sub check_volume_storage_type {
8235     my ($storecfg, $vol) = @_;
8236
8237     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8238     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8239     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8240
8241     die "storage '$storeid' does not support content-type '$vtype'\n"
8242         if !$scfg->{content}->{$vtype};
8243
8244     return 1;
8245 }
8246
8247 1;