PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file(
 106     '/qemu-server/',
 107     \&parse_vm_config,
 108     \&write_vm_config
 109 );
 110
 111 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 112     description => "Some command save/restore state from this location.",
 113     type => 'string',
 114     maxLength => 128,
 115     optional => 1,
 116 });
 117
 118 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 119         description => "Specifies the Qemu machine type.",
 120         type => 'string',
 121         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 122         maxLength => 40,
 123         optional => 1,
 124 });
 125
 126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 243             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 244             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 245             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 246             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 247             ." a hardware RNG from the host.",
 248     },
 249     max_bytes => {
 250         type => 'integer',
 251         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 252             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 253             ." `0` to disable limiting (potentially dangerous!).",
 254         optional => 1,
 255
 256         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 257         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 258         # reading from /dev/urandom
 259         default => 1024,
 260     },
 261     period => {
 262         type => 'integer',
 263         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 264             ." the guest to retrieve another 'max_bytes' of entropy.",
 265         optional => 1,
 266         default => 1000,
 267     },
 268 };
 269
 270 my $meta_info_fmt = {
 271     'ctime' => {
 272         type => 'integer',
 273         description => "The guest creation timestamp as UNIX epoch time",
 274         minimum => 0,
 275         optional => 1,
 276     },
 277     'creation-qemu' => {
 278         type => 'string',
 279         description => "The QEMU (machine) version from the time this VM was created.",
 280         pattern => '\d+(\.\d+)+',
 281         optional => 1,
 282     },
 283 };
 284
 285 my $confdesc = {
 286     onboot => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Specifies whether a VM will be started during system bootup.",
 290         default => 0,
 291     },
 292     autostart => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Automatic restart after crash (currently ignored).",
 296         default => 0,
 297     },
 298     hotplug => {
 299         optional => 1,
 300         type => 'string', format => 'pve-hotplug-features',
 301         description => "Selectively enable hotplug features. This is a comma separated list of"
 302             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 303             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 304             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 305             ." windows > 7.",
 306         default => 'network,disk,usb',
 307     },
 308     reboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 312         default => 1,
 313     },
 314     lock => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Lock/unlock the VM.",
 318         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 319     },
 320     cpulimit => {
 321         optional => 1,
 322         type => 'number',
 323         description => "Limit of CPU usage.",
 324         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 325             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 326         minimum => 0,
 327         maximum => 128,
 328         default => 0,
 329     },
 330     cpuunits => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 334         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 335             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 336             ." weights of all the other running VMs.",
 337         minimum => 1,
 338         maximum => 262144,
 339         default => 'cgroup v1: 1024, cgroup v2: 100',
 340     },
 341     memory => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 345             ." you use the balloon device.",
 346         minimum => 16,
 347         default => 512,
 348     },
 349     balloon => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 353         minimum => 0,
 354     },
 355     shares => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 359             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 360             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 361         minimum => 0,
 362         maximum => 50000,
 363         default => 1000,
 364     },
 365     keyboard => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Keyboard layout for VNC server. This option is generally not required and"
 369         ." is often better handled from within the guest OS.",
 370         enum => PVE::Tools::kvmkeymaplist(),
 371         default => undef,
 372     },
 373     name => {
 374         optional => 1,
 375         type => 'string', format => 'dns-name',
 376         description => "Set a name for the VM. Only used on the configuration web interface.",
 377     },
 378     scsihw => {
 379         optional => 1,
 380         type => 'string',
 381         description => "SCSI controller model",
 382         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 383         default => 'lsi',
 384     },
 385     description => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Description for the VM. Shown in the web-interface VM's summary."
 389             ." This is saved as comment inside the configuration file.",
 390         maxLength => 1024 * 8,
 391     },
 392     ostype => {
 393         optional => 1,
 394         type => 'string',
 395         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 396         description => "Specify guest operating system.",
 397         verbose_description => <<EODESC,
 398 Specify guest operating system. This is used to enable special
 399 optimization/features for specific operating systems:
 400
 401 [horizontal]
 402 other;; unspecified OS
 403 wxp;; Microsoft Windows XP
 404 w2k;; Microsoft Windows 2000
 405 w2k3;; Microsoft Windows 2003
 406 w2k8;; Microsoft Windows 2008
 407 wvista;; Microsoft Windows Vista
 408 win7;; Microsoft Windows 7
 409 win8;; Microsoft Windows 8/2012/2012r2
 410 win10;; Microsoft Windows 10/2016/2019
 411 win11;; Microsoft Windows 11/2022
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6 - 5.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-boot',
 420         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 421             ." key or 'legacy=' is deprecated.",
 422     },
 423     bootdisk => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-bootdisk',
 426         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 427         pattern => '(ide|sata|scsi|virtio)\d+',
 428     },
 429     smp => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPUs. Please use option -sockets instead.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     sockets => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPU sockets.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     cores => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of cores per socket.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     numa => {
 451         optional => 1,
 452         type => 'boolean',
 453         description => "Enable/disable NUMA.",
 454         default => 0,
 455     },
 456     hugepages => {
 457         optional => 1,
 458         type => 'string',
 459         description => "Enable/disable hugepages memory.",
 460         enum => [qw(any 2 1024)],
 461     },
 462     keephugepages => {
 463         optional => 1,
 464         type => 'boolean',
 465         default => 0,
 466         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 467             ." after VM shutdown and can be used for subsequent starts.",
 468     },
 469     vcpus => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Number of hotplugged vcpus.",
 473         minimum => 1,
 474         default => 0,
 475     },
 476     acpi => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable ACPI.",
 480         default => 1,
 481     },
 482     agent => {
 483         optional => 1,
 484         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 485         type => 'string',
 486         format => $agent_fmt,
 487     },
 488     kvm => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable KVM hardware virtualization.",
 492         default => 1,
 493     },
 494     tdf => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable time drift fix.",
 498         default => 0,
 499     },
 500     localtime => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 504             ." the `ostype` indicates a Microsoft Windows OS.",
 505     },
 506     freeze => {
 507         optional => 1,
 508         type => 'boolean',
 509         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 510     },
 511     vga => {
 512         optional => 1,
 513         type => 'string', format => $vga_fmt,
 514         description => "Configure the VGA hardware.",
 515         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 516             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 517             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 518             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 519             ." display server. For win* OS you can select how many independent displays you want,"
 520             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 521             ." using a serial device as terminal.",
 522     },
 523     watchdog => {
 524         optional => 1,
 525         type => 'string', format => 'pve-qm-watchdog',
 526         description => "Create a virtual hardware watchdog device.",
 527         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 528             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 529             ." else the watchdog will reset the guest (or execute the respective action specified)",
 530     },
 531     startdate => {
 532         optional => 1,
 533         type => 'string',
 534         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 535         description => "Set the initial date of the real time clock. Valid format for date are:"
 536             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 537         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 538         default => 'now',
 539     },
 540     startup =>  get_standard_option('pve-startup-order'),
 541     template => {
 542         optional => 1,
 543         type => 'boolean',
 544         description => "Enable/disable Template.",
 545         default => 0,
 546     },
 547     args => {
 548         optional => 1,
 549         type => 'string',
 550         description => "Arbitrary arguments passed to kvm.",
 551         verbose_description => <<EODESCR,
 552 Arbitrary arguments passed to kvm, for example:
 553
 554 args: -no-reboot -no-hpet
 555
 556 NOTE: this option is for experts only.
 557 EODESCR
 558     },
 559     tablet => {
 560         optional => 1,
 561         type => 'boolean',
 562         default => 1,
 563         description => "Enable/disable the USB tablet device.",
 564         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 565             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 566             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 567             ." may consider disabling this to save some context switches. This is turned off by"
 568             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 569     },
 570     migrate_speed => {
 571         optional => 1,
 572         type => 'integer',
 573         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 574         minimum => 0,
 575         default => 0,
 576     },
 577     migrate_downtime => {
 578         optional => 1,
 579         type => 'number',
 580         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 581         minimum => 0,
 582         default => 0.1,
 583     },
 584     cdrom => {
 585         optional => 1,
 586         type => 'string', format => 'pve-qm-ide',
 587         typetext => '<volume>',
 588         description => "This is an alias for option -ide2",
 589     },
 590     cpu => {
 591         optional => 1,
 592         description => "Emulated CPU type.",
 593         type => 'string',
 594         format => 'pve-vm-cpu-conf',
 595     },
 596     parent => get_standard_option('pve-snapshot-name', {
 597         optional => 1,
 598         description => "Parent snapshot name. This is used internally, and should not be modified.",
 599     }),
 600     snaptime => {
 601         optional => 1,
 602         description => "Timestamp for snapshots.",
 603         type => 'integer',
 604         minimum => 0,
 605     },
 606     vmstate => {
 607         optional => 1,
 608         type => 'string', format => 'pve-volume-id',
 609         description => "Reference to a volume which stores the VM state. This is used internally"
 610             ." for snapshots.",
 611     },
 612     vmstatestorage => get_standard_option('pve-storage-id', {
 613         description => "Default storage for VM state volumes/files.",
 614         optional => 1,
 615     }),
 616     runningmachine => get_standard_option('pve-qemu-machine', {
 617         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 618             ." for snapshots.",
 619     }),
 620     runningcpu => {
 621         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 622             ." internally for snapshots.",
 623         optional => 1,
 624         type => 'string',
 625         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 626         format_description => 'QEMU -cpu parameter'
 627     },
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 645             ." remove disk operations.",
 646         default => 0,
 647     },
 648     bios => {
 649         optional => 1,
 650         type => 'string',
 651         enum => [ qw(seabios ovmf) ],
 652         description => "Select BIOS implementation.",
 653         default => 'seabios',
 654     },
 655     vmgenid => {
 656         type => 'string',
 657         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 658         format_description => 'UUID',
 659         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 660             ." to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 662             ." value identifier to the guest OS. This allows to notify the guest operating system"
 663             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 664             ." execution or creation from a template). The guest operating system notices the"
 665             ." change, and is then able to react as appropriate by marking its copies of"
 666             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 667             ."Note that auto-creation only works when done through API/CLI create or update methods"
 668             .", but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 682             ." the host.",
 683         optional => 1,
 684     },
 685     audio0 => {
 686         type => 'string',
 687         format => $audio_fmt,
 688         description => "Configure a audio device, useful in combination with QXL/Spice.",
 689         optional => 1
 690     },
 691     spice_enhancements => {
 692         type => 'string',
 693         format => $spice_enhancements_fmt,
 694         description => "Configure additional enhancements for SPICE.",
 695         optional => 1
 696     },
 697     tags => {
 698         type => 'string', format => 'pve-tag-list',
 699         description => 'Tags of the VM. This is only meta information.',
 700         optional => 1,
 701     },
 702     rng0 => {
 703         type => 'string',
 704         format => $rng_fmt,
 705         description => "Configure a VirtIO-based Random Number Generator.",
 706         optional => 1,
 707     },
 708     meta => {
 709         type => 'string',
 710         format => $meta_info_fmt,
 711         description => "Some (read-only) meta-information about this guest.",
 712         optional => 1,
 713     },
 714     affinity => {
 715         type => 'string', format => 'pve-cpuset',
 716         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 717         optional => 1,
 718     },
 719 };
 720
 721 my $cicustom_fmt = {
 722     meta => {
 723         type => 'string',
 724         optional => 1,
 725         description => 'Specify a custom file containing all meta data passed to the VM via"
 726             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 727         format => 'pve-volume-id',
 728         format_description => 'volume',
 729     },
 730     network => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all network data passed to the VM via'
 734             .' cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'Specify a custom file containing all user data passed to the VM via'
 742             .' cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     vendor => {
 747     type => 'string',
 748     optional => 1,
 749     description => 'Specify a custom file containing all vendor data passed to the VM via'
 750      .' cloud-init.',
 751     format => 'pve-volume-id',
 752     format_description => 'volume',
 753     },
 754 };
 755 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 756
 757 my $confdesc_cloudinit = {
 758     citype => {
 759         optional => 1,
 760         type => 'string',
 761         description => 'Specifies the cloud-init configuration format. The default depends on the'
 762             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 763             .' and `configdrive2` for windows.',
 764         enum => ['configdrive2', 'nocloud', 'opennebula'],
 765     },
 766     ciuser => {
 767         optional => 1,
 768         type => 'string',
 769         description => "cloud-init: User name to change ssh keys and password for instead of the"
 770             ." image's configured default user.",
 771     },
 772     cipassword => {
 773         optional => 1,
 774         type => 'string',
 775         description => 'cloud-init: Password to assign the user. Using this is generally not'
 776             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 777             .' support hashed passwords.',
 778     },
 779     cicustom => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Specify custom files to replace the automatically generated'
 783             .' ones at start.',
 784         format => 'pve-qm-cicustom',
 785     },
 786     searchdomain => {
 787         optional => 1,
 788         type => 'string',
 789         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 790             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 791             .' are set.',
 792     },
 793     nameserver => {
 794         optional => 1,
 795         type => 'string', format => 'address-list',
 796         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 797             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 798             .' are set.',
 799     },
 800     sshkeys => {
 801         optional => 1,
 802         type => 'string',
 803         format => 'urlencoded',
 804         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 805     },
 806 };
 807
 808 # what about other qemu settings ?
 809 #cpu => 'string',
 810 #machine => 'string',
 811 #fda => 'file',
 812 #fdb => 'file',
 813 #mtdblock => 'file',
 814 #sd => 'file',
 815 #pflash => 'file',
 816 #snapshot => 'bool',
 817 #bootp => 'file',
 818 ##tftp => 'dir',
 819 ##smb => 'dir',
 820 #kernel => 'file',
 821 #append => 'string',
 822 #initrd => 'file',
 823 ##soundhw => 'string',
 824
 825 while (my ($k, $v) = each %$confdesc) {
 826     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 827 }
 828
 829 my $MAX_USB_DEVICES = 14;
 830 my $MAX_NETS = 32;
 831 my $MAX_SERIAL_PORTS = 4;
 832 my $MAX_PARALLEL_PORTS = 3;
 833 my $MAX_NUMA = 8;
 834
 835 my $numa_fmt = {
 836     cpus => {
 837         type => "string",
 838         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 839         description => "CPUs accessing this NUMA node.",
 840         format_description => "id[-id];...",
 841     },
 842     memory => {
 843         type => "number",
 844         description => "Amount of memory this NUMA node provides.",
 845         optional => 1,
 846     },
 847     hostnodes => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "Host NUMA nodes to use.",
 851         format_description => "id[-id];...",
 852         optional => 1,
 853     },
 854     policy => {
 855         type => 'string',
 856         enum => [qw(preferred bind interleave)],
 857         description => "NUMA allocation policy.",
 858         optional => 1,
 859     },
 860 };
 861 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 862 my $numadesc = {
 863     optional => 1,
 864     type => 'string', format => $numa_fmt,
 865     description => "NUMA topology.",
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 868
 869 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 870     $confdesc->{"numa$i"} = $numadesc;
 871 }
 872
 873 my $nic_model_list = [
 874     'e1000',
 875     'e1000-82540em',
 876     'e1000-82544gc',
 877     'e1000-82545em',
 878     'e1000e',
 879     'i82551',
 880     'i82557b',
 881     'i82559er',
 882     'ne2k_isa',
 883     'ne2k_pci',
 884     'pcnet',
 885     'rtl8139',
 886     'virtio',
 887     'vmxnet3',
 888 ];
 889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 890
 891 my $net_fmt_bridge_descr = <<__EOD__;
 892 Bridge to attach the network device to. The Proxmox VE standard bridge
 893 is called 'vmbr0'.
 894
 895 If you do not specify a bridge, we create a kvm user (NATed) network
 896 device, which provides DHCP and DNS services. The following addresses
 897 are used:
 898
 899  10.0.2.2   Gateway
 900  10.0.2.3   DNS Server
 901  10.0.2.4   SMB Server
 902
 903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 904 __EOD__
 905
 906 my $net_fmt = {
 907     macaddr  => get_standard_option('mac-addr', {
 908         description => "MAC address. That address must be unique withing your network. This is"
 909             ." automatically generated if not specified.",
 910     }),
 911     model => {
 912         type => 'string',
 913         description => "Network Card Model. The 'virtio' model provides the best performance with"
 914             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 915             ." best to use 'e1000'.",
 916         enum => $nic_model_list,
 917         default_key => 1,
 918     },
 919     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 920     bridge => get_standard_option('pve-bridge-id', {
 921         description => $net_fmt_bridge_descr,
 922         optional => 1,
 923     }),
 924     queues => {
 925         type => 'integer',
 926         minimum => 0, maximum => 64,
 927         description => 'Number of packet queues to be used on the device.',
 928         optional => 1,
 929     },
 930     rate => {
 931         type => 'number',
 932         minimum => 0,
 933         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 934         optional => 1,
 935     },
 936     tag => {
 937         type => 'integer',
 938         minimum => 1, maximum => 4094,
 939         description => 'VLAN tag to apply to packets on this interface.',
 940         optional => 1,
 941     },
 942     trunks => {
 943         type => 'string',
 944         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 945         description => 'VLAN trunks to pass through this interface.',
 946         format_description => 'vlanid[;vlanid...]',
 947         optional => 1,
 948     },
 949     firewall => {
 950         type => 'boolean',
 951         description => 'Whether this interface should be protected by the firewall.',
 952         optional => 1,
 953     },
 954     link_down => {
 955         type => 'boolean',
 956         description => 'Whether this interface should be disconnected (like pulling the plug).',
 957         optional => 1,
 958     },
 959     mtu => {
 960         type => 'integer',
 961         minimum => 1, maximum => 65520,
 962         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 963         optional => 1,
 964     },
 965 };
 966
 967 my $netdesc = {
 968     optional => 1,
 969     type => 'string', format => $net_fmt,
 970     description => "Specify network devices.",
 971 };
 972
 973 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 974
 975 my $ipconfig_fmt = {
 976     ip => {
 977         type => 'string',
 978         format => 'pve-ipv4-config',
 979         format_description => 'IPv4Format/CIDR',
 980         description => 'IPv4 address in CIDR format.',
 981         optional => 1,
 982         default => 'dhcp',
 983     },
 984     gw => {
 985         type => 'string',
 986         format => 'ipv4',
 987         format_description => 'GatewayIPv4',
 988         description => 'Default gateway for IPv4 traffic.',
 989         optional => 1,
 990         requires => 'ip',
 991     },
 992     ip6 => {
 993         type => 'string',
 994         format => 'pve-ipv6-config',
 995         format_description => 'IPv6Format/CIDR',
 996         description => 'IPv6 address in CIDR format.',
 997         optional => 1,
 998         default => 'dhcp',
 999     },
1000     gw6 => {
1001         type => 'string',
1002         format => 'ipv6',
1003         format_description => 'GatewayIPv6',
1004         description => 'Default gateway for IPv6 traffic.',
1005         optional => 1,
1006         requires => 'ip6',
1007     },
1008 };
1009 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1011     optional => 1,
1012     type => 'string', format => 'pve-qm-ipconfig',
1013     description => <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1015
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1017
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1022
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1024 dhcp on IPv4.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1028
1029 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1030     $confdesc->{"net$i"} = $netdesc;
1031     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1032 }
1033
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1039 sub pve_verify_cpuset {
1040     my ($set_text, $noerr) = @_;
1041
1042     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1043
1044     if ($@) {
1045         return if $noerr;
1046         die "unable to parse cpuset option\n";
1047     }
1048
1049     return PVE::CpuSet->new($members)->short_string();
1050 }
1051
1052 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1053 sub verify_volume_id_or_qm_path {
1054     my ($volid, $noerr) = @_;
1055
1056     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1057
1058     return verify_volume_id_or_absolute_path($volid, $noerr);
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1062 sub verify_volume_id_or_absolute_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid =~ m|^/|;
1066
1067     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1068     if ($@) {
1069         return if $noerr;
1070         die $@;
1071     }
1072     return $volid;
1073 }
1074
1075 my $usb_fmt = {
1076     host => {
1077         default_key => 1,
1078         type => 'string', format => 'pve-qm-usb-device',
1079         format_description => 'HOSTUSBDEVICE|spice',
1080         description => <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1082
1083  'bus-port(.port)*' (decimal numbers) or
1084  'vendor_id:product_id' (hexadeciaml numbers) or
1085  'spice'
1086
1087 You can use the 'lsusb -t' command to list existing usb devices.
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1091
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1093 EODESCR
1094     },
1095     usb3 => {
1096         optional => 1,
1097         type => 'boolean',
1098         description => "Specifies whether if given host option is a USB3 device or port."
1099             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100             ." is irrelevant (all devices are plugged into a xhci controller).",
1101         default => 0,
1102     },
1103 };
1104
1105 my $usbdesc = {
1106     optional => 1,
1107     type => 'string', format => $usb_fmt,
1108     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109         ." l26 or windows > 7, n can be up to 14).",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1112
1113 my $serialdesc = {
1114         optional => 1,
1115         type => 'string',
1116         pattern => '(/dev/.+|socket)',
1117         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1118         verbose_description =>  <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1122
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1125
1126 CAUTION: Experimental! User reported problems with this option.
1127 EODESCR
1128 };
1129
1130 my $paralleldesc= {
1131         optional => 1,
1132         type => 'string',
1133         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1134         description =>  "Map host parallel devices (n is 0 to 2).",
1135         verbose_description =>  <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1137
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1140
1141 CAUTION: Experimental! User reported problems with this option.
1142 EODESCR
1143 };
1144
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1146     $confdesc->{"parallel$i"} = $paralleldesc;
1147 }
1148
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1150     $confdesc->{"serial$i"} = $serialdesc;
1151 }
1152
1153 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1154     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1155 }
1156
1157 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1158     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1159 }
1160
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1162     $confdesc->{"usb$i"} = $usbdesc;
1163 }
1164
1165 my $boot_fmt = {
1166     legacy => {
1167         optional => 1,
1168         default_key => 1,
1169         type => 'string',
1170         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171                      . " Deprecated, use 'order=' instead.",
1172         pattern => '[acdn]{1,4}',
1173         format_description => "[acdn]{1,4}",
1174
1175         # note: this is also the fallback if boot: is not given at all
1176         default => 'cdn',
1177     },
1178     order => {
1179         optional => 1,
1180         type => 'string',
1181         format => 'pve-qm-bootdev-list',
1182         format_description => "device[;device...]",
1183         description => <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1185
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1189
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1193
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1195 EODESC
1196     },
1197 };
1198 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1199
1200 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1201 sub verify_bootdev {
1202     my ($dev, $noerr) = @_;
1203
1204     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1206
1207     my $check = sub {
1208         my ($base) = @_;
1209         return 0 if $dev !~ m/^$base\d+$/;
1210         return 0 if !$confdesc->{$dev};
1211         return 1;
1212     };
1213
1214     return $dev if $check->("net");
1215     return $dev if $check->("usb");
1216     return $dev if $check->("hostpci");
1217
1218     return if $noerr;
1219     die "invalid boot device '$dev'\n";
1220 }
1221
1222 sub print_bootorder {
1223     my ($devs) = @_;
1224     return "" if !@$devs;
1225     my $data = { order => join(';', @$devs) };
1226     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1227 }
1228
1229 my $kvm_api_version = 0;
1230
1231 sub kvm_version {
1232     return $kvm_api_version if $kvm_api_version;
1233
1234     open my $fh, '<', '/dev/kvm' or return;
1235
1236     # 0xae00 => KVM_GET_API_VERSION
1237     $kvm_api_version = ioctl($fh, 0xae00, 0);
1238     close($fh);
1239
1240     return $kvm_api_version;
1241 }
1242
1243 my $kvm_user_version = {};
1244 my $kvm_mtime = {};
1245
1246 sub kvm_user_version {
1247     my ($binary) = @_;
1248
1249     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1250     my $st = stat($binary);
1251
1252     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254         $cachedmtime == $st->mtime;
1255
1256     $kvm_user_version->{$binary} = 'unknown';
1257     $kvm_mtime->{$binary} = $st->mtime;
1258
1259     my $code = sub {
1260         my $line = shift;
1261         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262             $kvm_user_version->{$binary} = $2;
1263         }
1264     };
1265
1266     eval { run_command([$binary, '--version'], outfunc => $code); };
1267     warn $@ if $@;
1268
1269     return $kvm_user_version->{$binary};
1270
1271 }
1272 my sub extract_version {
1273     my ($machine_type, $version) = @_;
1274     $version = kvm_user_version() if !defined($version);
1275     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1276 }
1277
1278 sub kernel_has_vhost_net {
1279     return -c '/dev/vhost-net';
1280 }
1281
1282 sub option_exists {
1283     my $key = shift;
1284     return defined($confdesc->{$key});
1285 }
1286
1287 my $cdrom_path;
1288 sub get_cdrom_path {
1289
1290     return  $cdrom_path if $cdrom_path;
1291
1292     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1293     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1294     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1295 }
1296
1297 sub get_iso_path {
1298     my ($storecfg, $vmid, $cdrom) = @_;
1299
1300     if ($cdrom eq 'cdrom') {
1301         return get_cdrom_path();
1302     } elsif ($cdrom eq 'none') {
1303         return '';
1304     } elsif ($cdrom =~ m|^/|) {
1305         return $cdrom;
1306     } else {
1307         return PVE::Storage::path($storecfg, $cdrom);
1308     }
1309 }
1310
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id {
1313     my ($vmid, $file, $media) = @_;
1314
1315      if (!($file eq 'none' || $file eq 'cdrom' ||
1316           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1317
1318         return if $file =~ m|/|;
1319
1320         if ($media && $media eq 'cdrom') {
1321             $file = "local:iso/$file";
1322         } else {
1323             $file = "local:$vmid/$file";
1324         }
1325     }
1326
1327     return $file;
1328 }
1329
1330 sub verify_media_type {
1331     my ($opt, $vtype, $media) = @_;
1332
1333     return if !$media;
1334
1335     my $etype;
1336     if ($media eq 'disk') {
1337         $etype = 'images';
1338     } elsif ($media eq 'cdrom') {
1339         $etype = 'iso';
1340     } else {
1341         die "internal error";
1342     }
1343
1344     return if ($vtype eq $etype);
1345
1346     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1347 }
1348
1349 sub cleanup_drive_path {
1350     my ($opt, $storecfg, $drive) = @_;
1351
1352     # try to convert filesystem paths to volume IDs
1353
1354     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1355         ($drive->{file} !~ m|^/dev/.+|) &&
1356         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1357         ($drive->{file} !~ m/^\d+$/)) {
1358         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1359         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1360             if !$vtype;
1361         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1362         verify_media_type($opt, $vtype, $drive->{media});
1363         $drive->{file} = $volid;
1364     }
1365
1366     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1367 }
1368
1369 sub parse_hotplug_features {
1370     my ($data) = @_;
1371
1372     my $res = {};
1373
1374     return $res if $data eq '0';
1375
1376     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1377
1378     foreach my $feature (PVE::Tools::split_list($data)) {
1379         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1380             $res->{$1} = 1;
1381         } else {
1382             die "invalid hotplug feature '$feature'\n";
1383         }
1384     }
1385     return $res;
1386 }
1387
1388 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1389 sub pve_verify_hotplug_features {
1390     my ($value, $noerr) = @_;
1391
1392     return $value if parse_hotplug_features($value);
1393
1394     return if $noerr;
1395
1396     die "unable to parse hotplug option\n";
1397 }
1398
1399 sub scsi_inquiry {
1400     my($fh, $noerr) = @_;
1401
1402     my $SG_IO = 0x2285;
1403     my $SG_GET_VERSION_NUM = 0x2282;
1404
1405     my $versionbuf = "\x00" x 8;
1406     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1407     if (!$ret) {
1408         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1409         return;
1410     }
1411     my $version = unpack("I", $versionbuf);
1412     if ($version < 30000) {
1413         die "scsi generic interface too old\n"  if !$noerr;
1414         return;
1415     }
1416
1417     my $buf = "\x00" x 36;
1418     my $sensebuf = "\x00" x 8;
1419     my $cmd = pack("C x3 C x1", 0x12, 36);
1420
1421     # see /usr/include/scsi/sg.h
1422     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1423
1424     my $packet = pack(
1425         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1426     );
1427
1428     $ret = ioctl($fh, $SG_IO, $packet);
1429     if (!$ret) {
1430         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1431         return;
1432     }
1433
1434     my @res = unpack($sg_io_hdr_t, $packet);
1435     if ($res[17] || $res[18]) {
1436         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my $res = {};
1441     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1442
1443     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1444     $res->{type} &= 0x1F;
1445
1446     return $res;
1447 }
1448
1449 sub path_is_scsi {
1450     my ($path) = @_;
1451
1452     my $fh = IO::File->new("+<$path") || return;
1453     my $res = scsi_inquiry($fh, 1);
1454     close($fh);
1455
1456     return $res;
1457 }
1458
1459 sub print_tabletdevice_full {
1460     my ($conf, $arch) = @_;
1461
1462     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1463
1464     # we use uhci for old VMs because tablet driver was buggy in older qemu
1465     my $usbbus;
1466     if ($q35 || $arch eq 'aarch64') {
1467         $usbbus = 'ehci';
1468     } else {
1469         $usbbus = 'uhci';
1470     }
1471
1472     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1473 }
1474
1475 sub print_keyboarddevice_full {
1476     my ($conf, $arch) = @_;
1477
1478     return if $arch ne 'aarch64';
1479
1480     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1481 }
1482
1483 my sub get_drive_id {
1484     my ($drive) = @_;
1485     return "$drive->{interface}$drive->{index}";
1486 }
1487
1488 sub print_drivedevice_full {
1489     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1490
1491     my $device = '';
1492     my $maxdev = 0;
1493
1494     my $drive_id = get_drive_id($drive);
1495     if ($drive->{interface} eq 'virtio') {
1496         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1497         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1499     } elsif ($drive->{interface} eq 'scsi') {
1500
1501         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1502         my $unit = $drive->{index} % $maxdev;
1503         my $devicetype = 'hd';
1504         my $path = '';
1505         if (drive_is_cdrom($drive)) {
1506             $devicetype = 'cd';
1507         } else {
1508             if ($drive->{file} =~ m|^/|) {
1509                 $path = $drive->{file};
1510                 if (my $info = path_is_scsi($path)) {
1511                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1512                         $devicetype = 'block';
1513                     } elsif ($info->{type} == 1) { # tape
1514                         $devicetype = 'generic';
1515                     }
1516                 }
1517             } else {
1518                  $path = PVE::Storage::path($storecfg, $drive->{file});
1519             }
1520
1521             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522             my $version = extract_version($machine_type, kvm_user_version());
1523             if ($path =~ m/^iscsi\:\/\// &&
1524                !min_version($version, 4, 1)) {
1525                 $devicetype = 'generic';
1526             }
1527         }
1528
1529         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1531         } else {
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533                 .",lun=$drive->{index}";
1534         }
1535         $device .= ",drive=drive-$drive_id,id=$drive_id";
1536
1537         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538             $device .= ",rotation_rate=1";
1539         }
1540         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541
1542     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1543         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1544         my $controller = int($drive->{index} / $maxdev);
1545         my $unit = $drive->{index} % $maxdev;
1546         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1547
1548         $device = "ide-$devicetype";
1549         if ($drive->{interface} eq 'ide') {
1550             $device .= ",bus=ide.$controller,unit=$unit";
1551         } else {
1552             $device .= ",bus=ahci$controller.$unit";
1553         }
1554         $device .= ",drive=drive-$drive_id,id=$drive_id";
1555
1556         if ($devicetype eq 'hd') {
1557             if (my $model = $drive->{model}) {
1558                 $model = URI::Escape::uri_unescape($model);
1559                 $device .= ",model=$model";
1560             }
1561             if ($drive->{ssd}) {
1562                 $device .= ",rotation_rate=1";
1563             }
1564         }
1565         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1566     } elsif ($drive->{interface} eq 'usb') {
1567         die "implement me";
1568         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1569     } else {
1570         die "unsupported interface type";
1571     }
1572
1573     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1574
1575     if (my $serial = $drive->{serial}) {
1576         $serial = URI::Escape::uri_unescape($serial);
1577         $device .= ",serial=$serial";
1578     }
1579
1580
1581     return $device;
1582 }
1583
1584 sub get_initiator_name {
1585     my $initiator;
1586
1587     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1588     while (defined(my $line = <$fh>)) {
1589         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1590         $initiator = $1;
1591         last;
1592     }
1593     $fh->close();
1594
1595     return $initiator;
1596 }
1597
1598 sub print_drive_commandline_full {
1599     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1600
1601     my $path;
1602     my $volid = $drive->{file};
1603     my $format = $drive->{format};
1604     my $drive_id = get_drive_id($drive);
1605
1606     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1607     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1608
1609     if (drive_is_cdrom($drive)) {
1610         $path = get_iso_path($storecfg, $vmid, $volid);
1611         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1612     } else {
1613         if ($storeid) {
1614             $path = PVE::Storage::path($storecfg, $volid);
1615             $format //= qemu_img_format($scfg, $volname);
1616         } else {
1617             $path = $volid;
1618             $format //= "raw";
1619         }
1620    }
1621
1622    my $is_rbd = $path =~ m/^rbd:/;
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1628     }
1629
1630     # snapshot only accepts on|off
1631     if (defined($drive->{snapshot})) {
1632         my $v = $drive->{snapshot} ? 'on' : 'off';
1633         $opts .= ",snapshot=$v";
1634     }
1635
1636     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1638     }
1639
1640     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1641         my ($dir, $qmpname) = @$type;
1642         if (my $v = $drive->{"mbps$dir"}) {
1643             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1644         }
1645         if (my $v = $drive->{"mbps${dir}_max"}) {
1646             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1647         }
1648         if (my $v = $drive->{"bps${dir}_max_length"}) {
1649             $opts .= ",throttling.bps$qmpname-max-length=$v";
1650         }
1651         if (my $v = $drive->{"iops${dir}"}) {
1652             $opts .= ",throttling.iops$qmpname=$v";
1653         }
1654         if (my $v = $drive->{"iops${dir}_max"}) {
1655             $opts .= ",throttling.iops$qmpname-max=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}_max_length"}) {
1658             $opts .= ",throttling.iops$qmpname-max-length=$v";
1659         }
1660     }
1661
1662     if ($pbs_name) {
1663         $format = "rbd" if $is_rbd;
1664         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1665             if !$format;
1666         $opts .= ",format=alloc-track,file.driver=$format";
1667     } elsif ($format) {
1668         $opts .= ",format=$format";
1669     }
1670
1671     my $cache_direct = 0;
1672
1673     if (my $cache = $drive->{cache}) {
1674         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1676         $opts .= ",cache=none";
1677         $cache_direct = 1;
1678     }
1679
1680     # io_uring with cache mode writeback or writethrough on krbd will hang...
1681     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1682
1683     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684     # sometimes, just plain disable...
1685     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1686
1687     # io_uring causes problems when used with CIFS since kernel 5.15
1688     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1690
1691     if (!$drive->{aio}) {
1692         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693             # io_uring supports all cache modes
1694             $opts .= ",aio=io_uring";
1695         } else {
1696             # aio native works only with O_DIRECT
1697             if($cache_direct) {
1698                 $opts .= ",aio=native";
1699             } else {
1700                 $opts .= ",aio=threads";
1701             }
1702         }
1703     }
1704
1705     if (!drive_is_cdrom($drive)) {
1706         my $detectzeroes;
1707         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1708             $detectzeroes = 'off';
1709         } elsif ($drive->{discard}) {
1710             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1711         } else {
1712             # This used to be our default with discard not being specified:
1713             $detectzeroes = 'on';
1714         }
1715
1716         # note: 'detect-zeroes' works per blockdev and we want it to persist
1717         # after the alloc-track is removed, so put it on 'file' directly
1718         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1719         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1720     }
1721
1722     if ($pbs_name) {
1723         $opts .= ",backing=$pbs_name";
1724         $opts .= ",auto-remove=on";
1725     }
1726
1727     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728     my $file_param = "file";
1729     if ($pbs_name) {
1730         # non-rbd drivers require the underlying file to be a seperate block
1731         # node, so add a second .file indirection
1732         $file_param .= ".file" if !$is_rbd;
1733         $file_param .= ".filename";
1734     }
1735     my $pathinfo = $path ? "$file_param=$path," : '';
1736
1737     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1738 }
1739
1740 sub print_pbs_blockdev {
1741     my ($pbs_conf, $pbs_name) = @_;
1742     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743     $blockdev .= ",repository=$pbs_conf->{repository}";
1744     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1745     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746     $blockdev .= ",archive=$pbs_conf->{archive}";
1747     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1748     return $blockdev;
1749 }
1750
1751 sub print_netdevice_full {
1752     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1753
1754     my $device = $net->{model};
1755     if ($net->{model} eq 'virtio') {
1756          $device = 'virtio-net-pci';
1757      };
1758
1759     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1760     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1762         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763         # and out of each queue plus one config interrupt and control vector queue
1764         my $vectors = $net->{queues} * 2 + 2;
1765         $tmpstr .= ",vectors=$vectors,mq=on";
1766         if (min_version($machine_version, 7, 1)) {
1767             $tmpstr .= ",packed=on";
1768         }
1769     }
1770
1771     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1772         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1773     }
1774
1775     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1776
1777     if (my $mtu = $net->{mtu}) {
1778         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1779             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1780             if ($mtu == 1) {
1781                  $mtu = $bridge_mtu;
1782             } elsif ($mtu < 576) {
1783                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784             } elsif ($mtu > $bridge_mtu) {
1785                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1786             }
1787             $tmpstr .= ",host_mtu=$mtu";
1788         } else {
1789             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1790         }
1791     }
1792
1793     if ($use_old_bios_files) {
1794         my $romfile;
1795         if ($device eq 'virtio-net-pci') {
1796             $romfile = 'pxe-virtio.rom';
1797         } elsif ($device eq 'e1000') {
1798             $romfile = 'pxe-e1000.rom';
1799         } elsif ($device eq 'e1000e') {
1800             $romfile = 'pxe-e1000e.rom';
1801         } elsif ($device eq 'ne2k') {
1802             $romfile = 'pxe-ne2k_pci.rom';
1803         } elsif ($device eq 'pcnet') {
1804             $romfile = 'pxe-pcnet.rom';
1805         } elsif ($device eq 'rtl8139') {
1806             $romfile = 'pxe-rtl8139.rom';
1807         }
1808         $tmpstr .= ",romfile=$romfile" if $romfile;
1809     }
1810
1811     return $tmpstr;
1812 }
1813
1814 sub print_netdev_full {
1815     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1816
1817     my $i = '';
1818     if ($netid =~ m/^net(\d+)$/) {
1819         $i = int($1);
1820     }
1821
1822     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1823
1824     my $ifname = "tap${vmid}i$i";
1825
1826     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827     die "interface name '$ifname' is too long (max 15 character)\n"
1828         if length($ifname) >= 16;
1829
1830     my $vhostparam = '';
1831     if (is_native($arch)) {
1832         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1833     }
1834
1835     my $vmname = $conf->{name} || "vm$vmid";
1836
1837     my $netdev = "";
1838     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1839
1840     if ($net->{bridge}) {
1841         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1843     } else {
1844         $netdev = "type=user,id=$netid,hostname=$vmname";
1845     }
1846
1847     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1848
1849     return $netdev;
1850 }
1851
1852 my $vga_map = {
1853     'cirrus' => 'cirrus-vga',
1854     'std' => 'VGA',
1855     'vmware' => 'vmware-svga',
1856     'virtio' => 'virtio-vga',
1857     'virtio-gl' => 'virtio-vga-gl',
1858 };
1859
1860 sub print_vga_device {
1861     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1862
1863     my $type = $vga_map->{$vga->{type}};
1864     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865         $type = 'virtio-gpu';
1866     }
1867     my $vgamem_mb = $vga->{memory};
1868
1869     my $max_outputs = '';
1870     if ($qxlnum) {
1871         $type = $id ? 'qxl' : 'qxl-vga';
1872
1873         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1874             # set max outputs so linux can have up to 4 qxl displays with one device
1875             if (min_version($machine_version, 4, 1)) {
1876                 $max_outputs = ",max_outputs=4";
1877             }
1878         }
1879     }
1880
1881     die "no devicetype for $vga->{type}\n" if !$type;
1882
1883     my $memory = "";
1884     if ($vgamem_mb) {
1885         if ($vga->{type} =~ /^virtio/) {
1886             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1887             $memory = ",max_hostmem=$bytes";
1888         } elsif ($qxlnum) {
1889             # from https://www.spice-space.org/multiple-monitors.html
1890             $memory = ",vgamem_mb=$vga->{memory}";
1891             my $ram = $vgamem_mb * 4;
1892             my $vram = $vgamem_mb * 2;
1893             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1894         } else {
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896         }
1897     } elsif ($qxlnum && $id) {
1898         $memory = ",ram_size=67108864,vram_size=33554432";
1899     }
1900
1901     my $edidoff = "";
1902     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1903         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1904     }
1905
1906     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1907     my $vgaid = "vga" . ($id // '');
1908     my $pciaddr;
1909     if ($q35 && $vgaid eq 'vga') {
1910         # the first display uses pcie.0 bus on q35 machines
1911         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1912     } else {
1913         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1914     }
1915
1916     if ($vga->{type} eq 'virtio-gl') {
1917         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1920
1921         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1923     }
1924
1925     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1926 }
1927
1928 sub parse_number_sets {
1929     my ($set) = @_;
1930     my $res = [];
1931     foreach my $part (split(/;/, $set)) {
1932         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934             push @$res, [ $1, $2 ];
1935         } else {
1936             die "invalid range: $part\n";
1937         }
1938     }
1939     return $res;
1940 }
1941
1942 sub parse_numa {
1943     my ($data) = @_;
1944
1945     my $res = parse_property_string($numa_fmt, $data);
1946     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1947     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1948     return $res;
1949 }
1950
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1952 sub parse_net {
1953     my ($data, $disable_mac_autogen) = @_;
1954
1955     my $res = eval { parse_property_string($net_fmt, $data) };
1956     if ($@) {
1957         warn $@;
1958         return;
1959     }
1960     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1961         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1962         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1963     }
1964     return $res;
1965 }
1966
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig {
1969     my ($data) = @_;
1970
1971     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1972     if ($@) {
1973         warn $@;
1974         return;
1975     }
1976
1977     if ($res->{gw} && !$res->{ip}) {
1978         warn 'gateway specified without specifying an IP address';
1979         return;
1980     }
1981     if ($res->{gw6} && !$res->{ip6}) {
1982         warn 'IPv6 gateway specified without specifying an IPv6 address';
1983         return;
1984     }
1985     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1986         warn 'gateway specified together with DHCP';
1987         return;
1988     }
1989     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1990         # gw6 + auto/dhcp
1991         warn "IPv6 gateway specified together with $res->{ip6} address";
1992         return;
1993     }
1994
1995     if (!$res->{ip} && !$res->{ip6}) {
1996         return { ip => 'dhcp', ip6 => 'dhcp' };
1997     }
1998
1999     return $res;
2000 }
2001
2002 sub print_net {
2003     my $net = shift;
2004
2005     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2006 }
2007
2008 sub add_random_macs {
2009     my ($settings) = @_;
2010
2011     foreach my $opt (keys %$settings) {
2012         next if $opt !~ m/^net(\d+)$/;
2013         my $net = parse_net($settings->{$opt});
2014         next if !$net;
2015         $settings->{$opt} = print_net($net);
2016     }
2017 }
2018
2019 sub vm_is_volid_owner {
2020     my ($storecfg, $vmid, $volid) = @_;
2021
2022     if ($volid !~  m|^/|) {
2023         my ($path, $owner);
2024         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2025         if ($owner && ($owner == $vmid)) {
2026             return 1;
2027         }
2028     }
2029
2030     return;
2031 }
2032
2033 sub vmconfig_register_unused_drive {
2034     my ($storecfg, $vmid, $conf, $drive) = @_;
2035
2036     if (drive_is_cloudinit($drive)) {
2037         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2038         warn $@ if $@;
2039         delete $conf->{cloudinit};
2040     } elsif (!drive_is_cdrom($drive)) {
2041         my $volid = $drive->{file};
2042         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2043             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2044         }
2045     }
2046 }
2047
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2049 my $smbios1_fmt = {
2050     uuid => {
2051         type => 'string',
2052         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053         format_description => 'UUID',
2054         description => "Set SMBIOS1 UUID.",
2055         optional => 1,
2056     },
2057     version => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 version.",
2062         optional => 1,
2063     },
2064     serial => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 serial number.",
2069         optional => 1,
2070     },
2071     manufacturer => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 manufacturer.",
2076         optional => 1,
2077     },
2078     product => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 product ID.",
2083         optional => 1,
2084     },
2085     sku => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 SKU string.",
2090         optional => 1,
2091     },
2092     family => {
2093         type => 'string',
2094         pattern => '[A-Za-z0-9+\/]+={0,2}',
2095         format_description => 'Base64 encoded string',
2096         description => "Set SMBIOS1 family string.",
2097         optional => 1,
2098     },
2099     base64 => {
2100         type => 'boolean',
2101         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2102         optional => 1,
2103     },
2104 };
2105
2106 sub parse_smbios1 {
2107     my ($data) = @_;
2108
2109     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2110     warn $@ if $@;
2111     return $res;
2112 }
2113
2114 sub print_smbios1 {
2115     my ($smbios1) = @_;
2116     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2117 }
2118
2119 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2120
2121 sub parse_watchdog {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub parse_guest_agent {
2132     my ($conf) = @_;
2133
2134     return {} if !defined($conf->{agent});
2135
2136     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2137     warn $@ if $@;
2138
2139     # if the agent is disabled ignore the other potentially set properties
2140     return {} if !$res->{enabled};
2141     return $res;
2142 }
2143
2144 sub get_qga_key {
2145     my ($conf, $key) = @_;
2146     return undef if !defined($conf->{agent});
2147
2148     my $agent = parse_guest_agent($conf);
2149     return $agent->{$key};
2150 }
2151
2152 sub parse_vga {
2153     my ($value) = @_;
2154
2155     return {} if !$value;
2156     my $res = eval { parse_property_string($vga_fmt, $value) };
2157     warn $@ if $@;
2158     return $res;
2159 }
2160
2161 sub parse_rng {
2162     my ($value) = @_;
2163
2164     return if !$value;
2165
2166     my $res = eval { parse_property_string($rng_fmt, $value) };
2167     warn $@ if $@;
2168     return $res;
2169 }
2170
2171 sub parse_meta_info {
2172     my ($value) = @_;
2173
2174     return if !$value;
2175
2176     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2177     warn $@ if $@;
2178     return $res;
2179 }
2180
2181 sub new_meta_info_string {
2182     my () = @_; # for now do not allow to override any value
2183
2184     return PVE::JSONSchema::print_property_string(
2185         {
2186             'creation-qemu' => kvm_user_version(),
2187             ctime => "". int(time()),
2188         },
2189         $meta_info_fmt
2190     );
2191 }
2192
2193 sub qemu_created_version_fixups {
2194     my ($conf, $forcemachine, $kvmver) = @_;
2195
2196     my $meta = parse_meta_info($conf->{meta}) // {};
2197     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2198
2199     # check if we need to apply some handling for VMs that always use the latest machine version but
2200     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201     # would be required (we do not want to pin machine version for non-windows OS type)
2202     if (
2203         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2207     ) {
2208         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2209         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2210             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211             # and thus with the predictable interface naming of systemd
2212             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2213         }
2214     }
2215     return;
2216 }
2217
2218 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2219 sub verify_usb_device {
2220     my ($value, $noerr) = @_;
2221
2222     return $value if parse_usb_device($value);
2223
2224     return if $noerr;
2225
2226     die "unable to parse usb device\n";
2227 }
2228
2229 # add JSON properties for create and set function
2230 sub json_config_properties {
2231     my ($prop, $with_disk_alloc) = @_;
2232
2233     my $skip_json_config_opts = {
2234         parent => 1,
2235         snaptime => 1,
2236         vmstate => 1,
2237         runningmachine => 1,
2238         runningcpu => 1,
2239         meta => 1,
2240     };
2241
2242     foreach my $opt (keys %$confdesc) {
2243         next if $skip_json_config_opts->{$opt};
2244
2245         if ($with_disk_alloc && is_valid_drivename($opt)) {
2246             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2247         } else {
2248             $prop->{$opt} = $confdesc->{$opt};
2249         }
2250     }
2251
2252     return $prop;
2253 }
2254
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties {
2257     my $prop = {};
2258
2259     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2260         $prop->{$device} = {
2261             type => 'string',
2262             format => 'pve-volume-id-or-absolute-path',
2263             description => "Disk image that gets imported to $device",
2264             optional => 1,
2265         };
2266     }
2267
2268     $prop->{cores} = {
2269         type => 'integer',
2270         description => "The number of CPU cores.",
2271         optional => 1,
2272     };
2273     $prop->{memory} = {
2274         type => 'integer',
2275         description => "Amount of RAM for the VM in MB.",
2276         optional => 1,
2277     };
2278     $prop->{name} = {
2279         type => 'string',
2280         description => "Name of the VM.",
2281         optional => 1,
2282     };
2283
2284     return $prop;
2285 }
2286
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties {
2289
2290     return dclone($confdesc_cloudinit);
2291 }
2292
2293 sub check_type {
2294     my ($key, $value) = @_;
2295
2296     die "unknown setting '$key'\n" if !$confdesc->{$key};
2297
2298     my $type = $confdesc->{$key}->{type};
2299
2300     if (!defined($value)) {
2301         die "got undefined value\n";
2302     }
2303
2304     if ($value =~ m/[\n\r]/) {
2305         die "property contains a line feed\n";
2306     }
2307
2308     if ($type eq 'boolean') {
2309         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2310         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2311         die "type check ('boolean') failed - got '$value'\n";
2312     } elsif ($type eq 'integer') {
2313         return int($1) if $value =~ m/^(\d+)$/;
2314         die "type check ('integer') failed - got '$value'\n";
2315     } elsif ($type eq 'number') {
2316         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2317         die "type check ('number') failed - got '$value'\n";
2318     } elsif ($type eq 'string') {
2319         if (my $fmt = $confdesc->{$key}->{format}) {
2320             PVE::JSONSchema::check_format($fmt, $value);
2321             return $value;
2322         }
2323         $value =~ s/^\"(.*)\"$/$1/;
2324         return $value;
2325     } else {
2326         die "internal error"
2327     }
2328 }
2329
2330 sub destroy_vm {
2331     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2332
2333     my $conf = PVE::QemuConfig->load_config($vmid);
2334
2335     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2336
2337     if ($conf->{template}) {
2338         # check if any base image is still used by a linked clone
2339         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2340                 my ($ds, $drive) = @_;
2341                 return if drive_is_cdrom($drive);
2342
2343                 my $volid = $drive->{file};
2344                 return if !$volid || $volid =~ m|^/|;
2345
2346                 die "base volume '$volid' is still in use by linked cloned\n"
2347                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2348
2349         });
2350     }
2351
2352     my $volids = {};
2353     my $remove_owned_drive = sub {
2354         my ($ds, $drive) = @_;
2355         return if drive_is_cdrom($drive, 1);
2356
2357         my $volid = $drive->{file};
2358         return if !$volid || $volid =~ m|^/|;
2359         return if $volids->{$volid};
2360
2361         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2362         return if !$path || !$owner || ($owner != $vmid);
2363
2364         $volids->{$volid} = 1;
2365         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2366         warn "Could not remove disk '$volid', check manually: $@" if $@;
2367     };
2368
2369     # only remove disks owned by this VM (referenced in the config)
2370     my $include_opts = {
2371         include_unused => 1,
2372         extra_keys => ['vmstate'],
2373     };
2374     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2375
2376     for my $snap (values %{$conf->{snapshots}}) {
2377         next if !defined($snap->{vmstate});
2378         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2379         next if !defined($drive);
2380         $remove_owned_drive->('vmstate', $drive);
2381     }
2382
2383     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2384
2385     if ($purge_unreferenced) { # also remove unreferenced disk
2386         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2387         PVE::Storage::foreach_volid($vmdisks, sub {
2388             my ($volid, $sid, $volname, $d) = @_;
2389             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2390             warn $@ if $@;
2391         });
2392     }
2393
2394     if (defined $replacement_conf) {
2395         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2396     } else {
2397         PVE::QemuConfig->destroy_config($vmid);
2398     }
2399 }
2400
2401 sub parse_vm_config {
2402     my ($filename, $raw, $strict) = @_;
2403
2404     return if !defined($raw);
2405
2406     my $res = {
2407         digest => Digest::SHA::sha1_hex($raw),
2408         snapshots => {},
2409         pending => {},
2410         cloudinit => {},
2411     };
2412
2413     my $handle_error = sub {
2414         my ($msg) = @_;
2415
2416         if ($strict) {
2417             die $msg;
2418         } else {
2419             warn $msg;
2420         }
2421     };
2422
2423     $filename =~ m|/qemu-server/(\d+)\.conf$|
2424         || die "got strange filename '$filename'";
2425
2426     my $vmid = $1;
2427
2428     my $conf = $res;
2429     my $descr;
2430     my $finish_description = sub {
2431         if (defined($descr)) {
2432             $descr =~ s/\s+$//;
2433             $conf->{description} = $descr;
2434         }
2435         $descr = undef;
2436     };
2437     my $section = '';
2438
2439     my @lines = split(/\n/, $raw);
2440     foreach my $line (@lines) {
2441         next if $line =~ m/^\s*$/;
2442
2443         if ($line =~ m/^\[PENDING\]\s*$/i) {
2444             $section = 'pending';
2445             $finish_description->();
2446             $conf = $res->{$section} = {};
2447             next;
2448         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2449             $section = 'cloudinit';
2450             $finish_description->();
2451             $conf = $res->{$section} = {};
2452             next;
2453
2454         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2455             $section = $1;
2456             $finish_description->();
2457             $conf = $res->{snapshots}->{$section} = {};
2458             next;
2459         }
2460
2461         if ($line =~ m/^\#(.*)$/) {
2462             $descr = '' if !defined($descr);
2463             $descr .= PVE::Tools::decode_text($1) . "\n";
2464             next;
2465         }
2466
2467         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2468             $descr = '' if !defined($descr);
2469             $descr .= PVE::Tools::decode_text($2);
2470         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2471             $conf->{snapstate} = $1;
2472         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2473             my $key = $1;
2474             my $value = $2;
2475             $conf->{$key} = $value;
2476         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2477             my $value = $1;
2478             if ($section eq 'pending') {
2479                 $conf->{delete} = $value; # we parse this later
2480             } else {
2481                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2482             }
2483         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2484             my $key = $1;
2485             my $value = $2;
2486             eval { $value = check_type($key, $value); };
2487             if ($@) {
2488                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2489             } else {
2490                 $key = 'ide2' if $key eq 'cdrom';
2491                 my $fmt = $confdesc->{$key}->{format};
2492                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2493                     my $v = parse_drive($key, $value);
2494                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2495                         $v->{file} = $volid;
2496                         $value = print_drive($v);
2497                     } else {
2498                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2499                         next;
2500                     }
2501                 }
2502
2503                 $conf->{$key} = $value;
2504             }
2505         } else {
2506             $handle_error->("vm $vmid - unable to parse config: $line\n");
2507         }
2508     }
2509
2510     $finish_description->();
2511     delete $res->{snapstate}; # just to be sure
2512
2513     return $res;
2514 }
2515
2516 sub write_vm_config {
2517     my ($filename, $conf) = @_;
2518
2519     delete $conf->{snapstate}; # just to be sure
2520
2521     if ($conf->{cdrom}) {
2522         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2523         $conf->{ide2} = $conf->{cdrom};
2524         delete $conf->{cdrom};
2525     }
2526
2527     # we do not use 'smp' any longer
2528     if ($conf->{sockets}) {
2529         delete $conf->{smp};
2530     } elsif ($conf->{smp}) {
2531         $conf->{sockets} = $conf->{smp};
2532         delete $conf->{cores};
2533         delete $conf->{smp};
2534     }
2535
2536     my $used_volids = {};
2537
2538     my $cleanup_config = sub {
2539         my ($cref, $pending, $snapname) = @_;
2540
2541         foreach my $key (keys %$cref) {
2542             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2543                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2544             my $value = $cref->{$key};
2545             if ($key eq 'delete') {
2546                 die "propertry 'delete' is only allowed in [PENDING]\n"
2547                     if !$pending;
2548                 # fixme: check syntax?
2549                 next;
2550             }
2551             eval { $value = check_type($key, $value); };
2552             die "unable to parse value of '$key' - $@" if $@;
2553
2554             $cref->{$key} = $value;
2555
2556             if (!$snapname && is_valid_drivename($key)) {
2557                 my $drive = parse_drive($key, $value);
2558                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2559             }
2560         }
2561     };
2562
2563     &$cleanup_config($conf);
2564
2565     &$cleanup_config($conf->{pending}, 1);
2566
2567     &$cleanup_config($conf->{cloudinit});
2568
2569     foreach my $snapname (keys %{$conf->{snapshots}}) {
2570         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2571         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2572     }
2573
2574     # remove 'unusedX' settings if we re-add a volume
2575     foreach my $key (keys %$conf) {
2576         my $value = $conf->{$key};
2577         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2578             delete $conf->{$key};
2579         }
2580     }
2581
2582     my $generate_raw_config = sub {
2583         my ($conf, $pending) = @_;
2584
2585         my $raw = '';
2586
2587         # add description as comment to top of file
2588         if (defined(my $descr = $conf->{description})) {
2589             if ($descr) {
2590                 foreach my $cl (split(/\n/, $descr)) {
2591                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2592                 }
2593             } else {
2594                 $raw .= "#\n" if $pending;
2595             }
2596         }
2597
2598         foreach my $key (sort keys %$conf) {
2599             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2600             $raw .= "$key: $conf->{$key}\n";
2601         }
2602         return $raw;
2603     };
2604
2605     my $raw = &$generate_raw_config($conf);
2606
2607     if (scalar(keys %{$conf->{pending}})){
2608         $raw .= "\n[PENDING]\n";
2609         $raw .= &$generate_raw_config($conf->{pending}, 1);
2610     }
2611
2612     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2613         $raw .= "\n[special:cloudinit]\n";
2614         $raw .= &$generate_raw_config($conf->{cloudinit});
2615     }
2616
2617     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2618         $raw .= "\n[$snapname]\n";
2619         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2620     }
2621
2622     return $raw;
2623 }
2624
2625 sub load_defaults {
2626
2627     my $res = {};
2628
2629     # we use static defaults from our JSON schema configuration
2630     foreach my $key (keys %$confdesc) {
2631         if (defined(my $default = $confdesc->{$key}->{default})) {
2632             $res->{$key} = $default;
2633         }
2634     }
2635
2636     return $res;
2637 }
2638
2639 sub config_list {
2640     my $vmlist = PVE::Cluster::get_vmlist();
2641     my $res = {};
2642     return $res if !$vmlist || !$vmlist->{ids};
2643     my $ids = $vmlist->{ids};
2644     my $nodename = nodename();
2645
2646     foreach my $vmid (keys %$ids) {
2647         my $d = $ids->{$vmid};
2648         next if !$d->{node} || $d->{node} ne $nodename;
2649         next if !$d->{type} || $d->{type} ne 'qemu';
2650         $res->{$vmid}->{exists} = 1;
2651     }
2652     return $res;
2653 }
2654
2655 # test if VM uses local resources (to prevent migration)
2656 sub check_local_resources {
2657     my ($conf, $noerr) = @_;
2658
2659     my @loc_res = ();
2660
2661     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2662     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2663
2664     push @loc_res, "ivshmem" if $conf->{ivshmem};
2665
2666     foreach my $k (keys %$conf) {
2667         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2668         # sockets are safe: they will recreated be on the target side post-migrate
2669         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2670         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2671     }
2672
2673     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2674
2675     return \@loc_res;
2676 }
2677
2678 # check if used storages are available on all nodes (use by migrate)
2679 sub check_storage_availability {
2680     my ($storecfg, $conf, $node) = @_;
2681
2682     PVE::QemuConfig->foreach_volume($conf, sub {
2683         my ($ds, $drive) = @_;
2684
2685         my $volid = $drive->{file};
2686         return if !$volid;
2687
2688         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2689         return if !$sid;
2690
2691         # check if storage is available on both nodes
2692         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2693         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2694
2695         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2696
2697         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2698             if !$scfg->{content}->{$vtype};
2699    });
2700 }
2701
2702 # list nodes where all VM images are available (used by has_feature API)
2703 sub shared_nodes {
2704     my ($conf, $storecfg) = @_;
2705
2706     my $nodelist = PVE::Cluster::get_nodelist();
2707     my $nodehash = { map { $_ => 1 } @$nodelist };
2708     my $nodename = nodename();
2709
2710     PVE::QemuConfig->foreach_volume($conf, sub {
2711         my ($ds, $drive) = @_;
2712
2713         my $volid = $drive->{file};
2714         return if !$volid;
2715
2716         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2717         if ($storeid) {
2718             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2719             if ($scfg->{disable}) {
2720                 $nodehash = {};
2721             } elsif (my $avail = $scfg->{nodes}) {
2722                 foreach my $node (keys %$nodehash) {
2723                     delete $nodehash->{$node} if !$avail->{$node};
2724                 }
2725             } elsif (!$scfg->{shared}) {
2726                 foreach my $node (keys %$nodehash) {
2727                     delete $nodehash->{$node} if $node ne $nodename
2728                 }
2729             }
2730         }
2731     });
2732
2733     return $nodehash
2734 }
2735
2736 sub check_local_storage_availability {
2737     my ($conf, $storecfg) = @_;
2738
2739     my $nodelist = PVE::Cluster::get_nodelist();
2740     my $nodehash = { map { $_ => {} } @$nodelist };
2741
2742     PVE::QemuConfig->foreach_volume($conf, sub {
2743         my ($ds, $drive) = @_;
2744
2745         my $volid = $drive->{file};
2746         return if !$volid;
2747
2748         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2749         if ($storeid) {
2750             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2751
2752             if ($scfg->{disable}) {
2753                 foreach my $node (keys %$nodehash) {
2754                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2755                 }
2756             } elsif (my $avail = $scfg->{nodes}) {
2757                 foreach my $node (keys %$nodehash) {
2758                     if (!$avail->{$node}) {
2759                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2760                     }
2761                 }
2762             }
2763         }
2764     });
2765
2766     foreach my $node (values %$nodehash) {
2767         if (my $unavail = $node->{unavailable_storages}) {
2768             $node->{unavailable_storages} = [ sort keys %$unavail ];
2769         }
2770     }
2771
2772     return $nodehash
2773 }
2774
2775 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2776 sub check_running {
2777     my ($vmid, $nocheck, $node) = @_;
2778
2779     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2780     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2781 }
2782
2783 sub vzlist {
2784
2785     my $vzlist = config_list();
2786
2787     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2788
2789     while (defined(my $de = $fd->read)) {
2790         next if $de !~ m/^(\d+)\.pid$/;
2791         my $vmid = $1;
2792         next if !defined($vzlist->{$vmid});
2793         if (my $pid = check_running($vmid)) {
2794             $vzlist->{$vmid}->{pid} = $pid;
2795         }
2796     }
2797
2798     return $vzlist;
2799 }
2800
2801 our $vmstatus_return_properties = {
2802     vmid => get_standard_option('pve-vmid'),
2803     status => {
2804         description => "Qemu process status.",
2805         type => 'string',
2806         enum => ['stopped', 'running'],
2807     },
2808     maxmem => {
2809         description => "Maximum memory in bytes.",
2810         type => 'integer',
2811         optional => 1,
2812         renderer => 'bytes',
2813     },
2814     maxdisk => {
2815         description => "Root disk size in bytes.",
2816         type => 'integer',
2817         optional => 1,
2818         renderer => 'bytes',
2819     },
2820     name => {
2821         description => "VM name.",
2822         type => 'string',
2823         optional => 1,
2824     },
2825     qmpstatus => {
2826         description => "Qemu QMP agent status.",
2827         type => 'string',
2828         optional => 1,
2829     },
2830     pid => {
2831         description => "PID of running qemu process.",
2832         type => 'integer',
2833         optional => 1,
2834     },
2835     uptime => {
2836         description => "Uptime.",
2837         type => 'integer',
2838         optional => 1,
2839         renderer => 'duration',
2840     },
2841     cpus => {
2842         description => "Maximum usable CPUs.",
2843         type => 'number',
2844         optional => 1,
2845     },
2846     lock => {
2847         description => "The current config lock, if any.",
2848         type => 'string',
2849         optional => 1,
2850     },
2851     tags => {
2852         description  => "The current configured tags, if any",
2853         type => 'string',
2854         optional => 1,
2855     },
2856     'running-machine' => {
2857         description => "The currently running machine type (if running).",
2858         type => 'string',
2859         optional => 1,
2860     },
2861     'running-qemu' => {
2862         description => "The currently running QEMU version (if running).",
2863         type => 'string',
2864         optional => 1,
2865     },
2866 };
2867
2868 my $last_proc_pid_stat;
2869
2870 # get VM status information
2871 # This must be fast and should not block ($full == false)
2872 # We only query KVM using QMP if $full == true (this can be slow)
2873 sub vmstatus {
2874     my ($opt_vmid, $full) = @_;
2875
2876     my $res = {};
2877
2878     my $storecfg = PVE::Storage::config();
2879
2880     my $list = vzlist();
2881     my $defaults = load_defaults();
2882
2883     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2884
2885     my $cpucount = $cpuinfo->{cpus} || 1;
2886
2887     foreach my $vmid (keys %$list) {
2888         next if $opt_vmid && ($vmid ne $opt_vmid);
2889
2890         my $conf = PVE::QemuConfig->load_config($vmid);
2891
2892         my $d = { vmid => int($vmid) };
2893         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2894
2895         # fixme: better status?
2896         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2897
2898         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2899         if (defined($size)) {
2900             $d->{disk} = 0; # no info available
2901             $d->{maxdisk} = $size;
2902         } else {
2903             $d->{disk} = 0;
2904             $d->{maxdisk} = 0;
2905         }
2906
2907         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2908             * ($conf->{cores} || $defaults->{cores});
2909         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2910         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2911
2912         $d->{name} = $conf->{name} || "VM $vmid";
2913         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2914             : $defaults->{memory}*(1024*1024);
2915
2916         if ($conf->{balloon}) {
2917             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2918             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2919                 : $defaults->{shares};
2920         }
2921
2922         $d->{uptime} = 0;
2923         $d->{cpu} = 0;
2924         $d->{mem} = 0;
2925
2926         $d->{netout} = 0;
2927         $d->{netin} = 0;
2928
2929         $d->{diskread} = 0;
2930         $d->{diskwrite} = 0;
2931
2932         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2933
2934         $d->{serial} = 1 if conf_has_serial($conf);
2935         $d->{lock} = $conf->{lock} if $conf->{lock};
2936         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2937
2938         $res->{$vmid} = $d;
2939     }
2940
2941     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2942     foreach my $dev (keys %$netdev) {
2943         next if $dev !~ m/^tap([1-9]\d*)i/;
2944         my $vmid = $1;
2945         my $d = $res->{$vmid};
2946         next if !$d;
2947
2948         $d->{netout} += $netdev->{$dev}->{receive};
2949         $d->{netin} += $netdev->{$dev}->{transmit};
2950
2951         if ($full) {
2952             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2953             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2954         }
2955
2956     }
2957
2958     my $ctime = gettimeofday;
2959
2960     foreach my $vmid (keys %$list) {
2961
2962         my $d = $res->{$vmid};
2963         my $pid = $d->{pid};
2964         next if !$pid;
2965
2966         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2967         next if !$pstat; # not running
2968
2969         my $used = $pstat->{utime} + $pstat->{stime};
2970
2971         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2972
2973         if ($pstat->{vsize}) {
2974             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2975         }
2976
2977         my $old = $last_proc_pid_stat->{$pid};
2978         if (!$old) {
2979             $last_proc_pid_stat->{$pid} = {
2980                 time => $ctime,
2981                 used => $used,
2982                 cpu => 0,
2983             };
2984             next;
2985         }
2986
2987         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2988
2989         if ($dtime > 1000) {
2990             my $dutime = $used -  $old->{used};
2991
2992             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2993             $last_proc_pid_stat->{$pid} = {
2994                 time => $ctime,
2995                 used => $used,
2996                 cpu => $d->{cpu},
2997             };
2998         } else {
2999             $d->{cpu} = $old->{cpu};
3000         }
3001     }
3002
3003     return $res if !$full;
3004
3005     my $qmpclient = PVE::QMPClient->new();
3006
3007     my $ballooncb = sub {
3008         my ($vmid, $resp) = @_;
3009
3010         my $info = $resp->{'return'};
3011         return if !$info->{max_mem};
3012
3013         my $d = $res->{$vmid};
3014
3015         # use memory assigned to VM
3016         $d->{maxmem} = $info->{max_mem};
3017         $d->{balloon} = $info->{actual};
3018
3019         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3020             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3021             $d->{freemem} = $info->{free_mem};
3022         }
3023
3024         $d->{ballooninfo} = $info;
3025     };
3026
3027     my $blockstatscb = sub {
3028         my ($vmid, $resp) = @_;
3029         my $data = $resp->{'return'} || [];
3030         my $totalrdbytes = 0;
3031         my $totalwrbytes = 0;
3032
3033         for my $blockstat (@$data) {
3034             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3035             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3036
3037             $blockstat->{device} =~ s/drive-//;
3038             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3039         }
3040         $res->{$vmid}->{diskread} = $totalrdbytes;
3041         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3042     };
3043
3044     my $machinecb = sub {
3045         my ($vmid, $resp) = @_;
3046         my $data = $resp->{'return'} || [];
3047
3048         $res->{$vmid}->{'running-machine'} =
3049             PVE::QemuServer::Machine::current_from_query_machines($data);
3050     };
3051
3052     my $versioncb = sub {
3053         my ($vmid, $resp) = @_;
3054         my $data = $resp->{'return'} // {};
3055         my $version = 'unknown';
3056
3057         if (my $v = $data->{qemu}) {
3058             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3059         }
3060
3061         $res->{$vmid}->{'running-qemu'} = $version;
3062     };
3063
3064     my $statuscb = sub {
3065         my ($vmid, $resp) = @_;
3066
3067         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3068         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3069         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3070         # this fails if ballon driver is not loaded, so this must be
3071         # the last commnand (following command are aborted if this fails).
3072         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3073
3074         my $status = 'unknown';
3075         if (!defined($status = $resp->{'return'}->{status})) {
3076             warn "unable to get VM status\n";
3077             return;
3078         }
3079
3080         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3081     };
3082
3083     foreach my $vmid (keys %$list) {
3084         next if $opt_vmid && ($vmid ne $opt_vmid);
3085         next if !$res->{$vmid}->{pid}; # not running
3086         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3087     }
3088
3089     $qmpclient->queue_execute(undef, 2);
3090
3091     foreach my $vmid (keys %$list) {
3092         next if $opt_vmid && ($vmid ne $opt_vmid);
3093         next if !$res->{$vmid}->{pid}; #not running
3094
3095         # we can't use the $qmpclient since it might have already aborted on
3096         # 'query-balloon', but this might also fail for older versions...
3097         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3098         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3099     }
3100
3101     foreach my $vmid (keys %$list) {
3102         next if $opt_vmid && ($vmid ne $opt_vmid);
3103         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3104     }
3105
3106     return $res;
3107 }
3108
3109 sub conf_has_serial {
3110     my ($conf) = @_;
3111
3112     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3113         if ($conf->{"serial$i"}) {
3114             return 1;
3115         }
3116     }
3117
3118     return 0;
3119 }
3120
3121 sub conf_has_audio {
3122     my ($conf, $id) = @_;
3123
3124     $id //= 0;
3125     my $audio = $conf->{"audio$id"};
3126     return if !defined($audio);
3127
3128     my $audioproperties = parse_property_string($audio_fmt, $audio);
3129     my $audiodriver = $audioproperties->{driver} // 'spice';
3130
3131     return {
3132         dev => $audioproperties->{device},
3133         dev_id => "audiodev$id",
3134         backend => $audiodriver,
3135         backend_id => "$audiodriver-backend${id}",
3136     };
3137 }
3138
3139 sub audio_devs {
3140     my ($audio, $audiopciaddr, $machine_version) = @_;
3141
3142     my $devs = [];
3143
3144     my $id = $audio->{dev_id};
3145     my $audiodev = "";
3146     if (min_version($machine_version, 4, 2)) {
3147         $audiodev = ",audiodev=$audio->{backend_id}";
3148     }
3149
3150     if ($audio->{dev} eq 'AC97') {
3151         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3152     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3153         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3154         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3155         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3156     } else {
3157         die "unkown audio device '$audio->{dev}', implement me!";
3158     }
3159
3160     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3161
3162     return $devs;
3163 }
3164
3165 sub get_tpm_paths {
3166     my ($vmid) = @_;
3167     return {
3168         socket => "/var/run/qemu-server/$vmid.swtpm",
3169         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3170     };
3171 }
3172
3173 sub add_tpm_device {
3174     my ($vmid, $devices, $conf) = @_;
3175
3176     return if !$conf->{tpmstate0};
3177
3178     my $paths = get_tpm_paths($vmid);
3179
3180     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3181     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3182     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3183 }
3184
3185 sub start_swtpm {
3186     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3187
3188     return if !$tpmdrive;
3189
3190     my $state;
3191     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3192     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3193     if ($storeid) {
3194         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3195     } else {
3196         $state = $tpm->{file};
3197     }
3198
3199     my $paths = get_tpm_paths($vmid);
3200
3201     # during migration, we will get state from remote
3202     #
3203     if (!$migration) {
3204         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3205         my $setup_cmd = [
3206             "swtpm_setup",
3207             "--tpmstate",
3208             "file://$state",
3209             "--createek",
3210             "--create-ek-cert",
3211             "--create-platform-cert",
3212             "--lock-nvram",
3213             "--config",
3214             "/etc/swtpm_setup.conf", # do not use XDG configs
3215             "--runas",
3216             "0", # force creation as root, error if not possible
3217             "--not-overwrite", # ignore existing state, do not modify
3218         ];
3219
3220         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3221         # TPM 2.0 supports ECC crypto, use if possible
3222         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3223
3224         run_command($setup_cmd, outfunc => sub {
3225             print "swtpm_setup: $1\n";
3226         });
3227     }
3228
3229     my $emulator_cmd = [
3230         "swtpm",
3231         "socket",
3232         "--tpmstate",
3233         "backend-uri=file://$state,mode=0600",
3234         "--ctrl",
3235         "type=unixio,path=$paths->{socket},mode=0600",
3236         "--pid",
3237         "file=$paths->{pid}",
3238         "--terminate", # terminate on QEMU disconnect
3239         "--daemon",
3240     ];
3241     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3242     run_command($emulator_cmd, outfunc => sub { print $1; });
3243
3244     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3245     while (! -e $paths->{pid}) {
3246         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3247         usleep(50_000);
3248     }
3249
3250     # return untainted PID of swtpm daemon so it can be killed on error
3251     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3252     return $1;
3253 }
3254
3255 sub vga_conf_has_spice {
3256     my ($vga) = @_;
3257
3258     my $vgaconf = parse_vga($vga);
3259     my $vgatype = $vgaconf->{type};
3260     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3261
3262     return $1 || 1;
3263 }
3264
3265 sub is_native($) {
3266     my ($arch) = @_;
3267     return get_host_arch() eq $arch;
3268 }
3269
3270 sub get_vm_arch {
3271     my ($conf) = @_;
3272     return $conf->{arch} // get_host_arch();
3273 }
3274
3275 my $default_machines = {
3276     x86_64 => 'pc',
3277     aarch64 => 'virt',
3278 };
3279
3280 sub get_installed_machine_version {
3281     my ($kvmversion) = @_;
3282     $kvmversion = kvm_user_version() if !defined($kvmversion);
3283     $kvmversion =~ m/^(\d+\.\d+)/;
3284     return $1;
3285 }
3286
3287 sub windows_get_pinned_machine_version {
3288     my ($machine, $base_version, $kvmversion) = @_;
3289
3290     my $pin_version = $base_version;
3291     if (!defined($base_version) ||
3292         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3293     ) {
3294         $pin_version = get_installed_machine_version($kvmversion);
3295     }
3296     if (!$machine || $machine eq 'pc') {
3297         $machine = "pc-i440fx-$pin_version";
3298     } elsif ($machine eq 'q35') {
3299         $machine = "pc-q35-$pin_version";
3300     } elsif ($machine eq 'virt') {
3301         $machine = "virt-$pin_version";
3302     } else {
3303         warn "unknown machine type '$machine', not touching that!\n";
3304     }
3305
3306     return $machine;
3307 }
3308
3309 sub get_vm_machine {
3310     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3311
3312     my $machine = $forcemachine || $conf->{machine};
3313
3314     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3315         $kvmversion //= kvm_user_version();
3316         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3317         # layout which confuses windows quite a bit and may result in various regressions..
3318         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3319         if (windows_version($conf->{ostype})) {
3320             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3321         }
3322         $arch //= 'x86_64';
3323         $machine ||= $default_machines->{$arch};
3324         if ($add_pve_version) {
3325             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3326             $machine .= "+pve$pvever";
3327         }
3328     }
3329
3330     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3331         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3332         $machine = $1 if $is_pxe;
3333
3334         # for version-pinned machines that do not include a pve-version (e.g.
3335         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3336         $machine .= '+pve0';
3337
3338         $machine .= '.pxe' if $is_pxe;
3339     }
3340
3341     return $machine;
3342 }
3343
3344 sub get_ovmf_files($$$) {
3345     my ($arch, $efidisk, $smm) = @_;
3346
3347     my $types = $OVMF->{$arch}
3348         or die "no OVMF images known for architecture '$arch'\n";
3349
3350     my $type = 'default';
3351     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3352         $type = $smm ? "4m" : "4m-no-smm";
3353         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3354     }
3355
3356     return $types->{$type}->@*;
3357 }
3358
3359 my $Arch2Qemu = {
3360     aarch64 => '/usr/bin/qemu-system-aarch64',
3361     x86_64 => '/usr/bin/qemu-system-x86_64',
3362 };
3363 sub get_command_for_arch($) {
3364     my ($arch) = @_;
3365     return '/usr/bin/kvm' if is_native($arch);
3366
3367     my $cmd = $Arch2Qemu->{$arch}
3368         or die "don't know how to emulate architecture '$arch'\n";
3369     return $cmd;
3370 }
3371
3372 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3373 # to use in a QEMU command line (-cpu element), first array_intersect the result
3374 # of query_supported_ with query_understood_. This is necessary because:
3375 #
3376 # a) query_understood_ returns flags the host cannot use and
3377 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3378 #    flags, but CPU settings - with most of them being flags. Those settings
3379 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3380 #
3381 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3382 # expensive. If you need the value returned from this, you can get it much
3383 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3384 # $accel being 'kvm' or 'tcg'.
3385 #
3386 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3387 # changes, automatically populating pmxcfs.
3388 #
3389 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3390 # since kvm and tcg machines support different flags
3391 #
3392 sub query_supported_cpu_flags {
3393     my ($arch) = @_;
3394
3395     $arch //= get_host_arch();
3396     my $default_machine = $default_machines->{$arch};
3397
3398     my $flags = {};
3399
3400     # FIXME: Once this is merged, the code below should work for ARM as well:
3401     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3402     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3403         $arch eq "aarch64";
3404
3405     my $kvm_supported = defined(kvm_version());
3406     my $qemu_cmd = get_command_for_arch($arch);
3407     my $fakevmid = -1;
3408     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3409
3410     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3411     my $query_supported_run_qemu = sub {
3412         my ($kvm) = @_;
3413
3414         my $flags = {};
3415         my $cmd = [
3416             $qemu_cmd,
3417             '-machine', $default_machine,
3418             '-display', 'none',
3419             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3420             '-mon', 'chardev=qmp,mode=control',
3421             '-pidfile', $pidfile,
3422             '-S', '-daemonize'
3423         ];
3424
3425         if (!$kvm) {
3426             push @$cmd, '-accel', 'tcg';
3427         }
3428
3429         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3430         die "QEMU flag querying VM exited with code " . $rc if $rc;
3431
3432         eval {
3433             my $cmd_result = mon_cmd(
3434                 $fakevmid,
3435                 'query-cpu-model-expansion',
3436                 type => 'full',
3437                 model => { name => 'host' }
3438             );
3439
3440             my $props = $cmd_result->{model}->{props};
3441             foreach my $prop (keys %$props) {
3442                 next if $props->{$prop} ne '1';
3443                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3444                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3445                 # We only keep those with underscores, to match /proc/cpuinfo
3446                 $prop =~ s/\.|-/_/g;
3447                 $flags->{$prop} = 1;
3448             }
3449         };
3450         my $err = $@;
3451
3452         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3453         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3454
3455         die $err if $err;
3456
3457         return [ sort keys %$flags ];
3458     };
3459
3460     # We need to query QEMU twice, since KVM and TCG have different supported flags
3461     PVE::QemuConfig->lock_config($fakevmid, sub {
3462         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3463         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3464
3465         if ($kvm_supported) {
3466             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3467             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3468         }
3469     });
3470
3471     return $flags;
3472 }
3473
3474 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3475 my $understood_cpu_flag_dir = "/usr/share/kvm";
3476 sub query_understood_cpu_flags {
3477     my $arch = get_host_arch();
3478     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3479
3480     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3481         if ! -e $filepath;
3482
3483     my $raw = file_get_contents($filepath);
3484     $raw =~ s/^\s+|\s+$//g;
3485     my @flags = split(/\s+/, $raw);
3486
3487     return \@flags;
3488 }
3489
3490 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3491 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3492 my sub should_disable_smm {
3493     my ($conf, $vga) = @_;
3494
3495     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3496         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3497 }
3498
3499 sub config_to_command {
3500     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3501         $pbs_backing) = @_;
3502
3503     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3504     my $devices = [];
3505     my $bridges = {};
3506     my $ostype = $conf->{ostype};
3507     my $winversion = windows_version($ostype);
3508     my $kvm = $conf->{kvm};
3509     my $nodename = nodename();
3510
3511     my $arch = get_vm_arch($conf);
3512     my $kvm_binary = get_command_for_arch($arch);
3513     my $kvmver = kvm_user_version($kvm_binary);
3514
3515     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3516         $kvmver //= "undefined";
3517         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3518     }
3519
3520     my $add_pve_version = min_version($kvmver, 4, 1);
3521
3522     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3523     my $machine_version = extract_version($machine_type, $kvmver);
3524     $kvm //= 1 if is_native($arch);
3525
3526     $machine_version =~ m/(\d+)\.(\d+)/;
3527     my ($machine_major, $machine_minor) = ($1, $2);
3528
3529     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3530         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3531     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3532         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3533             ." please upgrade node '$nodename'\n"
3534     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3535         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3536         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3537             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3538             ." node '$nodename'\n";
3539     }
3540
3541     # if a specific +pve version is required for a feature, use $version_guard
3542     # instead of min_version to allow machines to be run with the minimum
3543     # required version
3544     my $required_pve_version = 0;
3545     my $version_guard = sub {
3546         my ($major, $minor, $pve) = @_;
3547         return 0 if !min_version($machine_version, $major, $minor, $pve);
3548         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3549         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3550         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3551         return 1;
3552     };
3553
3554     if ($kvm && !defined kvm_version()) {
3555         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3556             ." or enable in BIOS.\n";
3557     }
3558
3559     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3560     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3561     my $use_old_bios_files = undef;
3562     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3563
3564     my $cmd = [];
3565     if ($conf->{affinity}) {
3566         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3567     }
3568
3569     push @$cmd, $kvm_binary;
3570
3571     push @$cmd, '-id', $vmid;
3572
3573     my $vmname = $conf->{name} || "vm$vmid";
3574
3575     push @$cmd, '-name', "$vmname,debug-threads=on";
3576
3577     push @$cmd, '-no-shutdown';
3578
3579     my $use_virtio = 0;
3580
3581     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3582     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3583     push @$cmd, '-mon', "chardev=qmp,mode=control";
3584
3585     if (min_version($machine_version, 2, 12)) {
3586         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3587         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3588     }
3589
3590     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3591
3592     push @$cmd, '-daemonize';
3593
3594     if ($conf->{smbios1}) {
3595         my $smbios_conf = parse_smbios1($conf->{smbios1});
3596         if ($smbios_conf->{base64}) {
3597             # Do not pass base64 flag to qemu
3598             delete $smbios_conf->{base64};
3599             my $smbios_string = "";
3600             foreach my $key (keys %$smbios_conf) {
3601                 my $value;
3602                 if ($key eq "uuid") {
3603                     $value = $smbios_conf->{uuid}
3604                 } else {
3605                     $value = decode_base64($smbios_conf->{$key});
3606                 }
3607                 # qemu accepts any binary data, only commas need escaping by double comma
3608                 $value =~ s/,/,,/g;
3609                 $smbios_string .= "," . $key . "=" . $value if $value;
3610             }
3611             push @$cmd, '-smbios', "type=1" . $smbios_string;
3612         } else {
3613             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3614         }
3615     }
3616
3617     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3618         my $d;
3619         if (my $efidisk = $conf->{efidisk0}) {
3620             $d = parse_drive('efidisk0', $efidisk);
3621         }
3622
3623         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3624         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3625
3626         my ($path, $format);
3627         my $read_only_str = '';
3628         if ($d) {
3629             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3630             $format = $d->{format};
3631             if ($storeid) {
3632                 $path = PVE::Storage::path($storecfg, $d->{file});
3633                 if (!defined($format)) {
3634                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3635                     $format = qemu_img_format($scfg, $volname);
3636                 }
3637             } else {
3638                 $path = $d->{file};
3639                 die "efidisk format must be specified\n"
3640                     if !defined($format);
3641             }
3642
3643             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3644         } else {
3645             log_warn("no efidisk configured! Using temporary efivars disk.");
3646             $path = "/tmp/$vmid-ovmf.fd";
3647             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3648             $format = 'raw';
3649         }
3650
3651         my $size_str = "";
3652
3653         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3654             $size_str = ",size=" . (-s $ovmf_vars);
3655         }
3656
3657         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3658         my $cache = "";
3659         if ($path =~ m/^rbd:/) {
3660                 $cache = ',cache=writeback';
3661                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3662         }
3663
3664         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3665         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3666     }
3667
3668     if ($q35) { # tell QEMU to load q35 config early
3669         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3670         if (min_version($machine_version, 4, 0)) {
3671             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3672         } else {
3673             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3674         }
3675     }
3676
3677     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3678         push @$cmd, $fixups->@*;
3679     }
3680
3681     if ($conf->{vmgenid}) {
3682         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3683     }
3684
3685     # add usb controllers
3686     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3687         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3688     push @$devices, @usbcontrollers if @usbcontrollers;
3689     my $vga = parse_vga($conf->{vga});
3690
3691     my $qxlnum = vga_conf_has_spice($conf->{vga});
3692     $vga->{type} = 'qxl' if $qxlnum;
3693
3694     if (!$vga->{type}) {
3695         if ($arch eq 'aarch64') {
3696             $vga->{type} = 'virtio';
3697         } elsif (min_version($machine_version, 2, 9)) {
3698             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3699         } else {
3700             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3701         }
3702     }
3703
3704     # enable absolute mouse coordinates (needed by vnc)
3705     my $tablet = $conf->{tablet};
3706     if (!defined($tablet)) {
3707         $tablet = $defaults->{tablet};
3708         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3709         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3710     }
3711
3712     if ($tablet) {
3713         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3714         my $kbd = print_keyboarddevice_full($conf, $arch);
3715         push @$devices, '-device', $kbd if defined($kbd);
3716     }
3717
3718     my $bootorder = device_bootorder($conf);
3719
3720     # host pci device passthrough
3721     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3722         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3723
3724     # usb devices
3725     my $usb_dev_features = {};
3726     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3727
3728     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3729         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3730     push @$devices, @usbdevices if @usbdevices;
3731
3732     # serial devices
3733     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3734         my $path = $conf->{"serial$i"} or next;
3735         if ($path eq 'socket') {
3736             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3737             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3738             # On aarch64, serial0 is the UART device. Qemu only allows
3739             # connecting UART devices via the '-serial' command line, as
3740             # the device has a fixed slot on the hardware...
3741             if ($arch eq 'aarch64' && $i == 0) {
3742                 push @$devices, '-serial', "chardev:serial$i";
3743             } else {
3744                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3745             }
3746         } else {
3747             die "no such serial device\n" if ! -c $path;
3748             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3749             push @$devices, '-device', "isa-serial,chardev=serial$i";
3750         }
3751     }
3752
3753     # parallel devices
3754     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3755         if (my $path = $conf->{"parallel$i"}) {
3756             die "no such parallel device\n" if ! -c $path;
3757             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3758             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3759             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3760         }
3761     }
3762
3763     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3764         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3765         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3766         push @$devices, @$audio_devs;
3767     }
3768
3769     add_tpm_device($vmid, $devices, $conf);
3770
3771     my $sockets = 1;
3772     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3773     $sockets = $conf->{sockets} if  $conf->{sockets};
3774
3775     my $cores = $conf->{cores} || 1;
3776
3777     my $maxcpus = $sockets * $cores;
3778
3779     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3780
3781     my $allowed_vcpus = $cpuinfo->{cpus};
3782
3783     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3784
3785     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3786         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3787         for (my $i = 2; $i <= $vcpus; $i++)  {
3788             my $cpustr = print_cpu_device($conf,$i);
3789             push @$cmd, '-device', $cpustr;
3790         }
3791
3792     } else {
3793
3794         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3795     }
3796     push @$cmd, '-nodefaults';
3797
3798     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3799
3800     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3801
3802     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3803
3804     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3805         push @$devices, '-device', print_vga_device(
3806             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3807
3808         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3809
3810         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3811         push @$cmd,  '-vnc', "unix:$socket,password=on";
3812     } else {
3813         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3814         push @$cmd, '-nographic';
3815     }
3816
3817     # time drift fix
3818     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3819     my $useLocaltime = $conf->{localtime};
3820
3821     if ($winversion >= 5) { # windows
3822         $useLocaltime = 1 if !defined($conf->{localtime});
3823
3824         # use time drift fix when acpi is enabled
3825         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3826             $tdf = 1 if !defined($conf->{tdf});
3827         }
3828     }
3829
3830     if ($winversion >= 6) {
3831         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3832         push @$cmd, '-no-hpet';
3833     }
3834
3835     push @$rtcFlags, 'driftfix=slew' if $tdf;
3836
3837     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3838         push @$rtcFlags, "base=$conf->{startdate}";
3839     } elsif ($useLocaltime) {
3840         push @$rtcFlags, 'base=localtime';
3841     }
3842
3843     if ($forcecpu) {
3844         push @$cmd, '-cpu', $forcecpu;
3845     } else {
3846         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3847     }
3848
3849     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3850
3851     push @$cmd, '-S' if $conf->{freeze};
3852
3853     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3854
3855     my $guest_agent = parse_guest_agent($conf);
3856
3857     if ($guest_agent->{enabled}) {
3858         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3859         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3860
3861         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3862             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3863             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3864             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3865         } elsif ($guest_agent->{type} eq 'isa') {
3866             push @$devices, '-device', "isa-serial,chardev=qga0";
3867         }
3868     }
3869
3870     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3871     if ($rng && $version_guard->(4, 1, 2)) {
3872         check_rng_source($rng->{source});
3873
3874         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3875         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3876         my $limiter_str = "";
3877         if ($max_bytes) {
3878             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3879         }
3880
3881         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3882         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3883         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3884     }
3885
3886     my $spice_port;
3887
3888     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3889         if ($qxlnum > 1) {
3890             if ($winversion){
3891                 for (my $i = 1; $i < $qxlnum; $i++){
3892                     push @$devices, '-device', print_vga_device(
3893                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3894                 }
3895             } else {
3896                 # assume other OS works like Linux
3897                 my ($ram, $vram) = ("134217728", "67108864");
3898                 if ($vga->{memory}) {
3899                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3900                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3901                 }
3902                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3903                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3904             }
3905         }
3906
3907         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3908
3909         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3910         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3911         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3912
3913         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3914         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3915         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3916
3917         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3918         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3919
3920         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3921         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3922         if ($spice_enhancement->{foldersharing}) {
3923             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3924             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3925         }
3926
3927         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3928         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3929             if $spice_enhancement->{videostreaming};
3930
3931         push @$devices, '-spice', "$spice_opts";
3932     }
3933
3934     # enable balloon by default, unless explicitly disabled
3935     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3936         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3937         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3938         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3939         push @$devices, '-device', $ballooncmd;
3940     }
3941
3942     if ($conf->{watchdog}) {
3943         my $wdopts = parse_watchdog($conf->{watchdog});
3944         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3945         my $watchdog = $wdopts->{model} || 'i6300esb';
3946         push @$devices, '-device', "$watchdog$pciaddr";
3947         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3948     }
3949
3950     my $vollist = [];
3951     my $scsicontroller = {};
3952     my $ahcicontroller = {};
3953     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3954
3955     # Add iscsi initiator name if available
3956     if (my $initiator = get_initiator_name()) {
3957         push @$devices, '-iscsi', "initiator-name=$initiator";
3958     }
3959
3960     PVE::QemuConfig->foreach_volume($conf, sub {
3961         my ($ds, $drive) = @_;
3962
3963         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3964             check_volume_storage_type($storecfg, $drive->{file});
3965             push @$vollist, $drive->{file};
3966         }
3967
3968         # ignore efidisk here, already added in bios/fw handling code above
3969         return if $drive->{interface} eq 'efidisk';
3970         # similar for TPM
3971         return if $drive->{interface} eq 'tpmstate';
3972
3973         $use_virtio = 1 if $ds =~ m/^virtio/;
3974
3975         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3976
3977         if ($drive->{interface} eq 'virtio'){
3978            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3979         }
3980
3981         if ($drive->{interface} eq 'scsi') {
3982
3983             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3984
3985             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3986                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3987
3988             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3989             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3990
3991             my $iothread = '';
3992             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3993                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3994                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3995             } elsif ($drive->{iothread}) {
3996                 log_warn(
3997                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
3998                 );
3999             }
4000
4001             my $queues = '';
4002             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4003                 $queues = ",num_queues=$drive->{queues}";
4004             }
4005
4006             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4007                 if !$scsicontroller->{$controller};
4008             $scsicontroller->{$controller}=1;
4009         }
4010
4011         if ($drive->{interface} eq 'sata') {
4012             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4013             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4014             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4015                 if !$ahcicontroller->{$controller};
4016             $ahcicontroller->{$controller}=1;
4017         }
4018
4019         my $pbs_conf = $pbs_backing->{$ds};
4020         my $pbs_name = undef;
4021         if ($pbs_conf) {
4022             $pbs_name = "drive-$ds-pbs";
4023             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4024         }
4025
4026         my $drive_cmd = print_drive_commandline_full(
4027             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4028
4029         # extra protection for templates, but SATA and IDE don't support it..
4030         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4031
4032         push @$devices, '-drive',$drive_cmd;
4033         push @$devices, '-device', print_drivedevice_full(
4034             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4035     });
4036
4037     for (my $i = 0; $i < $MAX_NETS; $i++) {
4038         my $netname = "net$i";
4039
4040         next if !$conf->{$netname};
4041         my $d = parse_net($conf->{$netname});
4042         next if !$d;
4043         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4044
4045         $use_virtio = 1 if $d->{model} eq 'virtio';
4046
4047         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4048
4049         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4050         push @$devices, '-netdev', $netdevfull;
4051
4052         my $netdevicefull = print_netdevice_full(
4053             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4054
4055         push @$devices, '-device', $netdevicefull;
4056     }
4057
4058     if ($conf->{ivshmem}) {
4059         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4060
4061         my $bus;
4062         if ($q35) {
4063             $bus = print_pcie_addr("ivshmem");
4064         } else {
4065             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4066         }
4067
4068         my $ivshmem_name = $ivshmem->{name} // $vmid;
4069         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4070
4071         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4072         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4073             .",size=$ivshmem->{size}M";
4074     }
4075
4076     # pci.4 is nested in pci.1
4077     $bridges->{1} = 1 if $bridges->{4};
4078
4079     if (!$q35) { # add pci bridges
4080         if (min_version($machine_version, 2, 3)) {
4081            $bridges->{1} = 1;
4082            $bridges->{2} = 1;
4083         }
4084         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4085     }
4086
4087     for my $k (sort {$b cmp $a} keys %$bridges) {
4088         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4089
4090         my $k_name = $k;
4091         if ($k == 2 && $legacy_igd) {
4092             $k_name = "$k-igd";
4093         }
4094         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4095         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4096
4097         if ($q35) { # add after -readconfig pve-q35.cfg
4098             splice @$devices, 2, 0, '-device', $devstr;
4099         } else {
4100             unshift @$devices, '-device', $devstr if $k > 0;
4101         }
4102     }
4103
4104     if (!$kvm) {
4105         push @$machineFlags, 'accel=tcg';
4106     }
4107
4108     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4109
4110     my $machine_type_min = $machine_type;
4111     if ($add_pve_version) {
4112         $machine_type_min =~ s/\+pve\d+$//;
4113         $machine_type_min .= "+pve$required_pve_version";
4114     }
4115     push @$machineFlags, "type=${machine_type_min}";
4116
4117     push @$cmd, @$devices;
4118     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4119     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4120     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4121
4122     if (my $vmstate = $conf->{vmstate}) {
4123         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4124         push @$vollist, $vmstate;
4125         push @$cmd, '-loadstate', $statepath;
4126         print "activating and using '$vmstate' as vmstate\n";
4127     }
4128
4129     if (PVE::QemuConfig->is_template($conf)) {
4130         # needed to workaround base volumes being read-only
4131         push @$cmd, '-snapshot';
4132     }
4133
4134     # add custom args
4135     if ($conf->{args}) {
4136         my $aa = PVE::Tools::split_args($conf->{args});
4137         push @$cmd, @$aa;
4138     }
4139
4140     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4141 }
4142
4143 sub check_rng_source {
4144     my ($source) = @_;
4145
4146     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4147     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4148         if ! -e $source;
4149
4150     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4151     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4152         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4153         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4154         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4155             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4156             ." to the host.\n";
4157     }
4158 }
4159
4160 sub spice_port {
4161     my ($vmid) = @_;
4162
4163     my $res = mon_cmd($vmid, 'query-spice');
4164
4165     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4166 }
4167
4168 sub vm_devices_list {
4169     my ($vmid) = @_;
4170
4171     my $res = mon_cmd($vmid, 'query-pci');
4172     my $devices_to_check = [];
4173     my $devices = {};
4174     foreach my $pcibus (@$res) {
4175         push @$devices_to_check, @{$pcibus->{devices}},
4176     }
4177
4178     while (@$devices_to_check) {
4179         my $to_check = [];
4180         for my $d (@$devices_to_check) {
4181             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4182             next if !$d->{'pci_bridge'};
4183
4184             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4185             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4186         }
4187         $devices_to_check = $to_check;
4188     }
4189
4190     my $resblock = mon_cmd($vmid, 'query-block');
4191     foreach my $block (@$resblock) {
4192         if($block->{device} =~ m/^drive-(\S+)/){
4193                 $devices->{$1} = 1;
4194         }
4195     }
4196
4197     my $resmice = mon_cmd($vmid, 'query-mice');
4198     foreach my $mice (@$resmice) {
4199         if ($mice->{name} eq 'QEMU HID Tablet') {
4200             $devices->{tablet} = 1;
4201             last;
4202         }
4203     }
4204
4205     # for usb devices there is no query-usb
4206     # but we can iterate over the entries in
4207     # qom-list path=/machine/peripheral
4208     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4209     foreach my $per (@$resperipheral) {
4210         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4211             $devices->{$per->{name}} = 1;
4212         }
4213     }
4214
4215     return $devices;
4216 }
4217
4218 sub vm_deviceplug {
4219     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4220
4221     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4222
4223     my $devices_list = vm_devices_list($vmid);
4224     return 1 if defined($devices_list->{$deviceid});
4225
4226     # add PCI bridge if we need it for the device
4227     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4228
4229     if ($deviceid eq 'tablet') {
4230         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4231     } elsif ($deviceid eq 'keyboard') {
4232         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4233     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4234         my $id = $1;
4235         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4236         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4237     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4238         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4239     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4240         qemu_iothread_add($vmid, $deviceid, $device);
4241
4242         qemu_driveadd($storecfg, $vmid, $device);
4243         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4244
4245         qemu_deviceadd($vmid, $devicefull);
4246         eval { qemu_deviceaddverify($vmid, $deviceid); };
4247         if (my $err = $@) {
4248             eval { qemu_drivedel($vmid, $deviceid); };
4249             warn $@ if $@;
4250             die $err;
4251         }
4252     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4253         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4254         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4255         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4256
4257         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4258
4259         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4260             qemu_iothread_add($vmid, $deviceid, $device);
4261             $devicefull .= ",iothread=iothread-$deviceid";
4262         }
4263
4264         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4265             $devicefull .= ",num_queues=$device->{queues}";
4266         }
4267
4268         qemu_deviceadd($vmid, $devicefull);
4269         qemu_deviceaddverify($vmid, $deviceid);
4270     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4271         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4272         qemu_driveadd($storecfg, $vmid, $device);
4273
4274         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4275         eval { qemu_deviceadd($vmid, $devicefull); };
4276         if (my $err = $@) {
4277             eval { qemu_drivedel($vmid, $deviceid); };
4278             warn $@ if $@;
4279             die $err;
4280         }
4281     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4282         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4283
4284         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4285         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4286         my $use_old_bios_files = undef;
4287         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4288
4289         my $netdevicefull = print_netdevice_full(
4290             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4291         qemu_deviceadd($vmid, $netdevicefull);
4292         eval {
4293             qemu_deviceaddverify($vmid, $deviceid);
4294             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4295         };
4296         if (my $err = $@) {
4297             eval { qemu_netdevdel($vmid, $deviceid); };
4298             warn $@ if $@;
4299             die $err;
4300         }
4301     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4302         my $bridgeid = $2;
4303         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4304         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4305
4306         qemu_deviceadd($vmid, $devicefull);
4307         qemu_deviceaddverify($vmid, $deviceid);
4308     } else {
4309         die "can't hotplug device '$deviceid'\n";
4310     }
4311
4312     return 1;
4313 }
4314
4315 # fixme: this should raise exceptions on error!
4316 sub vm_deviceunplug {
4317     my ($vmid, $conf, $deviceid) = @_;
4318
4319     my $devices_list = vm_devices_list($vmid);
4320     return 1 if !defined($devices_list->{$deviceid});
4321
4322     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4323     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4324
4325     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4326         qemu_devicedel($vmid, $deviceid);
4327     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4328         qemu_devicedel($vmid, $deviceid);
4329         qemu_devicedelverify($vmid, $deviceid);
4330     } elsif ($deviceid =~ m/^usb\d+$/) {
4331         qemu_devicedel($vmid, $deviceid);
4332         qemu_devicedelverify($vmid, $deviceid);
4333     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4334         my $device = parse_drive($deviceid, $conf->{$deviceid});
4335
4336         qemu_devicedel($vmid, $deviceid);
4337         qemu_devicedelverify($vmid, $deviceid);
4338         qemu_drivedel($vmid, $deviceid);
4339         qemu_iothread_del($vmid, $deviceid, $device);
4340     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4341         qemu_devicedel($vmid, $deviceid);
4342         qemu_devicedelverify($vmid, $deviceid);
4343     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4344         my $device = parse_drive($deviceid, $conf->{$deviceid});
4345
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348         qemu_drivedel($vmid, $deviceid);
4349         qemu_deletescsihw($conf, $vmid, $deviceid);
4350
4351         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4352             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4353     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4354         qemu_devicedel($vmid, $deviceid);
4355         qemu_devicedelverify($vmid, $deviceid);
4356         qemu_netdevdel($vmid, $deviceid);
4357     } else {
4358         die "can't unplug device '$deviceid'\n";
4359     }
4360
4361     return 1;
4362 }
4363
4364 sub qemu_spice_usbredir_chardev_add {
4365     my ($vmid, $id) = @_;
4366
4367     mon_cmd($vmid, "chardev-add" , (
4368         id => $id,
4369         backend => {
4370             type => 'spicevmc',
4371             data => {
4372                 type => "usbredir",
4373             },
4374         },
4375     ));
4376 }
4377
4378 sub qemu_deviceadd {
4379     my ($vmid, $devicefull) = @_;
4380
4381     $devicefull = "driver=".$devicefull;
4382     my %options =  split(/[=,]/, $devicefull);
4383
4384     mon_cmd($vmid, "device_add" , %options);
4385 }
4386
4387 sub qemu_devicedel {
4388     my ($vmid, $deviceid) = @_;
4389
4390     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4391 }
4392
4393 sub qemu_iothread_add {
4394     my ($vmid, $deviceid, $device) = @_;
4395
4396     if ($device->{iothread}) {
4397         my $iothreads = vm_iothreads_list($vmid);
4398         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4399     }
4400 }
4401
4402 sub qemu_iothread_del {
4403     my ($vmid, $deviceid, $device) = @_;
4404
4405     if ($device->{iothread}) {
4406         my $iothreads = vm_iothreads_list($vmid);
4407         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4408     }
4409 }
4410
4411 sub qemu_objectadd {
4412     my ($vmid, $objectid, $qomtype) = @_;
4413
4414     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4415
4416     return 1;
4417 }
4418
4419 sub qemu_objectdel {
4420     my ($vmid, $objectid) = @_;
4421
4422     mon_cmd($vmid, "object-del", id => $objectid);
4423
4424     return 1;
4425 }
4426
4427 sub qemu_driveadd {
4428     my ($storecfg, $vmid, $device) = @_;
4429
4430     my $kvmver = get_running_qemu_version($vmid);
4431     my $io_uring = min_version($kvmver, 6, 0);
4432     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4433     $drive =~ s/\\/\\\\/g;
4434     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4435
4436     # If the command succeeds qemu prints: "OK"
4437     return 1 if $ret =~ m/OK/s;
4438
4439     die "adding drive failed: $ret\n";
4440 }
4441
4442 sub qemu_drivedel {
4443     my ($vmid, $deviceid) = @_;
4444
4445     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4446     $ret =~ s/^\s+//;
4447
4448     return 1 if $ret eq "";
4449
4450     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4451     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4452
4453     die "deleting drive $deviceid failed : $ret\n";
4454 }
4455
4456 sub qemu_deviceaddverify {
4457     my ($vmid, $deviceid) = @_;
4458
4459     for (my $i = 0; $i <= 5; $i++) {
4460          my $devices_list = vm_devices_list($vmid);
4461          return 1 if defined($devices_list->{$deviceid});
4462          sleep 1;
4463     }
4464
4465     die "error on hotplug device '$deviceid'\n";
4466 }
4467
4468
4469 sub qemu_devicedelverify {
4470     my ($vmid, $deviceid) = @_;
4471
4472     # need to verify that the device is correctly removed as device_del
4473     # is async and empty return is not reliable
4474
4475     for (my $i = 0; $i <= 5; $i++) {
4476          my $devices_list = vm_devices_list($vmid);
4477          return 1 if !defined($devices_list->{$deviceid});
4478          sleep 1;
4479     }
4480
4481     die "error on hot-unplugging device '$deviceid'\n";
4482 }
4483
4484 sub qemu_findorcreatescsihw {
4485     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4486
4487     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4488
4489     my $scsihwid="$controller_prefix$controller";
4490     my $devices_list = vm_devices_list($vmid);
4491
4492     if (!defined($devices_list->{$scsihwid})) {
4493         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4494     }
4495
4496     return 1;
4497 }
4498
4499 sub qemu_deletescsihw {
4500     my ($conf, $vmid, $opt) = @_;
4501
4502     my $device = parse_drive($opt, $conf->{$opt});
4503
4504     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4505         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4506         return 1;
4507     }
4508
4509     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4510
4511     my $devices_list = vm_devices_list($vmid);
4512     foreach my $opt (keys %{$devices_list}) {
4513         if (is_valid_drivename($opt)) {
4514             my $drive = parse_drive($opt, $conf->{$opt});
4515             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4516                 return 1;
4517             }
4518         }
4519     }
4520
4521     my $scsihwid="scsihw$controller";
4522
4523     vm_deviceunplug($vmid, $conf, $scsihwid);
4524
4525     return 1;
4526 }
4527
4528 sub qemu_add_pci_bridge {
4529     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4530
4531     my $bridges = {};
4532
4533     my $bridgeid;
4534
4535     print_pci_addr($device, $bridges, $arch, $machine_type);
4536
4537     while (my ($k, $v) = each %$bridges) {
4538         $bridgeid = $k;
4539     }
4540     return 1 if !defined($bridgeid) || $bridgeid < 1;
4541
4542     my $bridge = "pci.$bridgeid";
4543     my $devices_list = vm_devices_list($vmid);
4544
4545     if (!defined($devices_list->{$bridge})) {
4546         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4547     }
4548
4549     return 1;
4550 }
4551
4552 sub qemu_set_link_status {
4553     my ($vmid, $device, $up) = @_;
4554
4555     mon_cmd($vmid, "set_link", name => $device,
4556                up => $up ? JSON::true : JSON::false);
4557 }
4558
4559 sub qemu_netdevadd {
4560     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4561
4562     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4563     my %options =  split(/[=,]/, $netdev);
4564
4565     if (defined(my $vhost = $options{vhost})) {
4566         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4567     }
4568
4569     if (defined(my $queues = $options{queues})) {
4570         $options{queues} = $queues + 0;
4571     }
4572
4573     mon_cmd($vmid, "netdev_add",  %options);
4574     return 1;
4575 }
4576
4577 sub qemu_netdevdel {
4578     my ($vmid, $deviceid) = @_;
4579
4580     mon_cmd($vmid, "netdev_del", id => $deviceid);
4581 }
4582
4583 sub qemu_usb_hotplug {
4584     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4585
4586     return if !$device;
4587
4588     # remove the old one first
4589     vm_deviceunplug($vmid, $conf, $deviceid);
4590
4591     # check if xhci controller is necessary and available
4592     my $devicelist = vm_devices_list($vmid);
4593
4594     if (!$devicelist->{xhci}) {
4595         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4596         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4597     }
4598
4599     # print_usbdevice_full expects the parsed device
4600     my $d = parse_usb_device($device->{host});
4601     $d->{usb3} = $device->{usb3};
4602
4603     # add the new one
4604     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4605 }
4606
4607 sub qemu_cpu_hotplug {
4608     my ($vmid, $conf, $vcpus) = @_;
4609
4610     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4611
4612     my $sockets = 1;
4613     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4614     $sockets = $conf->{sockets} if  $conf->{sockets};
4615     my $cores = $conf->{cores} || 1;
4616     my $maxcpus = $sockets * $cores;
4617
4618     $vcpus = $maxcpus if !$vcpus;
4619
4620     die "you can't add more vcpus than maxcpus\n"
4621         if $vcpus > $maxcpus;
4622
4623     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4624
4625     if ($vcpus < $currentvcpus) {
4626
4627         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4628
4629             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4630                 qemu_devicedel($vmid, "cpu$i");
4631                 my $retry = 0;
4632                 my $currentrunningvcpus = undef;
4633                 while (1) {
4634                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4635                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4636                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4637                     $retry++;
4638                     sleep 1;
4639                 }
4640                 #update conf after each succesfull cpu unplug
4641                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4642                 PVE::QemuConfig->write_config($vmid, $conf);
4643             }
4644         } else {
4645             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4646         }
4647
4648         return;
4649     }
4650
4651     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4652     die "vcpus in running vm does not match its configuration\n"
4653         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4654
4655     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4656
4657         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4658             my $cpustr = print_cpu_device($conf, $i);
4659             qemu_deviceadd($vmid, $cpustr);
4660
4661             my $retry = 0;
4662             my $currentrunningvcpus = undef;
4663             while (1) {
4664                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4665                 last if scalar(@{$currentrunningvcpus}) == $i;
4666                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4667                 sleep 1;
4668                 $retry++;
4669             }
4670             #update conf after each succesfull cpu hotplug
4671             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4672             PVE::QemuConfig->write_config($vmid, $conf);
4673         }
4674     } else {
4675
4676         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4677             mon_cmd($vmid, "cpu-add", id => int($i));
4678         }
4679     }
4680 }
4681
4682 sub qemu_block_set_io_throttle {
4683     my ($vmid, $deviceid,
4684         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4685         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4686         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4687         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4688
4689     return if !check_running($vmid) ;
4690
4691     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4692         bps => int($bps),
4693         bps_rd => int($bps_rd),
4694         bps_wr => int($bps_wr),
4695         iops => int($iops),
4696         iops_rd => int($iops_rd),
4697         iops_wr => int($iops_wr),
4698         bps_max => int($bps_max),
4699         bps_rd_max => int($bps_rd_max),
4700         bps_wr_max => int($bps_wr_max),
4701         iops_max => int($iops_max),
4702         iops_rd_max => int($iops_rd_max),
4703         iops_wr_max => int($iops_wr_max),
4704         bps_max_length => int($bps_max_length),
4705         bps_rd_max_length => int($bps_rd_max_length),
4706         bps_wr_max_length => int($bps_wr_max_length),
4707         iops_max_length => int($iops_max_length),
4708         iops_rd_max_length => int($iops_rd_max_length),
4709         iops_wr_max_length => int($iops_wr_max_length),
4710     );
4711
4712 }
4713
4714 sub qemu_block_resize {
4715     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4716
4717     my $running = check_running($vmid);
4718
4719     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4720
4721     return if !$running;
4722
4723     my $padding = (1024 - $size % 1024) % 1024;
4724     $size = $size + $padding;
4725
4726     mon_cmd(
4727         $vmid,
4728         "block_resize",
4729         device => $deviceid,
4730         size => int($size),
4731         timeout => 60,
4732     );
4733 }
4734
4735 sub qemu_volume_snapshot {
4736     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4737
4738     my $running = check_running($vmid);
4739
4740     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4741         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4742     } else {
4743         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4744     }
4745 }
4746
4747 sub qemu_volume_snapshot_delete {
4748     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4749
4750     my $running = check_running($vmid);
4751
4752     if($running) {
4753
4754         $running = undef;
4755         my $conf = PVE::QemuConfig->load_config($vmid);
4756         PVE::QemuConfig->foreach_volume($conf, sub {
4757             my ($ds, $drive) = @_;
4758             $running = 1 if $drive->{file} eq $volid;
4759         });
4760     }
4761
4762     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4763         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4764     } else {
4765         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4766     }
4767 }
4768
4769 sub set_migration_caps {
4770     my ($vmid, $savevm) = @_;
4771
4772     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4773
4774     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4775     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4776
4777     my $cap_ref = [];
4778
4779     my $enabled_cap = {
4780         "auto-converge" => 1,
4781         "xbzrle" => 1,
4782         "x-rdma-pin-all" => 0,
4783         "zero-blocks" => 0,
4784         "compress" => 0,
4785         "dirty-bitmaps" => $dirty_bitmaps,
4786     };
4787
4788     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4789
4790     for my $supported_capability (@$supported_capabilities) {
4791         push @$cap_ref, {
4792             capability => $supported_capability->{capability},
4793             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4794         };
4795     }
4796
4797     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4798 }
4799
4800 sub foreach_volid {
4801     my ($conf, $func, @param) = @_;
4802
4803     my $volhash = {};
4804
4805     my $test_volid = sub {
4806         my ($key, $drive, $snapname) = @_;
4807
4808         my $volid = $drive->{file};
4809         return if !$volid;
4810
4811         $volhash->{$volid}->{cdrom} //= 1;
4812         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4813
4814         my $replicate = $drive->{replicate} // 1;
4815         $volhash->{$volid}->{replicate} //= 0;
4816         $volhash->{$volid}->{replicate} = 1 if $replicate;
4817
4818         $volhash->{$volid}->{shared} //= 0;
4819         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4820
4821         $volhash->{$volid}->{referenced_in_config} //= 0;
4822         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4823
4824         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4825             if defined($snapname);
4826
4827         my $size = $drive->{size};
4828         $volhash->{$volid}->{size} //= $size if $size;
4829
4830         $volhash->{$volid}->{is_vmstate} //= 0;
4831         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4832
4833         $volhash->{$volid}->{is_tpmstate} //= 0;
4834         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4835
4836         $volhash->{$volid}->{is_unused} //= 0;
4837         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4838
4839         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4840     };
4841
4842     my $include_opts = {
4843         extra_keys => ['vmstate'],
4844         include_unused => 1,
4845     };
4846
4847     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4848     foreach my $snapname (keys %{$conf->{snapshots}}) {
4849         my $snap = $conf->{snapshots}->{$snapname};
4850         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4851     }
4852
4853     foreach my $volid (keys %$volhash) {
4854         &$func($volid, $volhash->{$volid}, @param);
4855     }
4856 }
4857
4858 my $fast_plug_option = {
4859     'lock' => 1,
4860     'name' => 1,
4861     'onboot' => 1,
4862     'shares' => 1,
4863     'startup' => 1,
4864     'description' => 1,
4865     'protection' => 1,
4866     'vmstatestorage' => 1,
4867     'hookscript' => 1,
4868     'tags' => 1,
4869 };
4870
4871 for my $opt (keys %$confdesc_cloudinit) {
4872     $fast_plug_option->{$opt} = 1;
4873 };
4874
4875 # hotplug changes in [PENDING]
4876 # $selection hash can be used to only apply specified options, for
4877 # example: { cores => 1 } (only apply changed 'cores')
4878 # $errors ref is used to return error messages
4879 sub vmconfig_hotplug_pending {
4880     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4881
4882     my $defaults = load_defaults();
4883     my $arch = get_vm_arch($conf);
4884     my $machine_type = get_vm_machine($conf, undef, $arch);
4885
4886     # commit values which do not have any impact on running VM first
4887     # Note: those option cannot raise errors, we we do not care about
4888     # $selection and always apply them.
4889
4890     my $add_error = sub {
4891         my ($opt, $msg) = @_;
4892         $errors->{$opt} = "hotplug problem - $msg";
4893     };
4894
4895     my $changes = 0;
4896     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4897         if ($fast_plug_option->{$opt}) {
4898             $conf->{$opt} = $conf->{pending}->{$opt};
4899             delete $conf->{pending}->{$opt};
4900             $changes = 1;
4901         }
4902     }
4903
4904     if ($changes) {
4905         PVE::QemuConfig->write_config($vmid, $conf);
4906     }
4907
4908     my $ostype = $conf->{ostype};
4909     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4910     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4911     my $usb_hotplug = $hotplug_features->{usb}
4912         && min_version($version, 7, 1)
4913         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4914
4915     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4916     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4917     foreach my $opt (sort keys %$pending_delete_hash) {
4918         next if $selection && !$selection->{$opt};
4919         my $force = $pending_delete_hash->{$opt}->{force};
4920         eval {
4921             if ($opt eq 'hotplug') {
4922                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4923             } elsif ($opt eq 'tablet') {
4924                 die "skip\n" if !$hotplug_features->{usb};
4925                 if ($defaults->{tablet}) {
4926                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4927                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4928                         if $arch eq 'aarch64';
4929                 } else {
4930                     vm_deviceunplug($vmid, $conf, 'tablet');
4931                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4932                 }
4933             } elsif ($opt =~ m/^usb(\d+)$/) {
4934                 my $index = $1;
4935                 die "skip\n" if !$usb_hotplug;
4936                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4937                 vm_deviceunplug($vmid, $conf, $opt);
4938             } elsif ($opt eq 'vcpus') {
4939                 die "skip\n" if !$hotplug_features->{cpu};
4940                 qemu_cpu_hotplug($vmid, $conf, undef);
4941             } elsif ($opt eq 'balloon') {
4942                 # enable balloon device is not hotpluggable
4943                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4944                 # here we reset the ballooning value to memory
4945                 my $balloon = $conf->{memory} || $defaults->{memory};
4946                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4947             } elsif ($fast_plug_option->{$opt}) {
4948                 # do nothing
4949             } elsif ($opt =~ m/^net(\d+)$/) {
4950                 die "skip\n" if !$hotplug_features->{network};
4951                 vm_deviceunplug($vmid, $conf, $opt);
4952             } elsif (is_valid_drivename($opt)) {
4953                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4954                 vm_deviceunplug($vmid, $conf, $opt);
4955                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4956             } elsif ($opt =~ m/^memory$/) {
4957                 die "skip\n" if !$hotplug_features->{memory};
4958                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4959             } elsif ($opt eq 'cpuunits') {
4960                 $cgroup->change_cpu_shares(undef);
4961             } elsif ($opt eq 'cpulimit') {
4962                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4963             } else {
4964                 die "skip\n";
4965             }
4966         };
4967         if (my $err = $@) {
4968             &$add_error($opt, $err) if $err ne "skip\n";
4969         } else {
4970             delete $conf->{$opt};
4971             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4972         }
4973     }
4974
4975     foreach my $opt (keys %{$conf->{pending}}) {
4976         next if $selection && !$selection->{$opt};
4977         my $value = $conf->{pending}->{$opt};
4978         eval {
4979             if ($opt eq 'hotplug') {
4980                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4981             } elsif ($opt eq 'tablet') {
4982                 die "skip\n" if !$hotplug_features->{usb};
4983                 if ($value == 1) {
4984                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4985                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4986                         if $arch eq 'aarch64';
4987                 } elsif ($value == 0) {
4988                     vm_deviceunplug($vmid, $conf, 'tablet');
4989                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4990                 }
4991             } elsif ($opt =~ m/^usb(\d+)$/) {
4992                 my $index = $1;
4993                 die "skip\n" if !$usb_hotplug;
4994                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4995                 my $id = $opt;
4996                 if ($d->{host} eq 'spice')  {
4997                     $id = "usbredirdev$index";
4998                 }
4999                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5000             } elsif ($opt eq 'vcpus') {
5001                 die "skip\n" if !$hotplug_features->{cpu};
5002                 qemu_cpu_hotplug($vmid, $conf, $value);
5003             } elsif ($opt eq 'balloon') {
5004                 # enable/disable balloning device is not hotpluggable
5005                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5006                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5007                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5008
5009                 # allow manual ballooning if shares is set to zero
5010                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5011                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5012                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5013                 }
5014             } elsif ($opt =~ m/^net(\d+)$/) {
5015                 # some changes can be done without hotplug
5016                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5017                                     $vmid, $opt, $value, $arch, $machine_type);
5018             } elsif (is_valid_drivename($opt)) {
5019                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5020                 # some changes can be done without hotplug
5021                 my $drive = parse_drive($opt, $value);
5022                 if (drive_is_cloudinit($drive)) {
5023                     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5024                 }
5025                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5026                                      $vmid, $opt, $value, $arch, $machine_type);
5027             } elsif ($opt =~ m/^memory$/) { #dimms
5028                 die "skip\n" if !$hotplug_features->{memory};
5029                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5030             } elsif ($opt eq 'cpuunits') {
5031                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5032                 $cgroup->change_cpu_shares($new_cpuunits);
5033             } elsif ($opt eq 'cpulimit') {
5034                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5035                 $cgroup->change_cpu_quota($cpulimit, 100000);
5036             } elsif ($opt eq 'agent') {
5037                 vmconfig_update_agent($conf, $opt, $value);
5038             } else {
5039                 die "skip\n";  # skip non-hot-pluggable options
5040             }
5041         };
5042         if (my $err = $@) {
5043             &$add_error($opt, $err) if $err ne "skip\n";
5044         } else {
5045             $conf->{$opt} = $value;
5046             delete $conf->{pending}->{$opt};
5047         }
5048     }
5049
5050     # unplug xhci controller if no usb device is left
5051     if ($usb_hotplug) {
5052         my $has_usb = 0;
5053         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5054             next if !defined($conf->{"usb$i"});
5055             $has_usb = 1;
5056             last;
5057         }
5058         if (!$has_usb) {
5059             vm_deviceunplug($vmid, $conf, 'xhci');
5060         }
5061     }
5062
5063     PVE::QemuConfig->write_config($vmid, $conf);
5064
5065     if($hotplug_features->{cloudinit}) {
5066         my $pending = PVE::QemuServer::Cloudinit::get_pending_config($conf, $vmid);
5067         my $regenerate = undef;
5068         for my $item (@$pending) {
5069             $regenerate = 1 if defined($item->{delete}) or defined($item->{pending});
5070         }
5071         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid) if $regenerate;
5072     }
5073 }
5074
5075 sub try_deallocate_drive {
5076     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5077
5078     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5079         my $volid = $drive->{file};
5080         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5081             my $sid = PVE::Storage::parse_volume_id($volid);
5082             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5083
5084             # check if the disk is really unused
5085             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5086                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5087             PVE::Storage::vdisk_free($storecfg, $volid);
5088             return 1;
5089         } else {
5090             # If vm is not owner of this disk remove from config
5091             return 1;
5092         }
5093     }
5094
5095     return;
5096 }
5097
5098 sub vmconfig_delete_or_detach_drive {
5099     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5100
5101     my $drive = parse_drive($opt, $conf->{$opt});
5102
5103     my $rpcenv = PVE::RPCEnvironment::get();
5104     my $authuser = $rpcenv->get_user();
5105
5106     if ($force) {
5107         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5108         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5109     } else {
5110         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5111     }
5112 }
5113
5114
5115
5116 sub vmconfig_apply_pending {
5117     my ($vmid, $conf, $storecfg, $errors) = @_;
5118
5119     return if !scalar(keys %{$conf->{pending}});
5120
5121     my $add_apply_error = sub {
5122         my ($opt, $msg) = @_;
5123         my $err_msg = "unable to apply pending change $opt : $msg";
5124         $errors->{$opt} = $err_msg;
5125         warn $err_msg;
5126     };
5127
5128     # cold plug
5129
5130     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5131     foreach my $opt (sort keys %$pending_delete_hash) {
5132         my $force = $pending_delete_hash->{$opt}->{force};
5133         eval {
5134             if ($opt =~ m/^unused/) {
5135                 die "internal error";
5136             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5137                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5138             }
5139         };
5140         if (my $err = $@) {
5141             $add_apply_error->($opt, $err);
5142         } else {
5143             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5144             delete $conf->{$opt};
5145         }
5146     }
5147
5148     PVE::QemuConfig->cleanup_pending($conf);
5149
5150     my $generate_cloudnit = undef;
5151
5152     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5153         next if $opt eq 'delete'; # just to be sure
5154         eval {
5155             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5156                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5157             }
5158         };
5159         if (my $err = $@) {
5160             $add_apply_error->($opt, $err);
5161         } else {
5162
5163             if (is_valid_drivename($opt)) {
5164                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5165                 $generate_cloudnit = 1 if drive_is_cloudinit($drive);
5166             }
5167
5168             $conf->{$opt} = delete $conf->{pending}->{$opt};
5169         }
5170     }
5171
5172     # write all changes at once to avoid unnecessary i/o
5173     PVE::QemuConfig->write_config($vmid, $conf);
5174     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if $generate_cloudnit;
5175 }
5176
5177 sub vmconfig_update_net {
5178     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5179
5180     my $newnet = parse_net($value);
5181
5182     if ($conf->{$opt}) {
5183         my $oldnet = parse_net($conf->{$opt});
5184
5185         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5186             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5187             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5188             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5189
5190             # for non online change, we try to hot-unplug
5191             die "skip\n" if !$hotplug;
5192             vm_deviceunplug($vmid, $conf, $opt);
5193         } else {
5194
5195             die "internal error" if $opt !~ m/net(\d+)/;
5196             my $iface = "tap${vmid}i$1";
5197
5198             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5199                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5200                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5201                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5202                 PVE::Network::tap_unplug($iface);
5203
5204                 if ($have_sdn) {
5205                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5206                 } else {
5207                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5208                 }
5209             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5210                 # Rate can be applied on its own but any change above needs to
5211                 # include the rate in tap_plug since OVS resets everything.
5212                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5213             }
5214
5215             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5216                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5217             }
5218
5219             return 1;
5220         }
5221     }
5222
5223     if ($hotplug) {
5224         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5225     } else {
5226         die "skip\n";
5227     }
5228 }
5229
5230 sub vmconfig_update_agent {
5231     my ($conf, $opt, $value) = @_;
5232
5233     die "skip\n" if !$conf->{$opt};
5234
5235     my $hotplug_options = { fstrim_cloned_disks => 1 };
5236
5237     my $old_agent = parse_guest_agent($conf);
5238     my $agent = parse_guest_agent({$opt => $value});
5239
5240     for my $option (keys %$agent) { # added/changed options
5241         next if defined($hotplug_options->{$option});
5242         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5243     }
5244
5245     for my $option (keys %$old_agent) { # removed options
5246         next if defined($hotplug_options->{$option});
5247         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5248     }
5249
5250     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5251 }
5252
5253 sub vmconfig_update_disk {
5254     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5255
5256     my $drive = parse_drive($opt, $value);
5257
5258     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5259         my $media = $drive->{media} || 'disk';
5260         my $oldmedia = $old_drive->{media} || 'disk';
5261         die "unable to change media type\n" if $media ne $oldmedia;
5262
5263         if (!drive_is_cdrom($old_drive)) {
5264
5265             if ($drive->{file} ne $old_drive->{file}) {
5266
5267                 die "skip\n" if !$hotplug;
5268
5269                 # unplug and register as unused
5270                 vm_deviceunplug($vmid, $conf, $opt);
5271                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5272
5273             } else {
5274                 # update existing disk
5275
5276                 # skip non hotpluggable value
5277                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5278                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5279                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5280                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5281                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5282                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5283                     die "skip\n";
5284                 }
5285
5286                 # apply throttle
5287                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5288                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5289                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5290                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5291                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5292                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5293                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5294                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5295                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5296                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5297                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5298                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5299                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5300                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5301                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5302                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5303                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5304                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5305
5306                     qemu_block_set_io_throttle(
5307                         $vmid,"drive-$opt",
5308                         ($drive->{mbps} || 0)*1024*1024,
5309                         ($drive->{mbps_rd} || 0)*1024*1024,
5310                         ($drive->{mbps_wr} || 0)*1024*1024,
5311                          $drive->{iops} || 0,
5312                          $drive->{iops_rd} || 0,
5313                          $drive->{iops_wr} || 0,
5314                         ($drive->{mbps_max} || 0)*1024*1024,
5315                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5316                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5317                          $drive->{iops_max} || 0,
5318                          $drive->{iops_rd_max} || 0,
5319                          $drive->{iops_wr_max} || 0,
5320                          $drive->{bps_max_length} || 1,
5321                          $drive->{bps_rd_max_length} || 1,
5322                          $drive->{bps_wr_max_length} || 1,
5323                          $drive->{iops_max_length} || 1,
5324                          $drive->{iops_rd_max_length} || 1,
5325                          $drive->{iops_wr_max_length} || 1,
5326                     );
5327
5328                 }
5329
5330                 return 1;
5331             }
5332
5333         } else { # cdrom
5334
5335             if ($drive->{file} eq 'none') {
5336                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5337                 if (drive_is_cloudinit($old_drive)) {
5338                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5339                 }
5340             } else {
5341                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5342
5343                 # force eject if locked
5344                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5345
5346                 if ($path) {
5347                     mon_cmd($vmid, "blockdev-change-medium",
5348                         id => "$opt", filename => "$path");
5349                 }
5350             }
5351
5352             return 1;
5353         }
5354     }
5355
5356     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5357     # hotplug new disks
5358     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5359     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5360 }
5361
5362 sub vmconfig_update_cloudinit_drive {
5363     my ($storecfg, $conf, $vmid) = @_;
5364
5365     my $cloudinit_ds = undef;
5366     my $cloudinit_drive = undef;
5367
5368     PVE::QemuConfig->foreach_volume($conf, sub {
5369         my ($ds, $drive) = @_;
5370         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5371             $cloudinit_ds = $ds;
5372             $cloudinit_drive = $drive;
5373         }
5374     });
5375
5376     return if !$cloudinit_drive;
5377
5378     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5379     my $running = PVE::QemuServer::check_running($vmid);
5380
5381     if ($running) {
5382         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5383         if ($path) {
5384             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5385             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5386         }
5387     }
5388 }
5389
5390 # called in locked context by incoming migration
5391 sub vm_migrate_get_nbd_disks {
5392     my ($storecfg, $conf, $replicated_volumes) = @_;
5393
5394     my $local_volumes = {};
5395     PVE::QemuConfig->foreach_volume($conf, sub {
5396         my ($ds, $drive) = @_;
5397
5398         return if drive_is_cdrom($drive);
5399         return if $ds eq 'tpmstate0';
5400
5401         my $volid = $drive->{file};
5402
5403         return if !$volid;
5404
5405         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5406
5407         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5408         return if $scfg->{shared};
5409
5410         # replicated disks re-use existing state via bitmap
5411         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5412         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5413     });
5414     return $local_volumes;
5415 }
5416
5417 # called in locked context by incoming migration
5418 sub vm_migrate_alloc_nbd_disks {
5419     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5420
5421     my $nbd = {};
5422     foreach my $opt (sort keys %$source_volumes) {
5423         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5424
5425         if ($use_existing) {
5426             $nbd->{$opt}->{drivestr} = print_drive($drive);
5427             $nbd->{$opt}->{volid} = $volid;
5428             $nbd->{$opt}->{replicated} = 1;
5429             next;
5430         }
5431
5432         # storage mapping + volname = regular migration
5433         # storage mapping + format = remote migration
5434         # order of precedence, filtered by whether storage supports it:
5435         # 1. explicit requested format
5436         # 2. format of current volume
5437         # 3. default format of storage
5438         if (!$storagemap->{identity}) {
5439             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5440             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5441             if (!$format || !grep { $format eq $_ } @$validFormats) {
5442                 if ($volname) {
5443                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5444                     my $fileFormat = qemu_img_format($scfg, $volname);
5445                     $format = $fileFormat
5446                         if grep { $fileFormat eq $_ } @$validFormats;
5447                 }
5448                 $format //= $defFormat;
5449             }
5450         } else {
5451             # can't happen for remote migration, so $volname is always defined
5452             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5453             $format = qemu_img_format($scfg, $volname);
5454         }
5455
5456         my $size = $drive->{size} / 1024;
5457         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5458         my $newdrive = $drive;
5459         $newdrive->{format} = $format;
5460         $newdrive->{file} = $newvolid;
5461         my $drivestr = print_drive($newdrive);
5462         $nbd->{$opt}->{drivestr} = $drivestr;
5463         $nbd->{$opt}->{volid} = $newvolid;
5464     }
5465
5466     return $nbd;
5467 }
5468
5469 # see vm_start_nolock for parameters, additionally:
5470 # migrate_opts:
5471 #   storagemap = parsed storage map for allocating NBD disks
5472 sub vm_start {
5473     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5474
5475     return PVE::QemuConfig->lock_config($vmid, sub {
5476         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5477
5478         die "you can't start a vm if it's a template\n"
5479             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5480
5481         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5482         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5483
5484         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5485
5486         if ($has_backup_lock && $running) {
5487             # a backup is currently running, attempt to start the guest in the
5488             # existing QEMU instance
5489             return vm_resume($vmid);
5490         }
5491
5492         PVE::QemuConfig->check_lock($conf)
5493             if !($params->{skiplock} || $has_suspended_lock);
5494
5495         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5496
5497         die "VM $vmid already running\n" if $running;
5498
5499         if (my $storagemap = $migrate_opts->{storagemap}) {
5500             my $replicated = $migrate_opts->{replicated_volumes};
5501             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5502             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5503
5504             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5505                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5506             }
5507         }
5508
5509         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5510     });
5511 }
5512
5513
5514 # params:
5515 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5516 #   skiplock => 0/1, skip checking for config lock
5517 #   skiptemplate => 0/1, skip checking whether VM is template
5518 #   forcemachine => to force Qemu machine (rollback/migration)
5519 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5520 #   timeout => in seconds
5521 #   paused => start VM in paused state (backup)
5522 #   resume => resume from hibernation
5523 #   pbs-backing => {
5524 #      sata0 => {
5525 #         repository
5526 #         snapshot
5527 #         keyfile
5528 #         archive
5529 #      },
5530 #      virtio2 => ...
5531 #   }
5532 # migrate_opts:
5533 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5534 #   migratedfrom => source node
5535 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5536 #   network => CIDR of migration network
5537 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5538 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5539 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5540 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5541 #       contained in config
5542 sub vm_start_nolock {
5543     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5544
5545     my $statefile = $params->{statefile};
5546     my $resume = $params->{resume};
5547
5548     my $migratedfrom = $migrate_opts->{migratedfrom};
5549     my $migration_type = $migrate_opts->{type};
5550
5551     my $res = {};
5552
5553     # clean up leftover reboot request files
5554     eval { clear_reboot_request($vmid); };
5555     warn $@ if $@;
5556
5557     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5558         vmconfig_apply_pending($vmid, $conf, $storecfg);
5559         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5560     }
5561
5562     # don't regenerate the ISO if the VM is started as part of a live migration
5563     # this way we can reuse the old ISO with the correct config
5564     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5565
5566     # override offline migrated volumes, conf is out of date still
5567     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5568         for my $key (sort keys $offline_volumes->%*) {
5569             my $parsed = parse_drive($key, $conf->{$key});
5570             $parsed->{file} = $offline_volumes->{$key};
5571             $conf->{$key} = print_drive($parsed);
5572         }
5573     }
5574
5575     my $defaults = load_defaults();
5576
5577     # set environment variable useful inside network script
5578     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5579
5580     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5581
5582     my $forcemachine = $params->{forcemachine};
5583     my $forcecpu = $params->{forcecpu};
5584     if ($resume) {
5585         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5586         $forcemachine = $conf->{runningmachine};
5587         $forcecpu = $conf->{runningcpu};
5588         print "Resuming suspended VM\n";
5589     }
5590
5591     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5592         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5593
5594     my $migration_ip;
5595     my $get_migration_ip = sub {
5596         my ($nodename) = @_;
5597
5598         return $migration_ip if defined($migration_ip);
5599
5600         my $cidr = $migrate_opts->{network};
5601
5602         if (!defined($cidr)) {
5603             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5604             $cidr = $dc_conf->{migration}->{network};
5605         }
5606
5607         if (defined($cidr)) {
5608             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5609
5610             die "could not get IP: no address configured on local " .
5611                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5612
5613             die "could not get IP: multiple addresses configured on local " .
5614                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5615
5616             $migration_ip = @$ips[0];
5617         }
5618
5619         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5620             if !defined($migration_ip);
5621
5622         return $migration_ip;
5623     };
5624
5625     my $migrate_uri;
5626     if ($statefile) {
5627         if ($statefile eq 'tcp') {
5628             my $localip = "localhost";
5629             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5630             my $nodename = nodename();
5631
5632             if (!defined($migration_type)) {
5633                 if (defined($datacenterconf->{migration}->{type})) {
5634                     $migration_type = $datacenterconf->{migration}->{type};
5635                 } else {
5636                     $migration_type = 'secure';
5637                 }
5638             }
5639
5640             if ($migration_type eq 'insecure') {
5641                 $localip = $get_migration_ip->($nodename);
5642                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5643             }
5644
5645             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5646             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5647             $migrate_uri = "tcp:${localip}:${migrate_port}";
5648             push @$cmd, '-incoming', $migrate_uri;
5649             push @$cmd, '-S';
5650
5651         } elsif ($statefile eq 'unix') {
5652             # should be default for secure migrations as a ssh TCP forward
5653             # tunnel is not deterministic reliable ready and fails regurarly
5654             # to set up in time, so use UNIX socket forwards
5655             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5656             unlink $socket_addr;
5657
5658             $migrate_uri = "unix:$socket_addr";
5659
5660             push @$cmd, '-incoming', $migrate_uri;
5661             push @$cmd, '-S';
5662
5663         } elsif (-e $statefile) {
5664             push @$cmd, '-loadstate', $statefile;
5665         } else {
5666             my $statepath = PVE::Storage::path($storecfg, $statefile);
5667             push @$vollist, $statefile;
5668             push @$cmd, '-loadstate', $statepath;
5669         }
5670     } elsif ($params->{paused}) {
5671         push @$cmd, '-S';
5672     }
5673
5674     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5675
5676     my $pci_devices = {}; # host pci devices
5677     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5678         my $dev = $conf->{"hostpci$i"} or next;
5679         $pci_devices->{$i} = parse_hostpci($dev);
5680     }
5681
5682     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5683     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5684
5685     # map to a flat list of pci ids
5686     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5687
5688     # reserve all PCI IDs before actually doing anything with them
5689     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5690
5691     eval {
5692         my $uuid;
5693         for my $id (sort keys %$pci_devices) {
5694             my $d = $pci_devices->{$id};
5695             for my $dev ($d->{pciid}->@*) {
5696                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5697
5698                 # nvidia grid needs the uuid of the mdev as qemu parameter
5699                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5700                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5701                 }
5702             }
5703         }
5704         push @$cmd, '-uuid', $uuid if defined($uuid);
5705     };
5706     if (my $err = $@) {
5707         eval { cleanup_pci_devices($vmid, $conf) };
5708         warn $@ if $@;
5709         die $err;
5710     }
5711
5712     PVE::Storage::activate_volumes($storecfg, $vollist);
5713
5714     eval {
5715         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5716     };
5717     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5718     # timeout should be more than enough here...
5719     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5720
5721     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5722
5723     my %run_params = (
5724         timeout => $statefile ? undef : $start_timeout,
5725         umask => 0077,
5726         noerr => 1,
5727     );
5728
5729     # when migrating, prefix QEMU output so other side can pick up any
5730     # errors that might occur and show the user
5731     if ($migratedfrom) {
5732         $run_params{quiet} = 1;
5733         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5734     }
5735
5736     my %systemd_properties = (
5737         Slice => 'qemu.slice',
5738         KillMode => 'process',
5739         SendSIGKILL => 0,
5740         TimeoutStopUSec => ULONG_MAX, # infinity
5741     );
5742
5743     if (PVE::CGroup::cgroup_mode() == 2) {
5744         $systemd_properties{CPUWeight} = $cpuunits;
5745     } else {
5746         $systemd_properties{CPUShares} = $cpuunits;
5747     }
5748
5749     if (my $cpulimit = $conf->{cpulimit}) {
5750         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5751     }
5752     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5753
5754     my $run_qemu = sub {
5755         PVE::Tools::run_fork sub {
5756             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5757
5758             my $tpmpid;
5759             if (my $tpm = $conf->{tpmstate0}) {
5760                 # start the TPM emulator so QEMU can connect on start
5761                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5762             }
5763
5764             my $exitcode = run_command($cmd, %run_params);
5765             if ($exitcode) {
5766                 if ($tpmpid) {
5767                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5768                     kill 'TERM', $tpmpid;
5769                 }
5770                 die "QEMU exited with code $exitcode\n";
5771             }
5772         };
5773     };
5774
5775     if ($conf->{hugepages}) {
5776
5777         my $code = sub {
5778             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5779             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5780
5781             PVE::QemuServer::Memory::hugepages_mount();
5782             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5783
5784             eval { $run_qemu->() };
5785             if (my $err = $@) {
5786                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5787                     if !$conf->{keephugepages};
5788                 die $err;
5789             }
5790
5791             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5792                 if !$conf->{keephugepages};
5793         };
5794         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5795
5796     } else {
5797         eval { $run_qemu->() };
5798     }
5799
5800     if (my $err = $@) {
5801         # deactivate volumes if start fails
5802         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5803         warn $@ if $@;
5804         eval { cleanup_pci_devices($vmid, $conf) };
5805         warn $@ if $@;
5806
5807         die "start failed: $err";
5808     }
5809
5810     # re-reserve all PCI IDs now that we can know the actual VM PID
5811     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5812     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5813     warn $@ if $@;
5814
5815     print "migration listens on $migrate_uri\n" if $migrate_uri;
5816     $res->{migrate_uri} = $migrate_uri;
5817
5818     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5819         eval { mon_cmd($vmid, "cont"); };
5820         warn $@ if $@;
5821     }
5822
5823     #start nbd server for storage migration
5824     if (my $nbd = $migrate_opts->{nbd}) {
5825         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5826
5827         my $migrate_storage_uri;
5828         # nbd_protocol_version > 0 for unix socket support
5829         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5830             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5831             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5832             $migrate_storage_uri = "nbd:unix:$socket_path";
5833         } else {
5834             my $nodename = nodename();
5835             my $localip = $get_migration_ip->($nodename);
5836             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5837             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5838
5839             mon_cmd($vmid, "nbd-server-start", addr => {
5840                 type => 'inet',
5841                 data => {
5842                     host => "${localip}",
5843                     port => "${storage_migrate_port}",
5844                 },
5845             });
5846             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5847             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5848         }
5849
5850         $res->{migrate_storage_uri} = $migrate_storage_uri;
5851
5852         foreach my $opt (sort keys %$nbd) {
5853             my $drivestr = $nbd->{$opt}->{drivestr};
5854             my $volid = $nbd->{$opt}->{volid};
5855             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5856             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5857             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5858             print "re-using replicated volume: $opt - $volid\n"
5859                 if $nbd->{$opt}->{replicated};
5860
5861             $res->{drives}->{$opt} = $nbd->{$opt};
5862             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5863         }
5864     }
5865
5866     if ($migratedfrom) {
5867         eval {
5868             set_migration_caps($vmid);
5869         };
5870         warn $@ if $@;
5871
5872         if ($spice_port) {
5873             print "spice listens on port $spice_port\n";
5874             $res->{spice_port} = $spice_port;
5875             if ($migrate_opts->{spice_ticket}) {
5876                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5877                     $migrate_opts->{spice_ticket});
5878                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5879             }
5880         }
5881
5882     } else {
5883         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5884             if !$statefile && $conf->{balloon};
5885
5886         foreach my $opt (keys %$conf) {
5887             next if $opt !~  m/^net\d+$/;
5888             my $nicconf = parse_net($conf->{$opt});
5889             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5890         }
5891         add_nets_bridge_fdb($conf, $vmid);
5892     }
5893
5894     mon_cmd($vmid, 'qom-set',
5895                 path => "machine/peripheral/balloon0",
5896                 property => "guest-stats-polling-interval",
5897                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5898
5899     if ($resume) {
5900         print "Resumed VM, removing state\n";
5901         if (my $vmstate = $conf->{vmstate}) {
5902             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5903             PVE::Storage::vdisk_free($storecfg, $vmstate);
5904         }
5905         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5906         PVE::QemuConfig->write_config($vmid, $conf);
5907     }
5908
5909     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5910
5911     return $res;
5912 }
5913
5914 sub vm_commandline {
5915     my ($storecfg, $vmid, $snapname) = @_;
5916
5917     my $conf = PVE::QemuConfig->load_config($vmid);
5918
5919     my ($forcemachine, $forcecpu);
5920     if ($snapname) {
5921         my $snapshot = $conf->{snapshots}->{$snapname};
5922         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5923
5924         # check for machine or CPU overrides in snapshot
5925         $forcemachine = $snapshot->{runningmachine};
5926         $forcecpu = $snapshot->{runningcpu};
5927
5928         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5929
5930         $conf = $snapshot;
5931     }
5932
5933     my $defaults = load_defaults();
5934
5935     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5936
5937     return PVE::Tools::cmd2string($cmd);
5938 }
5939
5940 sub vm_reset {
5941     my ($vmid, $skiplock) = @_;
5942
5943     PVE::QemuConfig->lock_config($vmid, sub {
5944
5945         my $conf = PVE::QemuConfig->load_config($vmid);
5946
5947         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5948
5949         mon_cmd($vmid, "system_reset");
5950     });
5951 }
5952
5953 sub get_vm_volumes {
5954     my ($conf) = @_;
5955
5956     my $vollist = [];
5957     foreach_volid($conf, sub {
5958         my ($volid, $attr) = @_;
5959
5960         return if $volid =~ m|^/|;
5961
5962         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5963         return if !$sid;
5964
5965         push @$vollist, $volid;
5966     });
5967
5968     return $vollist;
5969 }
5970
5971 sub cleanup_pci_devices {
5972     my ($vmid, $conf) = @_;
5973
5974     foreach my $key (keys %$conf) {
5975         next if $key !~ m/^hostpci(\d+)$/;
5976         my $hostpciindex = $1;
5977         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5978         my $d = parse_hostpci($conf->{$key});
5979         if ($d->{mdev}) {
5980             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
5981             # don't want to break ABI just for this two liner
5982             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
5983             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
5984         }
5985     }
5986     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
5987 }
5988
5989 sub vm_stop_cleanup {
5990     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5991
5992     eval {
5993
5994         if (!$keepActive) {
5995             my $vollist = get_vm_volumes($conf);
5996             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5997
5998             if (my $tpmdrive = $conf->{tpmstate0}) {
5999                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6000                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6001                 if ($storeid) {
6002                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6003                 }
6004             }
6005         }
6006
6007         foreach my $ext (qw(mon qmp pid vnc qga)) {
6008             unlink "/var/run/qemu-server/${vmid}.$ext";
6009         }
6010
6011         if ($conf->{ivshmem}) {
6012             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6013             # just delete it for now, VMs which have this already open do not
6014             # are affected, but new VMs will get a separated one. If this
6015             # becomes an issue we either add some sort of ref-counting or just
6016             # add a "don't delete on stop" flag to the ivshmem format.
6017             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6018         }
6019
6020         cleanup_pci_devices($vmid, $conf);
6021
6022         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6023     };
6024     warn $@ if $@; # avoid errors - just warn
6025 }
6026
6027 # call only in locked context
6028 sub _do_vm_stop {
6029     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6030
6031     my $pid = check_running($vmid, $nocheck);
6032     return if !$pid;
6033
6034     my $conf;
6035     if (!$nocheck) {
6036         $conf = PVE::QemuConfig->load_config($vmid);
6037         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6038         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6039             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6040             $timeout = $opts->{down} if $opts->{down};
6041         }
6042         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6043     }
6044
6045     eval {
6046         if ($shutdown) {
6047             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6048                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6049             } else {
6050                 mon_cmd($vmid, "system_powerdown");
6051             }
6052         } else {
6053             mon_cmd($vmid, "quit");
6054         }
6055     };
6056     my $err = $@;
6057
6058     if (!$err) {
6059         $timeout = 60 if !defined($timeout);
6060
6061         my $count = 0;
6062         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6063             $count++;
6064             sleep 1;
6065         }
6066
6067         if ($count >= $timeout) {
6068             if ($force) {
6069                 warn "VM still running - terminating now with SIGTERM\n";
6070                 kill 15, $pid;
6071             } else {
6072                 die "VM quit/powerdown failed - got timeout\n";
6073             }
6074         } else {
6075             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6076             return;
6077         }
6078     } else {
6079        if (!check_running($vmid, $nocheck)) {
6080             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6081             return;
6082        }
6083        if ($force) {
6084             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6085             kill 15, $pid;
6086         } else {
6087             die "VM quit/powerdown failed\n";
6088         }
6089     }
6090
6091     # wait again
6092     $timeout = 10;
6093
6094     my $count = 0;
6095     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6096         $count++;
6097         sleep 1;
6098     }
6099
6100     if ($count >= $timeout) {
6101         warn "VM still running - terminating now with SIGKILL\n";
6102         kill 9, $pid;
6103         sleep 1;
6104     }
6105
6106     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6107 }
6108
6109 # Note: use $nocheck to skip tests if VM configuration file exists.
6110 # We need that when migration VMs to other nodes (files already moved)
6111 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6112 sub vm_stop {
6113     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6114
6115     $force = 1 if !defined($force) && !$shutdown;
6116
6117     if ($migratedfrom){
6118         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6119         kill 15, $pid if $pid;
6120         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6121         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6122         return;
6123     }
6124
6125     PVE::QemuConfig->lock_config($vmid, sub {
6126         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6127    });
6128 }
6129
6130 sub vm_reboot {
6131     my ($vmid, $timeout) = @_;
6132
6133     PVE::QemuConfig->lock_config($vmid, sub {
6134         eval {
6135
6136             # only reboot if running, as qmeventd starts it again on a stop event
6137             return if !check_running($vmid);
6138
6139             create_reboot_request($vmid);
6140
6141             my $storecfg = PVE::Storage::config();
6142             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6143
6144         };
6145         if (my $err = $@) {
6146             # avoid that the next normal shutdown will be confused for a reboot
6147             clear_reboot_request($vmid);
6148             die $err;
6149         }
6150    });
6151 }
6152
6153 # note: if using the statestorage parameter, the caller has to check privileges
6154 sub vm_suspend {
6155     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6156
6157     my $conf;
6158     my $path;
6159     my $storecfg;
6160     my $vmstate;
6161
6162     PVE::QemuConfig->lock_config($vmid, sub {
6163
6164         $conf = PVE::QemuConfig->load_config($vmid);
6165
6166         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6167         PVE::QemuConfig->check_lock($conf)
6168             if !($skiplock || $is_backing_up);
6169
6170         die "cannot suspend to disk during backup\n"
6171             if $is_backing_up && $includestate;
6172
6173         if ($includestate) {
6174             $conf->{lock} = 'suspending';
6175             my $date = strftime("%Y-%m-%d", localtime(time()));
6176             $storecfg = PVE::Storage::config();
6177             if (!$statestorage) {
6178                 $statestorage = find_vmstate_storage($conf, $storecfg);
6179                 # check permissions for the storage
6180                 my $rpcenv = PVE::RPCEnvironment::get();
6181                 if ($rpcenv->{type} ne 'cli') {
6182                     my $authuser = $rpcenv->get_user();
6183                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6184                 }
6185             }
6186
6187
6188             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6189                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6190             $path = PVE::Storage::path($storecfg, $vmstate);
6191             PVE::QemuConfig->write_config($vmid, $conf);
6192         } else {
6193             mon_cmd($vmid, "stop");
6194         }
6195     });
6196
6197     if ($includestate) {
6198         # save vm state
6199         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6200
6201         eval {
6202             set_migration_caps($vmid, 1);
6203             mon_cmd($vmid, "savevm-start", statefile => $path);
6204             for(;;) {
6205                 my $state = mon_cmd($vmid, "query-savevm");
6206                 if (!$state->{status}) {
6207                     die "savevm not active\n";
6208                 } elsif ($state->{status} eq 'active') {
6209                     sleep(1);
6210                     next;
6211                 } elsif ($state->{status} eq 'completed') {
6212                     print "State saved, quitting\n";
6213                     last;
6214                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6215                     die "query-savevm failed with error '$state->{error}'\n"
6216                 } else {
6217                     die "query-savevm returned status '$state->{status}'\n";
6218                 }
6219             }
6220         };
6221         my $err = $@;
6222
6223         PVE::QemuConfig->lock_config($vmid, sub {
6224             $conf = PVE::QemuConfig->load_config($vmid);
6225             if ($err) {
6226                 # cleanup, but leave suspending lock, to indicate something went wrong
6227                 eval {
6228                     mon_cmd($vmid, "savevm-end");
6229                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6230                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6231                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6232                     PVE::QemuConfig->write_config($vmid, $conf);
6233                 };
6234                 warn $@ if $@;
6235                 die $err;
6236             }
6237
6238             die "lock changed unexpectedly\n"
6239                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6240
6241             mon_cmd($vmid, "quit");
6242             $conf->{lock} = 'suspended';
6243             PVE::QemuConfig->write_config($vmid, $conf);
6244         });
6245     }
6246 }
6247
6248 sub vm_resume {
6249     my ($vmid, $skiplock, $nocheck) = @_;
6250
6251     PVE::QemuConfig->lock_config($vmid, sub {
6252         my $res = mon_cmd($vmid, 'query-status');
6253         my $resume_cmd = 'cont';
6254         my $reset = 0;
6255         my $conf = PVE::QemuConfig->load_config($vmid);
6256
6257         if ($res->{status}) {
6258             return if $res->{status} eq 'running'; # job done, go home
6259             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6260             $reset = 1 if $res->{status} eq 'shutdown';
6261         }
6262
6263         if (!$nocheck) {
6264
6265             PVE::QemuConfig->check_lock($conf)
6266                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6267         }
6268
6269         if ($reset) {
6270             # required if a VM shuts down during a backup and we get a resume
6271             # request before the backup finishes for example
6272             mon_cmd($vmid, "system_reset");
6273         }
6274
6275         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6276
6277         mon_cmd($vmid, $resume_cmd);
6278     });
6279 }
6280
6281 sub vm_sendkey {
6282     my ($vmid, $skiplock, $key) = @_;
6283
6284     PVE::QemuConfig->lock_config($vmid, sub {
6285
6286         my $conf = PVE::QemuConfig->load_config($vmid);
6287
6288         # there is no qmp command, so we use the human monitor command
6289         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6290         die $res if $res ne '';
6291     });
6292 }
6293
6294 # vzdump restore implementaion
6295
6296 sub tar_archive_read_firstfile {
6297     my $archive = shift;
6298
6299     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6300
6301     # try to detect archive type first
6302     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6303         die "unable to open file '$archive'\n";
6304     my $firstfile = <$fh>;
6305     kill 15, $pid;
6306     close $fh;
6307
6308     die "ERROR: archive contaions no data\n" if !$firstfile;
6309     chomp $firstfile;
6310
6311     return $firstfile;
6312 }
6313
6314 sub tar_restore_cleanup {
6315     my ($storecfg, $statfile) = @_;
6316
6317     print STDERR "starting cleanup\n";
6318
6319     if (my $fd = IO::File->new($statfile, "r")) {
6320         while (defined(my $line = <$fd>)) {
6321             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6322                 my $volid = $2;
6323                 eval {
6324                     if ($volid =~ m|^/|) {
6325                         unlink $volid || die 'unlink failed\n';
6326                     } else {
6327                         PVE::Storage::vdisk_free($storecfg, $volid);
6328                     }
6329                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6330                 };
6331                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6332             } else {
6333                 print STDERR "unable to parse line in statfile - $line";
6334             }
6335         }
6336         $fd->close();
6337     }
6338 }
6339
6340 sub restore_file_archive {
6341     my ($archive, $vmid, $user, $opts) = @_;
6342
6343     return restore_vma_archive($archive, $vmid, $user, $opts)
6344         if $archive eq '-';
6345
6346     my $info = PVE::Storage::archive_info($archive);
6347     my $format = $opts->{format} // $info->{format};
6348     my $comp = $info->{compression};
6349
6350     # try to detect archive format
6351     if ($format eq 'tar') {
6352         return restore_tar_archive($archive, $vmid, $user, $opts);
6353     } else {
6354         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6355     }
6356 }
6357
6358 # hepler to remove disks that will not be used after restore
6359 my $restore_cleanup_oldconf = sub {
6360     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6361
6362     my $kept_disks = {};
6363
6364     PVE::QemuConfig->foreach_volume($oldconf, sub {
6365         my ($ds, $drive) = @_;
6366
6367         return if drive_is_cdrom($drive, 1);
6368
6369         my $volid = $drive->{file};
6370         return if !$volid || $volid =~ m|^/|;
6371
6372         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6373         return if !$path || !$owner || ($owner != $vmid);
6374
6375         # Note: only delete disk we want to restore
6376         # other volumes will become unused
6377         if ($virtdev_hash->{$ds}) {
6378             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6379             if (my $err = $@) {
6380                 warn $err;
6381             }
6382         } else {
6383             $kept_disks->{$volid} = 1;
6384         }
6385     });
6386
6387     # after the restore we have no snapshots anymore
6388     for my $snapname (keys $oldconf->{snapshots}->%*) {
6389         my $snap = $oldconf->{snapshots}->{$snapname};
6390         if ($snap->{vmstate}) {
6391             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6392             if (my $err = $@) {
6393                 warn $err;
6394             }
6395         }
6396
6397         for my $volid (keys $kept_disks->%*) {
6398             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6399             warn $@ if $@;
6400         }
6401     }
6402 };
6403
6404 # Helper to parse vzdump backup device hints
6405 #
6406 # $rpcenv: Environment, used to ckeck storage permissions
6407 # $user: User ID, to check storage permissions
6408 # $storecfg: Storage configuration
6409 # $fh: the file handle for reading the configuration
6410 # $devinfo: should contain device sizes for all backu-up'ed devices
6411 # $options: backup options (pool, default storage)
6412 #
6413 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6414 my $parse_backup_hints = sub {
6415     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6416
6417     my $check_storage = sub { # assert if an image can be allocate
6418         my ($storeid, $scfg) = @_;
6419         die "Content type 'images' is not available on storage '$storeid'\n"
6420             if !$scfg->{content}->{images};
6421         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6422             if $user ne 'root@pam';
6423     };
6424
6425     my $virtdev_hash = {};
6426     while (defined(my $line = <$fh>)) {
6427         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6428             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6429             die "archive does not contain data for drive '$virtdev'\n"
6430                 if !$devinfo->{$devname};
6431
6432             if (defined($options->{storage})) {
6433                 $storeid = $options->{storage} || 'local';
6434             } elsif (!$storeid) {
6435                 $storeid = 'local';
6436             }
6437             $format = 'raw' if !$format;
6438             $devinfo->{$devname}->{devname} = $devname;
6439             $devinfo->{$devname}->{virtdev} = $virtdev;
6440             $devinfo->{$devname}->{format} = $format;
6441             $devinfo->{$devname}->{storeid} = $storeid;
6442
6443             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6444             $check_storage->($storeid, $scfg); # permission and content type check
6445
6446             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6447         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6448             my $virtdev = $1;
6449             my $drive = parse_drive($virtdev, $2);
6450
6451             if (drive_is_cloudinit($drive)) {
6452                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6453                 $storeid = $options->{storage} if defined ($options->{storage});
6454                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6455                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6456
6457                 $check_storage->($storeid, $scfg); # permission and content type check
6458
6459                 $virtdev_hash->{$virtdev} = {
6460                     format => $format,
6461                     storeid => $storeid,
6462                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6463                     is_cloudinit => 1,
6464                 };
6465             }
6466         }
6467     }
6468
6469     return $virtdev_hash;
6470 };
6471
6472 # Helper to allocate and activate all volumes required for a restore
6473 #
6474 # $storecfg: Storage configuration
6475 # $virtdev_hash: as returned by parse_backup_hints()
6476 #
6477 # Returns: { $virtdev => $volid }
6478 my $restore_allocate_devices = sub {
6479     my ($storecfg, $virtdev_hash, $vmid) = @_;
6480
6481     my $map = {};
6482     foreach my $virtdev (sort keys %$virtdev_hash) {
6483         my $d = $virtdev_hash->{$virtdev};
6484         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6485         my $storeid = $d->{storeid};
6486         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6487
6488         # test if requested format is supported
6489         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6490         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6491         $d->{format} = $defFormat if !$supported;
6492
6493         my $name;
6494         if ($d->{is_cloudinit}) {
6495             $name = "vm-$vmid-cloudinit";
6496             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6497             if ($scfg->{path}) {
6498                 $name .= ".$d->{format}";
6499             }
6500         }
6501
6502         my $volid = PVE::Storage::vdisk_alloc(
6503             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6504
6505         print STDERR "new volume ID is '$volid'\n";
6506         $d->{volid} = $volid;
6507
6508         PVE::Storage::activate_volumes($storecfg, [$volid]);
6509
6510         $map->{$virtdev} = $volid;
6511     }
6512
6513     return $map;
6514 };
6515
6516 sub restore_update_config_line {
6517     my ($cookie, $map, $line, $unique) = @_;
6518
6519     return '' if $line =~ m/^\#qmdump\#/;
6520     return '' if $line =~ m/^\#vzdump\#/;
6521     return '' if $line =~ m/^lock:/;
6522     return '' if $line =~ m/^unused\d+:/;
6523     return '' if $line =~ m/^parent:/;
6524
6525     my $res = '';
6526
6527     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6528     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6529         # try to convert old 1.X settings
6530         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6531         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6532             my ($model, $macaddr) = split(/\=/, $devconfig);
6533             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6534             my $net = {
6535                 model => $model,
6536                 bridge => "vmbr$ind",
6537                 macaddr => $macaddr,
6538             };
6539             my $netstr = print_net($net);
6540
6541             $res .= "net$cookie->{netcount}: $netstr\n";
6542             $cookie->{netcount}++;
6543         }
6544     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6545         my ($id, $netstr) = ($1, $2);
6546         my $net = parse_net($netstr);
6547         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6548         $netstr = print_net($net);
6549         $res .= "$id: $netstr\n";
6550     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6551         my $virtdev = $1;
6552         my $value = $3;
6553         my $di = parse_drive($virtdev, $value);
6554         if (defined($di->{backup}) && !$di->{backup}) {
6555             $res .= "#$line";
6556         } elsif ($map->{$virtdev}) {
6557             delete $di->{format}; # format can change on restore
6558             $di->{file} = $map->{$virtdev};
6559             $value = print_drive($di);
6560             $res .= "$virtdev: $value\n";
6561         } else {
6562             $res .= $line;
6563         }
6564     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6565         my $vmgenid = $1;
6566         if ($vmgenid ne '0') {
6567             # always generate a new vmgenid if there was a valid one setup
6568             $vmgenid = generate_uuid();
6569         }
6570         $res .= "vmgenid: $vmgenid\n";
6571     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6572         my ($uuid, $uuid_str);
6573         UUID::generate($uuid);
6574         UUID::unparse($uuid, $uuid_str);
6575         my $smbios1 = parse_smbios1($2);
6576         $smbios1->{uuid} = $uuid_str;
6577         $res .= $1.print_smbios1($smbios1)."\n";
6578     } else {
6579         $res .= $line;
6580     }
6581
6582     return $res;
6583 }
6584
6585 my $restore_deactivate_volumes = sub {
6586     my ($storecfg, $virtdev_hash) = @_;
6587
6588     my $vollist = [];
6589     for my $dev (values $virtdev_hash->%*) {
6590         push $vollist->@*, $dev->{volid} if $dev->{volid};
6591     }
6592
6593     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6594     print STDERR $@ if $@;
6595 };
6596
6597 my $restore_destroy_volumes = sub {
6598     my ($storecfg, $virtdev_hash) = @_;
6599
6600     for my $dev (values $virtdev_hash->%*) {
6601         my $volid = $dev->{volid} or next;
6602         eval {
6603             PVE::Storage::vdisk_free($storecfg, $volid);
6604             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6605         };
6606         print STDERR "unable to cleanup '$volid' - $@" if $@;
6607     }
6608 };
6609
6610 my $restore_merge_config = sub {
6611     my ($filename, $backup_conf_raw, $override_conf) = @_;
6612
6613     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6614     for my $key (keys $override_conf->%*) {
6615         $backup_conf->{$key} = $override_conf->{$key};
6616     }
6617
6618     return $backup_conf;
6619 };
6620
6621 sub scan_volids {
6622     my ($cfg, $vmid) = @_;
6623
6624     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6625
6626     my $volid_hash = {};
6627     foreach my $storeid (keys %$info) {
6628         foreach my $item (@{$info->{$storeid}}) {
6629             next if !($item->{volid} && $item->{size});
6630             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6631             $volid_hash->{$item->{volid}} = $item;
6632         }
6633     }
6634
6635     return $volid_hash;
6636 }
6637
6638 sub update_disk_config {
6639     my ($vmid, $conf, $volid_hash) = @_;
6640
6641     my $changes;
6642     my $prefix = "VM $vmid";
6643
6644     # used and unused disks
6645     my $referenced = {};
6646
6647     # Note: it is allowed to define multiple storages with same path (alias), so
6648     # we need to check both 'volid' and real 'path' (two different volid can point
6649     # to the same path).
6650
6651     my $referencedpath = {};
6652
6653     # update size info
6654     PVE::QemuConfig->foreach_volume($conf, sub {
6655         my ($opt, $drive) = @_;
6656
6657         my $volid = $drive->{file};
6658         return if !$volid;
6659         my $volume = $volid_hash->{$volid};
6660
6661         # mark volid as "in-use" for next step
6662         $referenced->{$volid} = 1;
6663         if ($volume && (my $path = $volume->{path})) {
6664             $referencedpath->{$path} = 1;
6665         }
6666
6667         return if drive_is_cdrom($drive);
6668         return if !$volume;
6669
6670         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6671         if (defined($updated)) {
6672             $changes = 1;
6673             $conf->{$opt} = print_drive($updated);
6674             print "$prefix ($opt): $msg\n";
6675         }
6676     });
6677
6678     # remove 'unusedX' entry if volume is used
6679     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6680         my ($opt, $drive) = @_;
6681
6682         my $volid = $drive->{file};
6683         return if !$volid;
6684
6685         my $path;
6686         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6687         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6688             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6689             $changes = 1;
6690             delete $conf->{$opt};
6691         }
6692
6693         $referenced->{$volid} = 1;
6694         $referencedpath->{$path} = 1 if $path;
6695     });
6696
6697     foreach my $volid (sort keys %$volid_hash) {
6698         next if $volid =~ m/vm-$vmid-state-/;
6699         next if $referenced->{$volid};
6700         my $path = $volid_hash->{$volid}->{path};
6701         next if !$path; # just to be sure
6702         next if $referencedpath->{$path};
6703         $changes = 1;
6704         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6705         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6706         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6707     }
6708
6709     return $changes;
6710 }
6711
6712 sub rescan {
6713     my ($vmid, $nolock, $dryrun) = @_;
6714
6715     my $cfg = PVE::Storage::config();
6716
6717     print "rescan volumes...\n";
6718     my $volid_hash = scan_volids($cfg, $vmid);
6719
6720     my $updatefn =  sub {
6721         my ($vmid) = @_;
6722
6723         my $conf = PVE::QemuConfig->load_config($vmid);
6724
6725         PVE::QemuConfig->check_lock($conf);
6726
6727         my $vm_volids = {};
6728         foreach my $volid (keys %$volid_hash) {
6729             my $info = $volid_hash->{$volid};
6730             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6731         }
6732
6733         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6734
6735         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6736     };
6737
6738     if (defined($vmid)) {
6739         if ($nolock) {
6740             &$updatefn($vmid);
6741         } else {
6742             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6743         }
6744     } else {
6745         my $vmlist = config_list();
6746         foreach my $vmid (keys %$vmlist) {
6747             if ($nolock) {
6748                 &$updatefn($vmid);
6749             } else {
6750                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6751             }
6752         }
6753     }
6754 }
6755
6756 sub restore_proxmox_backup_archive {
6757     my ($archive, $vmid, $user, $options) = @_;
6758
6759     my $storecfg = PVE::Storage::config();
6760
6761     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6762     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6763
6764     my $fingerprint = $scfg->{fingerprint};
6765     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6766
6767     my $repo = PVE::PBSClient::get_repository($scfg);
6768     my $namespace = $scfg->{namespace};
6769
6770     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6771     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6772     local $ENV{PBS_PASSWORD} = $password;
6773     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6774
6775     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6776         PVE::Storage::parse_volname($storecfg, $archive);
6777
6778     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6779
6780     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6781
6782     my $tmpdir = "/var/tmp/vzdumptmp$$";
6783     rmtree $tmpdir;
6784     mkpath $tmpdir;
6785
6786     my $conffile = PVE::QemuConfig->config_file($vmid);
6787      # disable interrupts (always do cleanups)
6788     local $SIG{INT} =
6789         local $SIG{TERM} =
6790         local $SIG{QUIT} =
6791         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6792
6793     # Note: $oldconf is undef if VM does not exists
6794     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6795     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6796     my $new_conf_raw = '';
6797
6798     my $rpcenv = PVE::RPCEnvironment::get();
6799     my $devinfo = {}; # info about drives included in backup
6800     my $virtdev_hash = {}; # info about allocated drives
6801
6802     eval {
6803         # enable interrupts
6804         local $SIG{INT} =
6805             local $SIG{TERM} =
6806             local $SIG{QUIT} =
6807             local $SIG{HUP} =
6808             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6809
6810         my $cfgfn = "$tmpdir/qemu-server.conf";
6811         my $firewall_config_fn = "$tmpdir/fw.conf";
6812         my $index_fn = "$tmpdir/index.json";
6813
6814         my $cmd = "restore";
6815
6816         my $param = [$pbs_backup_name, "index.json", $index_fn];
6817         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6818         my $index = PVE::Tools::file_get_contents($index_fn);
6819         $index = decode_json($index);
6820
6821         foreach my $info (@{$index->{files}}) {
6822             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6823                 my $devname = $1;
6824                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6825                     $devinfo->{$devname}->{size} = $1;
6826                 } else {
6827                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6828                 }
6829             }
6830         }
6831
6832         my $is_qemu_server_backup = scalar(
6833             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6834         );
6835         if (!$is_qemu_server_backup) {
6836             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6837         }
6838         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6839
6840         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6841         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6842
6843         if ($has_firewall_config) {
6844             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6845             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6846
6847             my $pve_firewall_dir = '/etc/pve/firewall';
6848             mkdir $pve_firewall_dir; # make sure the dir exists
6849             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6850         }
6851
6852         my $fh = IO::File->new($cfgfn, "r") ||
6853             die "unable to read qemu-server.conf - $!\n";
6854
6855         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6856
6857         # fixme: rate limit?
6858
6859         # create empty/temp config
6860         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6861
6862         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6863
6864         # allocate volumes
6865         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6866
6867         foreach my $virtdev (sort keys %$virtdev_hash) {
6868             my $d = $virtdev_hash->{$virtdev};
6869             next if $d->{is_cloudinit}; # no need to restore cloudinit
6870
6871             # this fails if storage is unavailable
6872             my $volid = $d->{volid};
6873             my $path = PVE::Storage::path($storecfg, $volid);
6874
6875             # for live-restore we only want to preload the efidisk and TPM state
6876             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6877
6878             my @ns_arg;
6879             if (defined(my $ns = $scfg->{namespace})) {
6880                 @ns_arg = ('--ns', $ns);
6881             }
6882
6883             my $pbs_restore_cmd = [
6884                 '/usr/bin/pbs-restore',
6885                 '--repository', $repo,
6886                 @ns_arg,
6887                 $pbs_backup_name,
6888                 "$d->{devname}.img.fidx",
6889                 $path,
6890                 '--verbose',
6891                 ];
6892
6893             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6894             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6895
6896             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6897                 push @$pbs_restore_cmd, '--skip-zero';
6898             }
6899
6900             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6901             print "restore proxmox backup image: $dbg_cmdstring\n";
6902             run_command($pbs_restore_cmd);
6903         }
6904
6905         $fh->seek(0, 0) || die "seek failed - $!\n";
6906
6907         my $cookie = { netcount => 0 };
6908         while (defined(my $line = <$fh>)) {
6909             $new_conf_raw .= restore_update_config_line(
6910                 $cookie,
6911                 $map,
6912                 $line,
6913                 $options->{unique},
6914             );
6915         }
6916
6917         $fh->close();
6918     };
6919     my $err = $@;
6920
6921     if ($err || !$options->{live}) {
6922         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6923     }
6924
6925     rmtree $tmpdir;
6926
6927     if ($err) {
6928         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6929         die $err;
6930     }
6931
6932     if ($options->{live}) {
6933         # keep lock during live-restore
6934         $new_conf_raw .= "\nlock: create";
6935     }
6936
6937     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6938     PVE::QemuConfig->write_config($vmid, $new_conf);
6939
6940     eval { rescan($vmid, 1); };
6941     warn $@ if $@;
6942
6943     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6944
6945     if ($options->{live}) {
6946         # enable interrupts
6947         local $SIG{INT} =
6948             local $SIG{TERM} =
6949             local $SIG{QUIT} =
6950             local $SIG{HUP} =
6951             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6952
6953         my $conf = PVE::QemuConfig->load_config($vmid);
6954         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6955
6956         # these special drives are already restored before start
6957         delete $devinfo->{'drive-efidisk0'};
6958         delete $devinfo->{'drive-tpmstate0-backup'};
6959
6960         my $pbs_opts = {
6961             repo => $repo,
6962             keyfile => $keyfile,
6963             snapshot => $pbs_backup_name,
6964             namespace => $namespace,
6965         };
6966         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6967
6968         PVE::QemuConfig->remove_lock($vmid, "create");
6969     }
6970 }
6971
6972 sub pbs_live_restore {
6973     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6974
6975     print "starting VM for live-restore\n";
6976     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6977
6978     my $pbs_backing = {};
6979     for my $ds (keys %$restored_disks) {
6980         $ds =~ m/^drive-(.*)$/;
6981         my $confname = $1;
6982         $pbs_backing->{$confname} = {
6983             repository => $opts->{repo},
6984             snapshot => $opts->{snapshot},
6985             archive => "$ds.img.fidx",
6986         };
6987         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6988         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6989
6990         my $drive = parse_drive($confname, $conf->{$confname});
6991         print "restoring '$ds' to '$drive->{file}'\n";
6992     }
6993
6994     my $drives_streamed = 0;
6995     eval {
6996         # make sure HA doesn't interrupt our restore by stopping the VM
6997         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6998             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6999         }
7000
7001         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7002         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7003         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7004
7005         my $qmeventd_fd = register_qmeventd_handle($vmid);
7006
7007         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7008         # this will effectively collapse the backing image chain consisting of
7009         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7010         # removes itself once all backing images vanish with 'auto-remove=on')
7011         my $jobs = {};
7012         for my $ds (sort keys %$restored_disks) {
7013             my $job_id = "restore-$ds";
7014             mon_cmd($vmid, 'block-stream',
7015                 'job-id' => $job_id,
7016                 device => "$ds",
7017             );
7018             $jobs->{$job_id} = {};
7019         }
7020
7021         mon_cmd($vmid, 'cont');
7022         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7023
7024         print "restore-drive jobs finished successfully, removing all tracking block devices"
7025             ." to disconnect from Proxmox Backup Server\n";
7026
7027         for my $ds (sort keys %$restored_disks) {
7028             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7029         }
7030
7031         close($qmeventd_fd);
7032     };
7033
7034     my $err = $@;
7035
7036     if ($err) {
7037         warn "An error occurred during live-restore: $err\n";
7038         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7039         die "live-restore failed\n";
7040     }
7041 }
7042
7043 sub restore_vma_archive {
7044     my ($archive, $vmid, $user, $opts, $comp) = @_;
7045
7046     my $readfrom = $archive;
7047
7048     my $cfg = PVE::Storage::config();
7049     my $commands = [];
7050     my $bwlimit = $opts->{bwlimit};
7051
7052     my $dbg_cmdstring = '';
7053     my $add_pipe = sub {
7054         my ($cmd) = @_;
7055         push @$commands, $cmd;
7056         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7057         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7058         $readfrom = '-';
7059     };
7060
7061     my $input = undef;
7062     if ($archive eq '-') {
7063         $input = '<&STDIN';
7064     } else {
7065         # If we use a backup from a PVE defined storage we also consider that
7066         # storage's rate limit:
7067         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7068         if (defined($volid)) {
7069             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7070             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7071             if ($readlimit) {
7072                 print STDERR "applying read rate limit: $readlimit\n";
7073                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7074                 $add_pipe->($cstream);
7075             }
7076         }
7077     }
7078
7079     if ($comp) {
7080         my $info = PVE::Storage::decompressor_info('vma', $comp);
7081         my $cmd = $info->{decompressor};
7082         push @$cmd, $readfrom;
7083         $add_pipe->($cmd);
7084     }
7085
7086     my $tmpdir = "/var/tmp/vzdumptmp$$";
7087     rmtree $tmpdir;
7088
7089     # disable interrupts (always do cleanups)
7090     local $SIG{INT} =
7091         local $SIG{TERM} =
7092         local $SIG{QUIT} =
7093         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7094
7095     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7096     POSIX::mkfifo($mapfifo, 0600);
7097     my $fifofh;
7098     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7099
7100     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7101
7102     my $oldtimeout;
7103     my $timeout = 5;
7104
7105     my $devinfo = {}; # info about drives included in backup
7106     my $virtdev_hash = {}; # info about allocated drives
7107
7108     my $rpcenv = PVE::RPCEnvironment::get();
7109
7110     my $conffile = PVE::QemuConfig->config_file($vmid);
7111
7112     # Note: $oldconf is undef if VM does not exist
7113     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7114     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7115     my $new_conf_raw = '';
7116
7117     my %storage_limits;
7118
7119     my $print_devmap = sub {
7120         my $cfgfn = "$tmpdir/qemu-server.conf";
7121
7122         # we can read the config - that is already extracted
7123         my $fh = IO::File->new($cfgfn, "r") ||
7124             die "unable to read qemu-server.conf - $!\n";
7125
7126         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7127         if (-f $fwcfgfn) {
7128             my $pve_firewall_dir = '/etc/pve/firewall';
7129             mkdir $pve_firewall_dir; # make sure the dir exists
7130             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7131         }
7132
7133         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7134
7135         foreach my $info (values %{$virtdev_hash}) {
7136             my $storeid = $info->{storeid};
7137             next if defined($storage_limits{$storeid});
7138
7139             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7140             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7141             $storage_limits{$storeid} = $limit * 1024;
7142         }
7143
7144         foreach my $devname (keys %$devinfo) {
7145             die "found no device mapping information for device '$devname'\n"
7146                 if !$devinfo->{$devname}->{virtdev};
7147         }
7148
7149         # create empty/temp config
7150         if ($oldconf) {
7151             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7152             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7153         }
7154
7155         # allocate volumes
7156         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7157
7158         # print restore information to $fifofh
7159         foreach my $virtdev (sort keys %$virtdev_hash) {
7160             my $d = $virtdev_hash->{$virtdev};
7161             next if $d->{is_cloudinit}; # no need to restore cloudinit
7162
7163             my $storeid = $d->{storeid};
7164             my $volid = $d->{volid};
7165
7166             my $map_opts = '';
7167             if (my $limit = $storage_limits{$storeid}) {
7168                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7169             }
7170
7171             my $write_zeros = 1;
7172             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7173                 $write_zeros = 0;
7174             }
7175
7176             my $path = PVE::Storage::path($cfg, $volid);
7177
7178             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7179
7180             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7181         }
7182
7183         $fh->seek(0, 0) || die "seek failed - $!\n";
7184
7185         my $cookie = { netcount => 0 };
7186         while (defined(my $line = <$fh>)) {
7187             $new_conf_raw .= restore_update_config_line(
7188                 $cookie,
7189                 $map,
7190                 $line,
7191                 $opts->{unique},
7192             );
7193         }
7194
7195         $fh->close();
7196     };
7197
7198     eval {
7199         # enable interrupts
7200         local $SIG{INT} =
7201             local $SIG{TERM} =
7202             local $SIG{QUIT} =
7203             local $SIG{HUP} =
7204             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7205         local $SIG{ALRM} = sub { die "got timeout\n"; };
7206
7207         $oldtimeout = alarm($timeout);
7208
7209         my $parser = sub {
7210             my $line = shift;
7211
7212             print "$line\n";
7213
7214             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7215                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7216                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7217             } elsif ($line =~ m/^CTIME: /) {
7218                 # we correctly received the vma config, so we can disable
7219                 # the timeout now for disk allocation (set to 10 minutes, so
7220                 # that we always timeout if something goes wrong)
7221                 alarm(600);
7222                 &$print_devmap();
7223                 print $fifofh "done\n";
7224                 my $tmp = $oldtimeout || 0;
7225                 $oldtimeout = undef;
7226                 alarm($tmp);
7227                 close($fifofh);
7228                 $fifofh = undef;
7229             }
7230         };
7231
7232         print "restore vma archive: $dbg_cmdstring\n";
7233         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7234     };
7235     my $err = $@;
7236
7237     alarm($oldtimeout) if $oldtimeout;
7238
7239     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7240
7241     close($fifofh) if $fifofh;
7242     unlink $mapfifo;
7243     rmtree $tmpdir;
7244
7245     if ($err) {
7246         $restore_destroy_volumes->($cfg, $virtdev_hash);
7247         die $err;
7248     }
7249
7250     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7251     PVE::QemuConfig->write_config($vmid, $new_conf);
7252
7253     eval { rescan($vmid, 1); };
7254     warn $@ if $@;
7255
7256     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7257 }
7258
7259 sub restore_tar_archive {
7260     my ($archive, $vmid, $user, $opts) = @_;
7261
7262     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7263         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7264         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7265     }
7266
7267     if ($archive ne '-') {
7268         my $firstfile = tar_archive_read_firstfile($archive);
7269         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7270             if $firstfile ne 'qemu-server.conf';
7271     }
7272
7273     my $storecfg = PVE::Storage::config();
7274
7275     # avoid zombie disks when restoring over an existing VM -> cleanup first
7276     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7277     # skiplock=1 because qmrestore has set the 'create' lock itself already
7278     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7279     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7280
7281     my $tocmd = "/usr/lib/qemu-server/qmextract";
7282
7283     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7284     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7285     $tocmd .= ' --prealloc' if $opts->{prealloc};
7286     $tocmd .= ' --info' if $opts->{info};
7287
7288     # tar option "xf" does not autodetect compression when read from STDIN,
7289     # so we pipe to zcat
7290     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7291         PVE::Tools::shellquote("--to-command=$tocmd");
7292
7293     my $tmpdir = "/var/tmp/vzdumptmp$$";
7294     mkpath $tmpdir;
7295
7296     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7297     local $ENV{VZDUMP_VMID} = $vmid;
7298     local $ENV{VZDUMP_USER} = $user;
7299
7300     my $conffile = PVE::QemuConfig->config_file($vmid);
7301     my $new_conf_raw = '';
7302
7303     # disable interrupts (always do cleanups)
7304     local $SIG{INT} =
7305         local $SIG{TERM} =
7306         local $SIG{QUIT} =
7307         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7308
7309     eval {
7310         # enable interrupts
7311         local $SIG{INT} =
7312             local $SIG{TERM} =
7313             local $SIG{QUIT} =
7314             local $SIG{HUP} =
7315             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7316
7317         if ($archive eq '-') {
7318             print "extracting archive from STDIN\n";
7319             run_command($cmd, input => "<&STDIN");
7320         } else {
7321             print "extracting archive '$archive'\n";
7322             run_command($cmd);
7323         }
7324
7325         return if $opts->{info};
7326
7327         # read new mapping
7328         my $map = {};
7329         my $statfile = "$tmpdir/qmrestore.stat";
7330         if (my $fd = IO::File->new($statfile, "r")) {
7331             while (defined (my $line = <$fd>)) {
7332                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7333                     $map->{$1} = $2 if $1;
7334                 } else {
7335                     print STDERR "unable to parse line in statfile - $line\n";
7336                 }
7337             }
7338             $fd->close();
7339         }
7340
7341         my $confsrc = "$tmpdir/qemu-server.conf";
7342
7343         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7344
7345         my $cookie = { netcount => 0 };
7346         while (defined (my $line = <$srcfd>)) {
7347             $new_conf_raw .= restore_update_config_line(
7348                 $cookie,
7349                 $map,
7350                 $line,
7351                 $opts->{unique},
7352             );
7353         }
7354
7355         $srcfd->close();
7356     };
7357     if (my $err = $@) {
7358         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7359         die $err;
7360     }
7361
7362     rmtree $tmpdir;
7363
7364     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7365
7366     PVE::Cluster::cfs_update(); # make sure we read new file
7367
7368     eval { rescan($vmid, 1); };
7369     warn $@ if $@;
7370 };
7371
7372 sub foreach_storage_used_by_vm {
7373     my ($conf, $func) = @_;
7374
7375     my $sidhash = {};
7376
7377     PVE::QemuConfig->foreach_volume($conf, sub {
7378         my ($ds, $drive) = @_;
7379         return if drive_is_cdrom($drive);
7380
7381         my $volid = $drive->{file};
7382
7383         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7384         $sidhash->{$sid} = $sid if $sid;
7385     });
7386
7387     foreach my $sid (sort keys %$sidhash) {
7388         &$func($sid);
7389     }
7390 }
7391
7392 my $qemu_snap_storage = {
7393     rbd => 1,
7394 };
7395 sub do_snapshots_with_qemu {
7396     my ($storecfg, $volid, $deviceid) = @_;
7397
7398     return if $deviceid =~ m/tpmstate0/;
7399
7400     my $storage_name = PVE::Storage::parse_volume_id($volid);
7401     my $scfg = $storecfg->{ids}->{$storage_name};
7402     die "could not find storage '$storage_name'\n" if !defined($scfg);
7403
7404     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7405         return 1;
7406     }
7407
7408     if ($volid =~ m/\.(qcow2|qed)$/){
7409         return 1;
7410     }
7411
7412     return;
7413 }
7414
7415 sub qga_check_running {
7416     my ($vmid, $nowarn) = @_;
7417
7418     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7419     if ($@) {
7420         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7421         return 0;
7422     }
7423     return 1;
7424 }
7425
7426 sub template_create {
7427     my ($vmid, $conf, $disk) = @_;
7428
7429     my $storecfg = PVE::Storage::config();
7430
7431     PVE::QemuConfig->foreach_volume($conf, sub {
7432         my ($ds, $drive) = @_;
7433
7434         return if drive_is_cdrom($drive);
7435         return if $disk && $ds ne $disk;
7436
7437         my $volid = $drive->{file};
7438         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7439
7440         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7441         $drive->{file} = $voliddst;
7442         $conf->{$ds} = print_drive($drive);
7443         PVE::QemuConfig->write_config($vmid, $conf);
7444     });
7445 }
7446
7447 sub convert_iscsi_path {
7448     my ($path) = @_;
7449
7450     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7451         my $portal = $1;
7452         my $target = $2;
7453         my $lun = $3;
7454
7455         my $initiator_name = get_initiator_name();
7456
7457         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7458                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7459     }
7460
7461     die "cannot convert iscsi path '$path', unkown format\n";
7462 }
7463
7464 sub qemu_img_convert {
7465     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7466
7467     my $storecfg = PVE::Storage::config();
7468     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7469     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7470
7471     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7472
7473     my $cachemode;
7474     my $src_path;
7475     my $src_is_iscsi = 0;
7476     my $src_format;
7477
7478     if ($src_storeid) {
7479         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7480         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7481         $src_format = qemu_img_format($src_scfg, $src_volname);
7482         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7483         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7484         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7485     } elsif (-f $src_volid || -b $src_volid) {
7486         $src_path = $src_volid;
7487         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7488             $src_format = $1;
7489         }
7490     }
7491
7492     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7493
7494     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7495     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7496     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7497     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7498
7499     my $cmd = [];
7500     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7501     push @$cmd, '-l', "snapshot.name=$snapname"
7502         if $snapname && $src_format && $src_format eq "qcow2";
7503     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7504     push @$cmd, '-T', $cachemode if defined($cachemode);
7505
7506     if ($src_is_iscsi) {
7507         push @$cmd, '--image-opts';
7508         $src_path = convert_iscsi_path($src_path);
7509     } elsif ($src_format) {
7510         push @$cmd, '-f', $src_format;
7511     }
7512
7513     if ($dst_is_iscsi) {
7514         push @$cmd, '--target-image-opts';
7515         $dst_path = convert_iscsi_path($dst_path);
7516     } else {
7517         push @$cmd, '-O', $dst_format;
7518     }
7519
7520     push @$cmd, $src_path;
7521
7522     if (!$dst_is_iscsi && $is_zero_initialized) {
7523         push @$cmd, "zeroinit:$dst_path";
7524     } else {
7525         push @$cmd, $dst_path;
7526     }
7527
7528     my $parser = sub {
7529         my $line = shift;
7530         if($line =~ m/\((\S+)\/100\%\)/){
7531             my $percent = $1;
7532             my $transferred = int($size * $percent / 100);
7533             my $total_h = render_bytes($size, 1);
7534             my $transferred_h = render_bytes($transferred, 1);
7535
7536             print "transferred $transferred_h of $total_h ($percent%)\n";
7537         }
7538
7539     };
7540
7541     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7542     my $err = $@;
7543     die "copy failed: $err" if $err;
7544 }
7545
7546 sub qemu_img_format {
7547     my ($scfg, $volname) = @_;
7548
7549     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7550         return $1;
7551     } else {
7552         return "raw";
7553     }
7554 }
7555
7556 sub qemu_drive_mirror {
7557     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7558
7559     $jobs = {} if !$jobs;
7560
7561     my $qemu_target;
7562     my $format;
7563     $jobs->{"drive-$drive"} = {};
7564
7565     if ($dst_volid =~ /^nbd:/) {
7566         $qemu_target = $dst_volid;
7567         $format = "nbd";
7568     } else {
7569         my $storecfg = PVE::Storage::config();
7570         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7571
7572         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7573
7574         $format = qemu_img_format($dst_scfg, $dst_volname);
7575
7576         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7577
7578         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7579     }
7580
7581     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7582     $opts->{format} = $format if $format;
7583
7584     if (defined($src_bitmap)) {
7585         $opts->{sync} = 'incremental';
7586         $opts->{bitmap} = $src_bitmap;
7587         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7588     }
7589
7590     if (defined($bwlimit)) {
7591         $opts->{speed} = $bwlimit * 1024;
7592         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7593     } else {
7594         print "drive mirror is starting for drive-$drive\n";
7595     }
7596
7597     # if a job already runs for this device we get an error, catch it for cleanup
7598     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7599     if (my $err = $@) {
7600         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7601         warn "$@\n" if $@;
7602         die "mirroring error: $err\n";
7603     }
7604
7605     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7606 }
7607
7608 # $completion can be either
7609 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7610 # 'cancel': wait until all jobs are ready, block-job-cancel them
7611 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7612 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7613 sub qemu_drive_mirror_monitor {
7614     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7615
7616     $completion //= 'complete';
7617     $op //= "mirror";
7618
7619     eval {
7620         my $err_complete = 0;
7621
7622         my $starttime = time ();
7623         while (1) {
7624             die "block job ('$op') timed out\n" if $err_complete > 300;
7625
7626             my $stats = mon_cmd($vmid, "query-block-jobs");
7627             my $ctime = time();
7628
7629             my $running_jobs = {};
7630             for my $stat (@$stats) {
7631                 next if $stat->{type} ne $op;
7632                 $running_jobs->{$stat->{device}} = $stat;
7633             }
7634
7635             my $readycounter = 0;
7636
7637             for my $job_id (sort keys %$jobs) {
7638                 my $job = $running_jobs->{$job_id};
7639
7640                 my $vanished = !defined($job);
7641                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7642                 if($complete || ($vanished && $completion eq 'auto')) {
7643                     print "$job_id: $op-job finished\n";
7644                     delete $jobs->{$job_id};
7645                     next;
7646                 }
7647
7648                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7649
7650                 my $busy = $job->{busy};
7651                 my $ready = $job->{ready};
7652                 if (my $total = $job->{len}) {
7653                     my $transferred = $job->{offset} || 0;
7654                     my $remaining = $total - $transferred;
7655                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7656
7657                     my $duration = $ctime - $starttime;
7658                     my $total_h = render_bytes($total, 1);
7659                     my $transferred_h = render_bytes($transferred, 1);
7660
7661                     my $status = sprintf(
7662                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7663                         render_duration($duration),
7664                     );
7665
7666                     if ($ready) {
7667                         if ($busy) {
7668                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7669                         } else {
7670                             $status .= ", ready";
7671                         }
7672                     }
7673                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7674                     $jobs->{$job_id}->{ready} = $ready;
7675                 }
7676
7677                 $readycounter++ if $job->{ready};
7678             }
7679
7680             last if scalar(keys %$jobs) == 0;
7681
7682             if ($readycounter == scalar(keys %$jobs)) {
7683                 print "all '$op' jobs are ready\n";
7684
7685                 # do the complete later (or has already been done)
7686                 last if $completion eq 'skip' || $completion eq 'auto';
7687
7688                 if ($vmiddst && $vmiddst != $vmid) {
7689                     my $agent_running = $qga && qga_check_running($vmid);
7690                     if ($agent_running) {
7691                         print "freeze filesystem\n";
7692                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7693                         warn $@ if $@;
7694                     } else {
7695                         print "suspend vm\n";
7696                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7697                         warn $@ if $@;
7698                     }
7699
7700                     # if we clone a disk for a new target vm, we don't switch the disk
7701                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7702
7703                     if ($agent_running) {
7704                         print "unfreeze filesystem\n";
7705                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7706                         warn $@ if $@;
7707                     } else {
7708                         print "resume vm\n";
7709                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7710                         warn $@ if $@;
7711                     }
7712
7713                     last;
7714                 } else {
7715
7716                     for my $job_id (sort keys %$jobs) {
7717                         # try to switch the disk if source and destination are on the same guest
7718                         print "$job_id: Completing block job_id...\n";
7719
7720                         my $op;
7721                         if ($completion eq 'complete') {
7722                             $op = 'block-job-complete';
7723                         } elsif ($completion eq 'cancel') {
7724                             $op = 'block-job-cancel';
7725                         } else {
7726                             die "invalid completion value: $completion\n";
7727                         }
7728                         eval { mon_cmd($vmid, $op, device => $job_id) };
7729                         if ($@ =~ m/cannot be completed/) {
7730                             print "$job_id: block job cannot be completed, trying again.\n";
7731                             $err_complete++;
7732                         }else {
7733                             print "$job_id: Completed successfully.\n";
7734                             $jobs->{$job_id}->{complete} = 1;
7735                         }
7736                     }
7737                 }
7738             }
7739             sleep 1;
7740         }
7741     };
7742     my $err = $@;
7743
7744     if ($err) {
7745         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7746         die "block job ($op) error: $err";
7747     }
7748 }
7749
7750 sub qemu_blockjobs_cancel {
7751     my ($vmid, $jobs) = @_;
7752
7753     foreach my $job (keys %$jobs) {
7754         print "$job: Cancelling block job\n";
7755         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7756         $jobs->{$job}->{cancel} = 1;
7757     }
7758
7759     while (1) {
7760         my $stats = mon_cmd($vmid, "query-block-jobs");
7761
7762         my $running_jobs = {};
7763         foreach my $stat (@$stats) {
7764             $running_jobs->{$stat->{device}} = $stat;
7765         }
7766
7767         foreach my $job (keys %$jobs) {
7768
7769             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7770                 print "$job: Done.\n";
7771                 delete $jobs->{$job};
7772             }
7773         }
7774
7775         last if scalar(keys %$jobs) == 0;
7776
7777         sleep 1;
7778     }
7779 }
7780
7781 sub clone_disk {
7782     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7783
7784     my ($vmid, $running) = $source->@{qw(vmid running)};
7785     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7786
7787     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7788     my ($storage, $format) = $dest->@{qw(storage format)};
7789
7790     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7791
7792     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7793         die "cloning from/to EFI disk requires EFI disk\n"
7794             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7795         die "cloning from/to TPM state requires TPM state\n"
7796             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7797
7798         # This would lead to two device nodes in QEMU pointing to the same backing image!
7799         die "cannot change drive name when cloning disk from/to the same VM\n"
7800             if $use_drive_mirror && $vmid == $newvmid;
7801     }
7802
7803     die "cannot move TPM state while VM is running\n"
7804         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7805
7806     my $newvolid;
7807
7808     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7809     print "$src_drivename " if $src_drivename;
7810     print "($drive->{file})\n";
7811
7812     if (!$full) {
7813         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7814         push @$newvollist, $newvolid;
7815     } else {
7816
7817         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7818         $storeid = $storage if $storage;
7819
7820         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7821
7822         my $name = undef;
7823         my $size = undef;
7824         if (drive_is_cloudinit($drive)) {
7825             $name = "vm-$newvmid-cloudinit";
7826             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7827             if ($scfg->{path}) {
7828                 $name .= ".$dst_format";
7829             }
7830             $snapname = undef;
7831             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7832         } elsif ($dst_drivename eq 'efidisk0') {
7833             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7834         } elsif ($dst_drivename eq 'tpmstate0') {
7835             $dst_format = 'raw';
7836             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7837         } else {
7838             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7839         }
7840         $newvolid = PVE::Storage::vdisk_alloc(
7841             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7842         );
7843         push @$newvollist, $newvolid;
7844
7845         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7846
7847         if (drive_is_cloudinit($drive)) {
7848             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7849             # if this is the case, we have to complete any block-jobs still there from
7850             # previous drive-mirrors
7851             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7852                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7853             }
7854             goto no_data_clone;
7855         }
7856
7857         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7858         if ($use_drive_mirror) {
7859             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7860                 $completion, $qga, $bwlimit);
7861         } else {
7862             # TODO: handle bwlimits
7863             if ($dst_drivename eq 'efidisk0') {
7864                 # the relevant data on the efidisk may be smaller than the source
7865                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7866                 # that is given by the OVMF_VARS.fd
7867                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7868                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7869
7870                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7871
7872                 # better for Ceph if block size is not too small, see bug #3324
7873                 my $bs = 1024*1024;
7874
7875                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7876
7877                 if ($src_format eq 'qcow2' && $snapname) {
7878                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7879                         if !min_version(kvm_user_version(), 6, 2);
7880                     push $cmd->@*, '-l', $snapname;
7881                 }
7882                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7883                 run_command($cmd);
7884             } else {
7885                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7886             }
7887         }
7888     }
7889
7890 no_data_clone:
7891     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7892
7893     my $disk = dclone($drive);
7894     delete $disk->{format};
7895     $disk->{file} = $newvolid;
7896     $disk->{size} = $size if defined($size);
7897
7898     return $disk;
7899 }
7900
7901 sub get_running_qemu_version {
7902     my ($vmid) = @_;
7903     my $res = mon_cmd($vmid, "query-version");
7904     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7905 }
7906
7907 sub qemu_use_old_bios_files {
7908     my ($machine_type) = @_;
7909
7910     return if !$machine_type;
7911
7912     my $use_old_bios_files = undef;
7913
7914     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7915         $machine_type = $1;
7916         $use_old_bios_files = 1;
7917     } else {
7918         my $version = extract_version($machine_type, kvm_user_version());
7919         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7920         # load new efi bios files on migration. So this hack is required to allow
7921         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7922         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7923         $use_old_bios_files = !min_version($version, 2, 4);
7924     }
7925
7926     return ($use_old_bios_files, $machine_type);
7927 }
7928
7929 sub get_efivars_size {
7930     my ($conf, $efidisk) = @_;
7931
7932     my $arch = get_vm_arch($conf);
7933     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7934     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7935     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7936     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7937     return -s $ovmf_vars;
7938 }
7939
7940 sub update_efidisk_size {
7941     my ($conf) = @_;
7942
7943     return if !defined($conf->{efidisk0});
7944
7945     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7946     $disk->{size} = get_efivars_size($conf);
7947     $conf->{efidisk0} = print_drive($disk);
7948
7949     return;
7950 }
7951
7952 sub update_tpmstate_size {
7953     my ($conf) = @_;
7954
7955     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7956     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7957     $conf->{tpmstate0} = print_drive($disk);
7958 }
7959
7960 sub create_efidisk($$$$$$$) {
7961     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7962
7963     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7964     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7965
7966     my $vars_size_b = -s $ovmf_vars;
7967     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7968     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7969     PVE::Storage::activate_volumes($storecfg, [$volid]);
7970
7971     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7972     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7973
7974     return ($volid, $size/1024);
7975 }
7976
7977 sub vm_iothreads_list {
7978     my ($vmid) = @_;
7979
7980     my $res = mon_cmd($vmid, 'query-iothreads');
7981
7982     my $iothreads = {};
7983     foreach my $iothread (@$res) {
7984         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7985     }
7986
7987     return $iothreads;
7988 }
7989
7990 sub scsihw_infos {
7991     my ($conf, $drive) = @_;
7992
7993     my $maxdev = 0;
7994
7995     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7996         $maxdev = 7;
7997     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7998         $maxdev = 1;
7999     } else {
8000         $maxdev = 256;
8001     }
8002
8003     my $controller = int($drive->{index} / $maxdev);
8004     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8005         ? "virtioscsi"
8006         : "scsihw";
8007
8008     return ($maxdev, $controller, $controller_prefix);
8009 }
8010
8011 sub resolve_dst_disk_format {
8012         my ($storecfg, $storeid, $src_volname, $format) = @_;
8013         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8014
8015         if (!$format) {
8016             # if no target format is specified, use the source disk format as hint
8017             if ($src_volname) {
8018                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8019                 $format = qemu_img_format($scfg, $src_volname);
8020             } else {
8021                 return $defFormat;
8022             }
8023         }
8024
8025         # test if requested format is supported - else use default
8026         my $supported = grep { $_ eq $format } @$validFormats;
8027         $format = $defFormat if !$supported;
8028         return $format;
8029 }
8030
8031 # NOTE: if this logic changes, please update docs & possibly gui logic
8032 sub find_vmstate_storage {
8033     my ($conf, $storecfg) = @_;
8034
8035     # first, return storage from conf if set
8036     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8037
8038     my ($target, $shared, $local);
8039
8040     foreach_storage_used_by_vm($conf, sub {
8041         my ($sid) = @_;
8042         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8043         my $dst = $scfg->{shared} ? \$shared : \$local;
8044         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8045     });
8046
8047     # second, use shared storage where VM has at least one disk
8048     # third, use local storage where VM has at least one disk
8049     # fall back to local storage
8050     $target = $shared // $local // 'local';
8051
8052     return $target;
8053 }
8054
8055 sub generate_uuid {
8056     my ($uuid, $uuid_str);
8057     UUID::generate($uuid);
8058     UUID::unparse($uuid, $uuid_str);
8059     return $uuid_str;
8060 }
8061
8062 sub generate_smbios1_uuid {
8063     return "uuid=".generate_uuid();
8064 }
8065
8066 sub nbd_stop {
8067     my ($vmid) = @_;
8068
8069     mon_cmd($vmid, 'nbd-server-stop');
8070 }
8071
8072 sub create_reboot_request {
8073     my ($vmid) = @_;
8074     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8075         or die "failed to create reboot trigger file: $!\n";
8076     close($fh);
8077 }
8078
8079 sub clear_reboot_request {
8080     my ($vmid) = @_;
8081     my $path = "/run/qemu-server/$vmid.reboot";
8082     my $res = 0;
8083
8084     $res = unlink($path);
8085     die "could not remove reboot request for $vmid: $!"
8086         if !$res && $! != POSIX::ENOENT;
8087
8088     return $res;
8089 }
8090
8091 sub bootorder_from_legacy {
8092     my ($conf, $bootcfg) = @_;
8093
8094     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8095     my $bootindex_hash = {};
8096     my $i = 1;
8097     foreach my $o (split(//, $boot)) {
8098         $bootindex_hash->{$o} = $i*100;
8099         $i++;
8100     }
8101
8102     my $bootorder = {};
8103
8104     PVE::QemuConfig->foreach_volume($conf, sub {
8105         my ($ds, $drive) = @_;
8106
8107         if (drive_is_cdrom ($drive, 1)) {
8108             if ($bootindex_hash->{d}) {
8109                 $bootorder->{$ds} = $bootindex_hash->{d};
8110                 $bootindex_hash->{d} += 1;
8111             }
8112         } elsif ($bootindex_hash->{c}) {
8113             $bootorder->{$ds} = $bootindex_hash->{c}
8114                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8115             $bootindex_hash->{c} += 1;
8116         }
8117     });
8118
8119     if ($bootindex_hash->{n}) {
8120         for (my $i = 0; $i < $MAX_NETS; $i++) {
8121             my $netname = "net$i";
8122             next if !$conf->{$netname};
8123             $bootorder->{$netname} = $bootindex_hash->{n};
8124             $bootindex_hash->{n} += 1;
8125         }
8126     }
8127
8128     return $bootorder;
8129 }
8130
8131 # Generate default device list for 'boot: order=' property. Matches legacy
8132 # default boot order, but with explicit device names. This is important, since
8133 # the fallback for when neither 'order' nor the old format is specified relies
8134 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8135 sub get_default_bootdevices {
8136     my ($conf) = @_;
8137
8138     my @ret = ();
8139
8140     # harddisk
8141     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8142     push @ret, $first if $first;
8143
8144     # cdrom
8145     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8146     push @ret, $first if $first;
8147
8148     # network
8149     for (my $i = 0; $i < $MAX_NETS; $i++) {
8150         my $netname = "net$i";
8151         next if !$conf->{$netname};
8152         push @ret, $netname;
8153         last;
8154     }
8155
8156     return \@ret;
8157 }
8158
8159 sub device_bootorder {
8160     my ($conf) = @_;
8161
8162     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8163
8164     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8165
8166     my $bootorder = {};
8167     if (!defined($boot) || $boot->{legacy}) {
8168         $bootorder = bootorder_from_legacy($conf, $boot);
8169     } elsif ($boot->{order}) {
8170         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8171         for my $dev (PVE::Tools::split_list($boot->{order})) {
8172             $bootorder->{$dev} = $i++;
8173         }
8174     }
8175
8176     return $bootorder;
8177 }
8178
8179 sub register_qmeventd_handle {
8180     my ($vmid) = @_;
8181
8182     my $fh;
8183     my $peer = "/var/run/qmeventd.sock";
8184     my $count = 0;
8185
8186     for (;;) {
8187         $count++;
8188         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8189         last if $fh;
8190         if ($! != EINTR && $! != EAGAIN) {
8191             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8192         }
8193         if ($count > 4) {
8194             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8195               . "after $count retries\n";
8196         }
8197         usleep(25000);
8198     }
8199
8200     # send handshake to mark VM as backing up
8201     print $fh to_json({vzdump => {vmid => "$vmid"}});
8202
8203     # return handle to be closed later when inhibit is no longer required
8204     return $fh;
8205 }
8206
8207 # bash completion helper
8208
8209 sub complete_backup_archives {
8210     my ($cmdname, $pname, $cvalue) = @_;
8211
8212     my $cfg = PVE::Storage::config();
8213
8214     my $storeid;
8215
8216     if ($cvalue =~ m/^([^:]+):/) {
8217         $storeid = $1;
8218     }
8219
8220     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8221
8222     my $res = [];
8223     foreach my $id (keys %$data) {
8224         foreach my $item (@{$data->{$id}}) {
8225             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8226             push @$res, $item->{volid} if defined($item->{volid});
8227         }
8228     }
8229
8230     return $res;
8231 }
8232
8233 my $complete_vmid_full = sub {
8234     my ($running) = @_;
8235
8236     my $idlist = vmstatus();
8237
8238     my $res = [];
8239
8240     foreach my $id (keys %$idlist) {
8241         my $d = $idlist->{$id};
8242         if (defined($running)) {
8243             next if $d->{template};
8244             next if $running && $d->{status} ne 'running';
8245             next if !$running && $d->{status} eq 'running';
8246         }
8247         push @$res, $id;
8248
8249     }
8250     return $res;
8251 };
8252
8253 sub complete_vmid {
8254     return &$complete_vmid_full();
8255 }
8256
8257 sub complete_vmid_stopped {
8258     return &$complete_vmid_full(0);
8259 }
8260
8261 sub complete_vmid_running {
8262     return &$complete_vmid_full(1);
8263 }
8264
8265 sub complete_storage {
8266
8267     my $cfg = PVE::Storage::config();
8268     my $ids = $cfg->{ids};
8269
8270     my $res = [];
8271     foreach my $sid (keys %$ids) {
8272         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8273         next if !$ids->{$sid}->{content}->{images};
8274         push @$res, $sid;
8275     }
8276
8277     return $res;
8278 }
8279
8280 sub complete_migration_storage {
8281     my ($cmd, $param, $current_value, $all_args) = @_;
8282
8283     my $targetnode = @$all_args[1];
8284
8285     my $cfg = PVE::Storage::config();
8286     my $ids = $cfg->{ids};
8287
8288     my $res = [];
8289     foreach my $sid (keys %$ids) {
8290         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8291         next if !$ids->{$sid}->{content}->{images};
8292         push @$res, $sid;
8293     }
8294
8295     return $res;
8296 }
8297
8298 sub vm_is_paused {
8299     my ($vmid) = @_;
8300     my $qmpstatus = eval {
8301         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8302         mon_cmd($vmid, "query-status");
8303     };
8304     warn "$@\n" if $@;
8305     return $qmpstatus && $qmpstatus->{status} eq "paused";
8306 }
8307
8308 sub check_volume_storage_type {
8309     my ($storecfg, $vol) = @_;
8310
8311     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8312     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8313     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8314
8315     die "storage '$storeid' does not support content-type '$vtype'\n"
8316         if !$scfg->{content}->{$vtype};
8317
8318     return 1;
8319 }
8320
8321 sub add_nets_bridge_fdb {
8322     my ($conf, $vmid) = @_;
8323
8324     for my $opt (keys %$conf) {
8325         next if $opt !~ m/^net(\d+)$/;
8326         my $iface = "tap${vmid}i$1";
8327         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8328         my $net = parse_net($conf->{$opt}, 1) or next;
8329
8330         my $mac = $net->{macaddr};
8331         if (!$mac) {
8332             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8333                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8334             next;
8335         }
8336
8337         if ($have_sdn) {
8338             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
8339         } else {
8340             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8341         }
8342     }
8343 }
8344
8345 sub del_nets_bridge_fdb {
8346     my ($conf, $vmid) = @_;
8347
8348     for my $opt (keys %$conf) {
8349         next if $opt !~ m/^net(\d+)$/;
8350         my $iface = "tap${vmid}i$1";
8351
8352         my $net = parse_net($conf->{$opt}) or next;
8353         my $mac = $net->{macaddr} or next;
8354
8355         if ($have_sdn) {
8356             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
8357         } else {
8358             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8359         }
8360     }
8361 }
8362
8363 1;