PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file(
 106     '/qemu-server/',
 107     \&parse_vm_config,
 108     \&write_vm_config
 109 );
 110
 111 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 112     description => "Some command save/restore state from this location.",
 113     type => 'string',
 114     maxLength => 128,
 115     optional => 1,
 116 });
 117
 118 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 119         description => "Specifies the Qemu machine type.",
 120         type => 'string',
 121         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 122         maxLength => 40,
 123         optional => 1,
 124 });
 125
 126 # FIXME: remove in favor of just using the INotify one, it's cached there exactly the same way
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 243             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 244             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 245             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 246             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 247             ." a hardware RNG from the host.",
 248     },
 249     max_bytes => {
 250         type => 'integer',
 251         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 252             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 253             ." `0` to disable limiting (potentially dangerous!).",
 254         optional => 1,
 255
 256         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 257         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 258         # reading from /dev/urandom
 259         default => 1024,
 260     },
 261     period => {
 262         type => 'integer',
 263         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 264             ." the guest to retrieve another 'max_bytes' of entropy.",
 265         optional => 1,
 266         default => 1000,
 267     },
 268 };
 269
 270 my $meta_info_fmt = {
 271     'ctime' => {
 272         type => 'integer',
 273         description => "The guest creation timestamp as UNIX epoch time",
 274         minimum => 0,
 275         optional => 1,
 276     },
 277     'creation-qemu' => {
 278         type => 'string',
 279         description => "The QEMU (machine) version from the time this VM was created.",
 280         pattern => '\d+(\.\d+)+',
 281         optional => 1,
 282     },
 283 };
 284
 285 my $confdesc = {
 286     onboot => {
 287         optional => 1,
 288         type => 'boolean',
 289         description => "Specifies whether a VM will be started during system bootup.",
 290         default => 0,
 291     },
 292     autostart => {
 293         optional => 1,
 294         type => 'boolean',
 295         description => "Automatic restart after crash (currently ignored).",
 296         default => 0,
 297     },
 298     hotplug => {
 299         optional => 1,
 300         type => 'string', format => 'pve-hotplug-features',
 301         description => "Selectively enable hotplug features. This is a comma separated list of"
 302             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 303             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 304             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 305             ." windows > 7.",
 306         default => 'network,disk,usb',
 307     },
 308     reboot => {
 309         optional => 1,
 310         type => 'boolean',
 311         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 312         default => 1,
 313     },
 314     lock => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Lock/unlock the VM.",
 318         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 319     },
 320     cpulimit => {
 321         optional => 1,
 322         type => 'number',
 323         description => "Limit of CPU usage.",
 324         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 325             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 326         minimum => 0,
 327         maximum => 128,
 328         default => 0,
 329     },
 330     cpuunits => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 334         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 335             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 336             ." weights of all the other running VMs.",
 337         minimum => 1,
 338         maximum => 262144,
 339         default => 'cgroup v1: 1024, cgroup v2: 100',
 340     },
 341     memory => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 345             ." you use the balloon device.",
 346         minimum => 16,
 347         default => 512,
 348     },
 349     balloon => {
 350         optional => 1,
 351         type => 'integer',
 352         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 353         minimum => 0,
 354     },
 355     shares => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 359             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 360             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 361         minimum => 0,
 362         maximum => 50000,
 363         default => 1000,
 364     },
 365     keyboard => {
 366         optional => 1,
 367         type => 'string',
 368         description => "Keyboard layout for VNC server. This option is generally not required and"
 369         ." is often better handled from within the guest OS.",
 370         enum => PVE::Tools::kvmkeymaplist(),
 371         default => undef,
 372     },
 373     name => {
 374         optional => 1,
 375         type => 'string', format => 'dns-name',
 376         description => "Set a name for the VM. Only used on the configuration web interface.",
 377     },
 378     scsihw => {
 379         optional => 1,
 380         type => 'string',
 381         description => "SCSI controller model",
 382         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 383         default => 'lsi',
 384     },
 385     description => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Description for the VM. Shown in the web-interface VM's summary."
 389             ." This is saved as comment inside the configuration file.",
 390         maxLength => 1024 * 8,
 391     },
 392     ostype => {
 393         optional => 1,
 394         type => 'string',
 395         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 396         description => "Specify guest operating system.",
 397         verbose_description => <<EODESC,
 398 Specify guest operating system. This is used to enable special
 399 optimization/features for specific operating systems:
 400
 401 [horizontal]
 402 other;; unspecified OS
 403 wxp;; Microsoft Windows XP
 404 w2k;; Microsoft Windows 2000
 405 w2k3;; Microsoft Windows 2003
 406 w2k8;; Microsoft Windows 2008
 407 wvista;; Microsoft Windows Vista
 408 win7;; Microsoft Windows 7
 409 win8;; Microsoft Windows 8/2012/2012r2
 410 win10;; Microsoft Windows 10/2016/2019
 411 win11;; Microsoft Windows 11/2022
 412 l24;; Linux 2.4 Kernel
 413 l26;; Linux 2.6 - 5.X Kernel
 414 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 415 EODESC
 416     },
 417     boot => {
 418         optional => 1,
 419         type => 'string', format => 'pve-qm-boot',
 420         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 421             ." key or 'legacy=' is deprecated.",
 422     },
 423     bootdisk => {
 424         optional => 1,
 425         type => 'string', format => 'pve-qm-bootdisk',
 426         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 427         pattern => '(ide|sata|scsi|virtio)\d+',
 428     },
 429     smp => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPUs. Please use option -sockets instead.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     sockets => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of CPU sockets.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     cores => {
 444         optional => 1,
 445         type => 'integer',
 446         description => "The number of cores per socket.",
 447         minimum => 1,
 448         default => 1,
 449     },
 450     numa => {
 451         optional => 1,
 452         type => 'boolean',
 453         description => "Enable/disable NUMA.",
 454         default => 0,
 455     },
 456     hugepages => {
 457         optional => 1,
 458         type => 'string',
 459         description => "Enable/disable hugepages memory.",
 460         enum => [qw(any 2 1024)],
 461     },
 462     keephugepages => {
 463         optional => 1,
 464         type => 'boolean',
 465         default => 0,
 466         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 467             ." after VM shutdown and can be used for subsequent starts.",
 468     },
 469     vcpus => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Number of hotplugged vcpus.",
 473         minimum => 1,
 474         default => 0,
 475     },
 476     acpi => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Enable/disable ACPI.",
 480         default => 1,
 481     },
 482     agent => {
 483         optional => 1,
 484         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 485         type => 'string',
 486         format => $agent_fmt,
 487     },
 488     kvm => {
 489         optional => 1,
 490         type => 'boolean',
 491         description => "Enable/disable KVM hardware virtualization.",
 492         default => 1,
 493     },
 494     tdf => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Enable/disable time drift fix.",
 498         default => 0,
 499     },
 500     localtime => {
 501         optional => 1,
 502         type => 'boolean',
 503         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 504             ." the `ostype` indicates a Microsoft Windows OS.",
 505     },
 506     freeze => {
 507         optional => 1,
 508         type => 'boolean',
 509         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 510     },
 511     vga => {
 512         optional => 1,
 513         type => 'string', format => $vga_fmt,
 514         description => "Configure the VGA hardware.",
 515         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 516             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 517             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 518             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 519             ." display server. For win* OS you can select how many independent displays you want,"
 520             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 521             ." using a serial device as terminal.",
 522     },
 523     watchdog => {
 524         optional => 1,
 525         type => 'string', format => 'pve-qm-watchdog',
 526         description => "Create a virtual hardware watchdog device.",
 527         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 528             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 529             ." else the watchdog will reset the guest (or execute the respective action specified)",
 530     },
 531     startdate => {
 532         optional => 1,
 533         type => 'string',
 534         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 535         description => "Set the initial date of the real time clock. Valid format for date are:"
 536             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 537         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 538         default => 'now',
 539     },
 540     startup =>  get_standard_option('pve-startup-order'),
 541     template => {
 542         optional => 1,
 543         type => 'boolean',
 544         description => "Enable/disable Template.",
 545         default => 0,
 546     },
 547     args => {
 548         optional => 1,
 549         type => 'string',
 550         description => "Arbitrary arguments passed to kvm.",
 551         verbose_description => <<EODESCR,
 552 Arbitrary arguments passed to kvm, for example:
 553
 554 args: -no-reboot -no-hpet
 555
 556 NOTE: this option is for experts only.
 557 EODESCR
 558     },
 559     tablet => {
 560         optional => 1,
 561         type => 'boolean',
 562         default => 1,
 563         description => "Enable/disable the USB tablet device.",
 564         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 565             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 566             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 567             ." may consider disabling this to save some context switches. This is turned off by"
 568             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 569     },
 570     migrate_speed => {
 571         optional => 1,
 572         type => 'integer',
 573         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 574         minimum => 0,
 575         default => 0,
 576     },
 577     migrate_downtime => {
 578         optional => 1,
 579         type => 'number',
 580         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 581         minimum => 0,
 582         default => 0.1,
 583     },
 584     cdrom => {
 585         optional => 1,
 586         type => 'string', format => 'pve-qm-ide',
 587         typetext => '<volume>',
 588         description => "This is an alias for option -ide2",
 589     },
 590     cpu => {
 591         optional => 1,
 592         description => "Emulated CPU type.",
 593         type => 'string',
 594         format => 'pve-vm-cpu-conf',
 595     },
 596     parent => get_standard_option('pve-snapshot-name', {
 597         optional => 1,
 598         description => "Parent snapshot name. This is used internally, and should not be modified.",
 599     }),
 600     snaptime => {
 601         optional => 1,
 602         description => "Timestamp for snapshots.",
 603         type => 'integer',
 604         minimum => 0,
 605     },
 606     vmstate => {
 607         optional => 1,
 608         type => 'string', format => 'pve-volume-id',
 609         description => "Reference to a volume which stores the VM state. This is used internally"
 610             ." for snapshots.",
 611     },
 612     vmstatestorage => get_standard_option('pve-storage-id', {
 613         description => "Default storage for VM state volumes/files.",
 614         optional => 1,
 615     }),
 616     runningmachine => get_standard_option('pve-qemu-machine', {
 617         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 618             ." for snapshots.",
 619     }),
 620     runningcpu => {
 621         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 622             ." internally for snapshots.",
 623         optional => 1,
 624         type => 'string',
 625         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 626         format_description => 'QEMU -cpu parameter'
 627     },
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 645             ." remove disk operations.",
 646         default => 0,
 647     },
 648     bios => {
 649         optional => 1,
 650         type => 'string',
 651         enum => [ qw(seabios ovmf) ],
 652         description => "Select BIOS implementation.",
 653         default => 'seabios',
 654     },
 655     vmgenid => {
 656         type => 'string',
 657         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 658         format_description => 'UUID',
 659         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 660             ." to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 662             ." value identifier to the guest OS. This allows to notify the guest operating system"
 663             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 664             ." execution or creation from a template). The guest operating system notices the"
 665             ." change, and is then able to react as appropriate by marking its copies of"
 666             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 667             ."Note that auto-creation only works when done through API/CLI create or update methods"
 668             .", but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 682             ." the host.",
 683         optional => 1,
 684     },
 685     audio0 => {
 686         type => 'string',
 687         format => $audio_fmt,
 688         description => "Configure a audio device, useful in combination with QXL/Spice.",
 689         optional => 1
 690     },
 691     spice_enhancements => {
 692         type => 'string',
 693         format => $spice_enhancements_fmt,
 694         description => "Configure additional enhancements for SPICE.",
 695         optional => 1
 696     },
 697     tags => {
 698         type => 'string', format => 'pve-tag-list',
 699         description => 'Tags of the VM. This is only meta information.',
 700         optional => 1,
 701     },
 702     rng0 => {
 703         type => 'string',
 704         format => $rng_fmt,
 705         description => "Configure a VirtIO-based Random Number Generator.",
 706         optional => 1,
 707     },
 708     meta => {
 709         type => 'string',
 710         format => $meta_info_fmt,
 711         description => "Some (read-only) meta-information about this guest.",
 712         optional => 1,
 713     },
 714     affinity => {
 715         type => 'string', format => 'pve-cpuset',
 716         description => "List of host cores used to execute guest processes, for example: 0,5,8-11",
 717         optional => 1,
 718     },
 719 };
 720
 721 my $cicustom_fmt = {
 722     meta => {
 723         type => 'string',
 724         optional => 1,
 725         description => 'Specify a custom file containing all meta data passed to the VM via"
 726             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 727         format => 'pve-volume-id',
 728         format_description => 'volume',
 729     },
 730     network => {
 731         type => 'string',
 732         optional => 1,
 733         description => 'Specify a custom file containing all network data passed to the VM via'
 734             .' cloud-init.',
 735         format => 'pve-volume-id',
 736         format_description => 'volume',
 737     },
 738     user => {
 739         type => 'string',
 740         optional => 1,
 741         description => 'Specify a custom file containing all user data passed to the VM via'
 742             .' cloud-init.',
 743         format => 'pve-volume-id',
 744         format_description => 'volume',
 745     },
 746     vendor => {
 747     type => 'string',
 748     optional => 1,
 749     description => 'Specify a custom file containing all vendor data passed to the VM via'
 750      .' cloud-init.',
 751     format => 'pve-volume-id',
 752     format_description => 'volume',
 753     },
 754 };
 755 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 756
 757 my $confdesc_cloudinit = {
 758     citype => {
 759         optional => 1,
 760         type => 'string',
 761         description => 'Specifies the cloud-init configuration format. The default depends on the'
 762             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 763             .' and `configdrive2` for windows.',
 764         enum => ['configdrive2', 'nocloud', 'opennebula'],
 765     },
 766     ciuser => {
 767         optional => 1,
 768         type => 'string',
 769         description => "cloud-init: User name to change ssh keys and password for instead of the"
 770             ." image's configured default user.",
 771     },
 772     cipassword => {
 773         optional => 1,
 774         type => 'string',
 775         description => 'cloud-init: Password to assign the user. Using this is generally not'
 776             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 777             .' support hashed passwords.',
 778     },
 779     cicustom => {
 780         optional => 1,
 781         type => 'string',
 782         description => 'cloud-init: Specify custom files to replace the automatically generated'
 783             .' ones at start.',
 784         format => 'pve-qm-cicustom',
 785     },
 786     searchdomain => {
 787         optional => 1,
 788         type => 'string',
 789         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 790             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 791             .' are set.',
 792     },
 793     nameserver => {
 794         optional => 1,
 795         type => 'string', format => 'address-list',
 796         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 797             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 798             .' are set.',
 799     },
 800     sshkeys => {
 801         optional => 1,
 802         type => 'string',
 803         format => 'urlencoded',
 804         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 805     },
 806 };
 807
 808 # what about other qemu settings ?
 809 #cpu => 'string',
 810 #machine => 'string',
 811 #fda => 'file',
 812 #fdb => 'file',
 813 #mtdblock => 'file',
 814 #sd => 'file',
 815 #pflash => 'file',
 816 #snapshot => 'bool',
 817 #bootp => 'file',
 818 ##tftp => 'dir',
 819 ##smb => 'dir',
 820 #kernel => 'file',
 821 #append => 'string',
 822 #initrd => 'file',
 823 ##soundhw => 'string',
 824
 825 while (my ($k, $v) = each %$confdesc) {
 826     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 827 }
 828
 829 my $MAX_USB_DEVICES = 14;
 830 my $MAX_NETS = 32;
 831 my $MAX_SERIAL_PORTS = 4;
 832 my $MAX_PARALLEL_PORTS = 3;
 833 my $MAX_NUMA = 8;
 834
 835 my $numa_fmt = {
 836     cpus => {
 837         type => "string",
 838         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 839         description => "CPUs accessing this NUMA node.",
 840         format_description => "id[-id];...",
 841     },
 842     memory => {
 843         type => "number",
 844         description => "Amount of memory this NUMA node provides.",
 845         optional => 1,
 846     },
 847     hostnodes => {
 848         type => "string",
 849         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 850         description => "Host NUMA nodes to use.",
 851         format_description => "id[-id];...",
 852         optional => 1,
 853     },
 854     policy => {
 855         type => 'string',
 856         enum => [qw(preferred bind interleave)],
 857         description => "NUMA allocation policy.",
 858         optional => 1,
 859     },
 860 };
 861 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 862 my $numadesc = {
 863     optional => 1,
 864     type => 'string', format => $numa_fmt,
 865     description => "NUMA topology.",
 866 };
 867 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 868
 869 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 870     $confdesc->{"numa$i"} = $numadesc;
 871 }
 872
 873 my $nic_model_list = [
 874     'e1000',
 875     'e1000-82540em',
 876     'e1000-82544gc',
 877     'e1000-82545em',
 878     'e1000e',
 879     'i82551',
 880     'i82557b',
 881     'i82559er',
 882     'ne2k_isa',
 883     'ne2k_pci',
 884     'pcnet',
 885     'rtl8139',
 886     'virtio',
 887     'vmxnet3',
 888 ];
 889 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 890
 891 my $net_fmt_bridge_descr = <<__EOD__;
 892 Bridge to attach the network device to. The Proxmox VE standard bridge
 893 is called 'vmbr0'.
 894
 895 If you do not specify a bridge, we create a kvm user (NATed) network
 896 device, which provides DHCP and DNS services. The following addresses
 897 are used:
 898
 899  10.0.2.2   Gateway
 900  10.0.2.3   DNS Server
 901  10.0.2.4   SMB Server
 902
 903 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 904 __EOD__
 905
 906 my $net_fmt = {
 907     macaddr  => get_standard_option('mac-addr', {
 908         description => "MAC address. That address must be unique withing your network. This is"
 909             ." automatically generated if not specified.",
 910     }),
 911     model => {
 912         type => 'string',
 913         description => "Network Card Model. The 'virtio' model provides the best performance with"
 914             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 915             ." best to use 'e1000'.",
 916         enum => $nic_model_list,
 917         default_key => 1,
 918     },
 919     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 920     bridge => get_standard_option('pve-bridge-id', {
 921         description => $net_fmt_bridge_descr,
 922         optional => 1,
 923     }),
 924     queues => {
 925         type => 'integer',
 926         minimum => 0, maximum => 64,
 927         description => 'Number of packet queues to be used on the device.',
 928         optional => 1,
 929     },
 930     rate => {
 931         type => 'number',
 932         minimum => 0,
 933         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 934         optional => 1,
 935     },
 936     tag => {
 937         type => 'integer',
 938         minimum => 1, maximum => 4094,
 939         description => 'VLAN tag to apply to packets on this interface.',
 940         optional => 1,
 941     },
 942     trunks => {
 943         type => 'string',
 944         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 945         description => 'VLAN trunks to pass through this interface.',
 946         format_description => 'vlanid[;vlanid...]',
 947         optional => 1,
 948     },
 949     firewall => {
 950         type => 'boolean',
 951         description => 'Whether this interface should be protected by the firewall.',
 952         optional => 1,
 953     },
 954     link_down => {
 955         type => 'boolean',
 956         description => 'Whether this interface should be disconnected (like pulling the plug).',
 957         optional => 1,
 958     },
 959     mtu => {
 960         type => 'integer',
 961         minimum => 1, maximum => 65520,
 962         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 963         optional => 1,
 964     },
 965 };
 966
 967 my $netdesc = {
 968     optional => 1,
 969     type => 'string', format => $net_fmt,
 970     description => "Specify network devices.",
 971 };
 972
 973 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 974
 975 my $ipconfig_fmt = {
 976     ip => {
 977         type => 'string',
 978         format => 'pve-ipv4-config',
 979         format_description => 'IPv4Format/CIDR',
 980         description => 'IPv4 address in CIDR format.',
 981         optional => 1,
 982         default => 'dhcp',
 983     },
 984     gw => {
 985         type => 'string',
 986         format => 'ipv4',
 987         format_description => 'GatewayIPv4',
 988         description => 'Default gateway for IPv4 traffic.',
 989         optional => 1,
 990         requires => 'ip',
 991     },
 992     ip6 => {
 993         type => 'string',
 994         format => 'pve-ipv6-config',
 995         format_description => 'IPv6Format/CIDR',
 996         description => 'IPv6 address in CIDR format.',
 997         optional => 1,
 998         default => 'dhcp',
 999     },
1000     gw6 => {
1001         type => 'string',
1002         format => 'ipv6',
1003         format_description => 'GatewayIPv6',
1004         description => 'Default gateway for IPv6 traffic.',
1005         optional => 1,
1006         requires => 'ip6',
1007     },
1008 };
1009 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1010 my $ipconfigdesc = {
1011     optional => 1,
1012     type => 'string', format => 'pve-qm-ipconfig',
1013     description => <<'EODESCR',
1014 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1015
1016 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1017
1018 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1019 gateway should be provided.
1020 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1021 cloud-init 19.4 or newer.
1022
1023 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1024 dhcp on IPv4.
1025 EODESCR
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1028
1029 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1030     $confdesc->{"net$i"} = $netdesc;
1031     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1032 }
1033
1034 foreach my $key (keys %$confdesc_cloudinit) {
1035     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1036 }
1037
1038 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1039 sub pve_verify_cpuset {
1040     my ($set_text, $noerr) = @_;
1041
1042     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1043
1044     if ($@) {
1045         return if $noerr;
1046         die "unable to parse cpuset option\n";
1047     }
1048
1049     return PVE::CpuSet->new($members)->short_string();
1050 }
1051
1052 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1053 sub verify_volume_id_or_qm_path {
1054     my ($volid, $noerr) = @_;
1055
1056     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1057
1058     return verify_volume_id_or_absolute_path($volid, $noerr);
1059 }
1060
1061 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1062 sub verify_volume_id_or_absolute_path {
1063     my ($volid, $noerr) = @_;
1064
1065     return $volid if $volid =~ m|^/|;
1066
1067     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1068     if ($@) {
1069         return if $noerr;
1070         die $@;
1071     }
1072     return $volid;
1073 }
1074
1075 my $usb_fmt = {
1076     host => {
1077         default_key => 1,
1078         type => 'string', format => 'pve-qm-usb-device',
1079         format_description => 'HOSTUSBDEVICE|spice',
1080         description => <<EODESCR,
1081 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1082
1083  'bus-port(.port)*' (decimal numbers) or
1084  'vendor_id:product_id' (hexadeciaml numbers) or
1085  'spice'
1086
1087 You can use the 'lsusb -t' command to list existing usb devices.
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1090 machines - use with special care.
1091
1092 The value 'spice' can be used to add a usb redirection devices for spice.
1093 EODESCR
1094     },
1095     usb3 => {
1096         optional => 1,
1097         type => 'boolean',
1098         description => "Specifies whether if given host option is a USB3 device or port."
1099             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1100             ." is irrelevant (all devices are plugged into a xhci controller).",
1101         default => 0,
1102     },
1103 };
1104
1105 my $usbdesc = {
1106     optional => 1,
1107     type => 'string', format => $usb_fmt,
1108     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1109         ." l26 or windows > 7, n can be up to 14).",
1110 };
1111 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1112
1113 my $serialdesc = {
1114         optional => 1,
1115         type => 'string',
1116         pattern => '(/dev/.+|socket)',
1117         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1118         verbose_description =>  <<EODESCR,
1119 Create a serial device inside the VM (n is 0 to 3), and pass through a
1120 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1121 host side (use 'qm terminal' to open a terminal connection).
1122
1123 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1124 use with special care.
1125
1126 CAUTION: Experimental! User reported problems with this option.
1127 EODESCR
1128 };
1129
1130 my $paralleldesc= {
1131         optional => 1,
1132         type => 'string',
1133         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1134         description =>  "Map host parallel devices (n is 0 to 2).",
1135         verbose_description =>  <<EODESCR,
1136 Map host parallel devices (n is 0 to 2).
1137
1138 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1139 machines - use with special care.
1140
1141 CAUTION: Experimental! User reported problems with this option.
1142 EODESCR
1143 };
1144
1145 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1146     $confdesc->{"parallel$i"} = $paralleldesc;
1147 }
1148
1149 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1150     $confdesc->{"serial$i"} = $serialdesc;
1151 }
1152
1153 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1154     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1155 }
1156
1157 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1158     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1159 }
1160
1161 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1162     $confdesc->{"usb$i"} = $usbdesc;
1163 }
1164
1165 my $boot_fmt = {
1166     legacy => {
1167         optional => 1,
1168         default_key => 1,
1169         type => 'string',
1170         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1171                      . " Deprecated, use 'order=' instead.",
1172         pattern => '[acdn]{1,4}',
1173         format_description => "[acdn]{1,4}",
1174
1175         # note: this is also the fallback if boot: is not given at all
1176         default => 'cdn',
1177     },
1178     order => {
1179         optional => 1,
1180         type => 'string',
1181         format => 'pve-qm-bootdev-list',
1182         format_description => "device[;device...]",
1183         description => <<EODESC,
1184 The guest will attempt to boot from devices in the order they appear here.
1185
1186 Disks, optical drives and passed-through storage USB devices will be directly
1187 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1188 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1189
1190 Note that only devices in this list will be marked as bootable and thus loaded
1191 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1192 (e.g. software-raid), you need to specify all of them here.
1193
1194 Overrides the deprecated 'legacy=[acdn]*' value when given.
1195 EODESC
1196     },
1197 };
1198 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1199
1200 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1201 sub verify_bootdev {
1202     my ($dev, $noerr) = @_;
1203
1204     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1205     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1206
1207     my $check = sub {
1208         my ($base) = @_;
1209         return 0 if $dev !~ m/^$base\d+$/;
1210         return 0 if !$confdesc->{$dev};
1211         return 1;
1212     };
1213
1214     return $dev if $check->("net");
1215     return $dev if $check->("usb");
1216     return $dev if $check->("hostpci");
1217
1218     return if $noerr;
1219     die "invalid boot device '$dev'\n";
1220 }
1221
1222 sub print_bootorder {
1223     my ($devs) = @_;
1224     return "" if !@$devs;
1225     my $data = { order => join(';', @$devs) };
1226     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1227 }
1228
1229 my $kvm_api_version = 0;
1230
1231 sub kvm_version {
1232     return $kvm_api_version if $kvm_api_version;
1233
1234     open my $fh, '<', '/dev/kvm' or return;
1235
1236     # 0xae00 => KVM_GET_API_VERSION
1237     $kvm_api_version = ioctl($fh, 0xae00, 0);
1238     close($fh);
1239
1240     return $kvm_api_version;
1241 }
1242
1243 my $kvm_user_version = {};
1244 my $kvm_mtime = {};
1245
1246 sub kvm_user_version {
1247     my ($binary) = @_;
1248
1249     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1250     my $st = stat($binary);
1251
1252     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1253     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1254         $cachedmtime == $st->mtime;
1255
1256     $kvm_user_version->{$binary} = 'unknown';
1257     $kvm_mtime->{$binary} = $st->mtime;
1258
1259     my $code = sub {
1260         my $line = shift;
1261         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1262             $kvm_user_version->{$binary} = $2;
1263         }
1264     };
1265
1266     eval { run_command([$binary, '--version'], outfunc => $code); };
1267     warn $@ if $@;
1268
1269     return $kvm_user_version->{$binary};
1270
1271 }
1272 my sub extract_version {
1273     my ($machine_type, $version) = @_;
1274     $version = kvm_user_version() if !defined($version);
1275     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1276 }
1277
1278 sub kernel_has_vhost_net {
1279     return -c '/dev/vhost-net';
1280 }
1281
1282 sub option_exists {
1283     my $key = shift;
1284     return defined($confdesc->{$key});
1285 }
1286
1287 my $cdrom_path;
1288 sub get_cdrom_path {
1289
1290     return  $cdrom_path if $cdrom_path;
1291
1292     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1293     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1294     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1295 }
1296
1297 sub get_iso_path {
1298     my ($storecfg, $vmid, $cdrom) = @_;
1299
1300     if ($cdrom eq 'cdrom') {
1301         return get_cdrom_path();
1302     } elsif ($cdrom eq 'none') {
1303         return '';
1304     } elsif ($cdrom =~ m|^/|) {
1305         return $cdrom;
1306     } else {
1307         return PVE::Storage::path($storecfg, $cdrom);
1308     }
1309 }
1310
1311 # try to convert old style file names to volume IDs
1312 sub filename_to_volume_id {
1313     my ($vmid, $file, $media) = @_;
1314
1315      if (!($file eq 'none' || $file eq 'cdrom' ||
1316           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1317
1318         return if $file =~ m|/|;
1319
1320         if ($media && $media eq 'cdrom') {
1321             $file = "local:iso/$file";
1322         } else {
1323             $file = "local:$vmid/$file";
1324         }
1325     }
1326
1327     return $file;
1328 }
1329
1330 sub verify_media_type {
1331     my ($opt, $vtype, $media) = @_;
1332
1333     return if !$media;
1334
1335     my $etype;
1336     if ($media eq 'disk') {
1337         $etype = 'images';
1338     } elsif ($media eq 'cdrom') {
1339         $etype = 'iso';
1340     } else {
1341         die "internal error";
1342     }
1343
1344     return if ($vtype eq $etype);
1345
1346     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1347 }
1348
1349 sub cleanup_drive_path {
1350     my ($opt, $storecfg, $drive) = @_;
1351
1352     # try to convert filesystem paths to volume IDs
1353
1354     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1355         ($drive->{file} !~ m|^/dev/.+|) &&
1356         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1357         ($drive->{file} !~ m/^\d+$/)) {
1358         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1359         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1360             if !$vtype;
1361         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1362         verify_media_type($opt, $vtype, $drive->{media});
1363         $drive->{file} = $volid;
1364     }
1365
1366     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1367 }
1368
1369 sub parse_hotplug_features {
1370     my ($data) = @_;
1371
1372     my $res = {};
1373
1374     return $res if $data eq '0';
1375
1376     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1377
1378     foreach my $feature (PVE::Tools::split_list($data)) {
1379         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1380             $res->{$1} = 1;
1381         } else {
1382             die "invalid hotplug feature '$feature'\n";
1383         }
1384     }
1385     return $res;
1386 }
1387
1388 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1389 sub pve_verify_hotplug_features {
1390     my ($value, $noerr) = @_;
1391
1392     return $value if parse_hotplug_features($value);
1393
1394     return if $noerr;
1395
1396     die "unable to parse hotplug option\n";
1397 }
1398
1399 sub scsi_inquiry {
1400     my($fh, $noerr) = @_;
1401
1402     my $SG_IO = 0x2285;
1403     my $SG_GET_VERSION_NUM = 0x2282;
1404
1405     my $versionbuf = "\x00" x 8;
1406     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1407     if (!$ret) {
1408         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1409         return;
1410     }
1411     my $version = unpack("I", $versionbuf);
1412     if ($version < 30000) {
1413         die "scsi generic interface too old\n"  if !$noerr;
1414         return;
1415     }
1416
1417     my $buf = "\x00" x 36;
1418     my $sensebuf = "\x00" x 8;
1419     my $cmd = pack("C x3 C x1", 0x12, 36);
1420
1421     # see /usr/include/scsi/sg.h
1422     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1423
1424     my $packet = pack(
1425         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1426     );
1427
1428     $ret = ioctl($fh, $SG_IO, $packet);
1429     if (!$ret) {
1430         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1431         return;
1432     }
1433
1434     my @res = unpack($sg_io_hdr_t, $packet);
1435     if ($res[17] || $res[18]) {
1436         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1437         return;
1438     }
1439
1440     my $res = {};
1441     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1442
1443     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1444     $res->{type} &= 0x1F;
1445
1446     return $res;
1447 }
1448
1449 sub path_is_scsi {
1450     my ($path) = @_;
1451
1452     my $fh = IO::File->new("+<$path") || return;
1453     my $res = scsi_inquiry($fh, 1);
1454     close($fh);
1455
1456     return $res;
1457 }
1458
1459 sub print_tabletdevice_full {
1460     my ($conf, $arch) = @_;
1461
1462     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1463
1464     # we use uhci for old VMs because tablet driver was buggy in older qemu
1465     my $usbbus;
1466     if ($q35 || $arch eq 'aarch64') {
1467         $usbbus = 'ehci';
1468     } else {
1469         $usbbus = 'uhci';
1470     }
1471
1472     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1473 }
1474
1475 sub print_keyboarddevice_full {
1476     my ($conf, $arch) = @_;
1477
1478     return if $arch ne 'aarch64';
1479
1480     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1481 }
1482
1483 my sub get_drive_id {
1484     my ($drive) = @_;
1485     return "$drive->{interface}$drive->{index}";
1486 }
1487
1488 sub print_drivedevice_full {
1489     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1490
1491     my $device = '';
1492     my $maxdev = 0;
1493
1494     my $drive_id = get_drive_id($drive);
1495     if ($drive->{interface} eq 'virtio') {
1496         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1497         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1498         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1499     } elsif ($drive->{interface} eq 'scsi') {
1500
1501         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1502         my $unit = $drive->{index} % $maxdev;
1503         my $devicetype = 'hd';
1504         my $path = '';
1505         if (drive_is_cdrom($drive)) {
1506             $devicetype = 'cd';
1507         } else {
1508             if ($drive->{file} =~ m|^/|) {
1509                 $path = $drive->{file};
1510                 if (my $info = path_is_scsi($path)) {
1511                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1512                         $devicetype = 'block';
1513                     } elsif ($info->{type} == 1) { # tape
1514                         $devicetype = 'generic';
1515                     }
1516                 }
1517             } else {
1518                  $path = PVE::Storage::path($storecfg, $drive->{file});
1519             }
1520
1521             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1522             my $version = extract_version($machine_type, kvm_user_version());
1523             if ($path =~ m/^iscsi\:\/\// &&
1524                !min_version($version, 4, 1)) {
1525                 $devicetype = 'generic';
1526             }
1527         }
1528
1529         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1530             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1531         } else {
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1533                 .",lun=$drive->{index}";
1534         }
1535         $device .= ",drive=drive-$drive_id,id=$drive_id";
1536
1537         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1538             $device .= ",rotation_rate=1";
1539         }
1540         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1541
1542     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1543         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1544         my $controller = int($drive->{index} / $maxdev);
1545         my $unit = $drive->{index} % $maxdev;
1546         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1547
1548         $device = "ide-$devicetype";
1549         if ($drive->{interface} eq 'ide') {
1550             $device .= ",bus=ide.$controller,unit=$unit";
1551         } else {
1552             $device .= ",bus=ahci$controller.$unit";
1553         }
1554         $device .= ",drive=drive-$drive_id,id=$drive_id";
1555
1556         if ($devicetype eq 'hd') {
1557             if (my $model = $drive->{model}) {
1558                 $model = URI::Escape::uri_unescape($model);
1559                 $device .= ",model=$model";
1560             }
1561             if ($drive->{ssd}) {
1562                 $device .= ",rotation_rate=1";
1563             }
1564         }
1565         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1566     } elsif ($drive->{interface} eq 'usb') {
1567         die "implement me";
1568         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1569     } else {
1570         die "unsupported interface type";
1571     }
1572
1573     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1574
1575     if (my $serial = $drive->{serial}) {
1576         $serial = URI::Escape::uri_unescape($serial);
1577         $device .= ",serial=$serial";
1578     }
1579
1580
1581     return $device;
1582 }
1583
1584 sub get_initiator_name {
1585     my $initiator;
1586
1587     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1588     while (defined(my $line = <$fh>)) {
1589         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1590         $initiator = $1;
1591         last;
1592     }
1593     $fh->close();
1594
1595     return $initiator;
1596 }
1597
1598 sub print_drive_commandline_full {
1599     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1600
1601     my $path;
1602     my $volid = $drive->{file};
1603     my $format = $drive->{format};
1604     my $drive_id = get_drive_id($drive);
1605
1606     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1607     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1608
1609     if (drive_is_cdrom($drive)) {
1610         $path = get_iso_path($storecfg, $vmid, $volid);
1611         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1612     } else {
1613         if ($storeid) {
1614             $path = PVE::Storage::path($storecfg, $volid);
1615             $format //= qemu_img_format($scfg, $volname);
1616         } else {
1617             $path = $volid;
1618             $format //= "raw";
1619         }
1620    }
1621
1622    my $is_rbd = $path =~ m/^rbd:/;
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1628     }
1629
1630     # snapshot only accepts on|off
1631     if (defined($drive->{snapshot})) {
1632         my $v = $drive->{snapshot} ? 'on' : 'off';
1633         $opts .= ",snapshot=$v";
1634     }
1635
1636     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1637         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1638     }
1639
1640     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1641         my ($dir, $qmpname) = @$type;
1642         if (my $v = $drive->{"mbps$dir"}) {
1643             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1644         }
1645         if (my $v = $drive->{"mbps${dir}_max"}) {
1646             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1647         }
1648         if (my $v = $drive->{"bps${dir}_max_length"}) {
1649             $opts .= ",throttling.bps$qmpname-max-length=$v";
1650         }
1651         if (my $v = $drive->{"iops${dir}"}) {
1652             $opts .= ",throttling.iops$qmpname=$v";
1653         }
1654         if (my $v = $drive->{"iops${dir}_max"}) {
1655             $opts .= ",throttling.iops$qmpname-max=$v";
1656         }
1657         if (my $v = $drive->{"iops${dir}_max_length"}) {
1658             $opts .= ",throttling.iops$qmpname-max-length=$v";
1659         }
1660     }
1661
1662     if ($pbs_name) {
1663         $format = "rbd" if $is_rbd;
1664         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1665             if !$format;
1666         $opts .= ",format=alloc-track,file.driver=$format";
1667     } elsif ($format) {
1668         $opts .= ",format=$format";
1669     }
1670
1671     my $cache_direct = 0;
1672
1673     if (my $cache = $drive->{cache}) {
1674         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1675     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1676         $opts .= ",cache=none";
1677         $cache_direct = 1;
1678     }
1679
1680     # io_uring with cache mode writeback or writethrough on krbd will hang...
1681     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1682
1683     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1684     # sometimes, just plain disable...
1685     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1686
1687     # io_uring causes problems when used with CIFS since kernel 5.15
1688     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1689     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1690
1691     if (!$drive->{aio}) {
1692         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1693             # io_uring supports all cache modes
1694             $opts .= ",aio=io_uring";
1695         } else {
1696             # aio native works only with O_DIRECT
1697             if($cache_direct) {
1698                 $opts .= ",aio=native";
1699             } else {
1700                 $opts .= ",aio=threads";
1701             }
1702         }
1703     }
1704
1705     if (!drive_is_cdrom($drive)) {
1706         my $detectzeroes;
1707         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1708             $detectzeroes = 'off';
1709         } elsif ($drive->{discard}) {
1710             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1711         } else {
1712             # This used to be our default with discard not being specified:
1713             $detectzeroes = 'on';
1714         }
1715
1716         # note: 'detect-zeroes' works per blockdev and we want it to persist
1717         # after the alloc-track is removed, so put it on 'file' directly
1718         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1719         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1720     }
1721
1722     if ($pbs_name) {
1723         $opts .= ",backing=$pbs_name";
1724         $opts .= ",auto-remove=on";
1725     }
1726
1727     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1728     my $file_param = "file";
1729     if ($pbs_name) {
1730         # non-rbd drivers require the underlying file to be a seperate block
1731         # node, so add a second .file indirection
1732         $file_param .= ".file" if !$is_rbd;
1733         $file_param .= ".filename";
1734     }
1735     my $pathinfo = $path ? "$file_param=$path," : '';
1736
1737     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1738 }
1739
1740 sub print_pbs_blockdev {
1741     my ($pbs_conf, $pbs_name) = @_;
1742     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1743     $blockdev .= ",repository=$pbs_conf->{repository}";
1744     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1745     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1746     $blockdev .= ",archive=$pbs_conf->{archive}";
1747     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1748     return $blockdev;
1749 }
1750
1751 sub print_netdevice_full {
1752     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1753
1754     my $device = $net->{model};
1755     if ($net->{model} eq 'virtio') {
1756          $device = 'virtio-net-pci';
1757      };
1758
1759     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1760     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1761     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1762         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1763         # and out of each queue plus one config interrupt and control vector queue
1764         my $vectors = $net->{queues} * 2 + 2;
1765         $tmpstr .= ",vectors=$vectors,mq=on";
1766         if (min_version($machine_version, 7, 1)) {
1767             $tmpstr .= ",packed=on";
1768         }
1769     }
1770
1771     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1772         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1773     }
1774
1775     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1776
1777     if (my $mtu = $net->{mtu}) {
1778         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1779             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1780             if ($mtu == 1) {
1781                  $mtu = $bridge_mtu;
1782             } elsif ($mtu < 576) {
1783                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1784             } elsif ($mtu > $bridge_mtu) {
1785                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1786             }
1787             $tmpstr .= ",host_mtu=$mtu";
1788         } else {
1789             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1790         }
1791     }
1792
1793     if ($use_old_bios_files) {
1794         my $romfile;
1795         if ($device eq 'virtio-net-pci') {
1796             $romfile = 'pxe-virtio.rom';
1797         } elsif ($device eq 'e1000') {
1798             $romfile = 'pxe-e1000.rom';
1799         } elsif ($device eq 'e1000e') {
1800             $romfile = 'pxe-e1000e.rom';
1801         } elsif ($device eq 'ne2k') {
1802             $romfile = 'pxe-ne2k_pci.rom';
1803         } elsif ($device eq 'pcnet') {
1804             $romfile = 'pxe-pcnet.rom';
1805         } elsif ($device eq 'rtl8139') {
1806             $romfile = 'pxe-rtl8139.rom';
1807         }
1808         $tmpstr .= ",romfile=$romfile" if $romfile;
1809     }
1810
1811     return $tmpstr;
1812 }
1813
1814 sub print_netdev_full {
1815     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1816
1817     my $i = '';
1818     if ($netid =~ m/^net(\d+)$/) {
1819         $i = int($1);
1820     }
1821
1822     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1823
1824     my $ifname = "tap${vmid}i$i";
1825
1826     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1827     die "interface name '$ifname' is too long (max 15 character)\n"
1828         if length($ifname) >= 16;
1829
1830     my $vhostparam = '';
1831     if (is_native($arch)) {
1832         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1833     }
1834
1835     my $vmname = $conf->{name} || "vm$vmid";
1836
1837     my $netdev = "";
1838     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1839
1840     if ($net->{bridge}) {
1841         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1842             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1843     } else {
1844         $netdev = "type=user,id=$netid,hostname=$vmname";
1845     }
1846
1847     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1848
1849     return $netdev;
1850 }
1851
1852 my $vga_map = {
1853     'cirrus' => 'cirrus-vga',
1854     'std' => 'VGA',
1855     'vmware' => 'vmware-svga',
1856     'virtio' => 'virtio-vga',
1857     'virtio-gl' => 'virtio-vga-gl',
1858 };
1859
1860 sub print_vga_device {
1861     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1862
1863     my $type = $vga_map->{$vga->{type}};
1864     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1865         $type = 'virtio-gpu';
1866     }
1867     my $vgamem_mb = $vga->{memory};
1868
1869     my $max_outputs = '';
1870     if ($qxlnum) {
1871         $type = $id ? 'qxl' : 'qxl-vga';
1872
1873         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1874             # set max outputs so linux can have up to 4 qxl displays with one device
1875             if (min_version($machine_version, 4, 1)) {
1876                 $max_outputs = ",max_outputs=4";
1877             }
1878         }
1879     }
1880
1881     die "no devicetype for $vga->{type}\n" if !$type;
1882
1883     my $memory = "";
1884     if ($vgamem_mb) {
1885         if ($vga->{type} =~ /^virtio/) {
1886             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1887             $memory = ",max_hostmem=$bytes";
1888         } elsif ($qxlnum) {
1889             # from https://www.spice-space.org/multiple-monitors.html
1890             $memory = ",vgamem_mb=$vga->{memory}";
1891             my $ram = $vgamem_mb * 4;
1892             my $vram = $vgamem_mb * 2;
1893             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1894         } else {
1895             $memory = ",vgamem_mb=$vga->{memory}";
1896         }
1897     } elsif ($qxlnum && $id) {
1898         $memory = ",ram_size=67108864,vram_size=33554432";
1899     }
1900
1901     my $edidoff = "";
1902     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1903         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1904     }
1905
1906     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1907     my $vgaid = "vga" . ($id // '');
1908     my $pciaddr;
1909     if ($q35 && $vgaid eq 'vga') {
1910         # the first display uses pcie.0 bus on q35 machines
1911         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1912     } else {
1913         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1914     }
1915
1916     if ($vga->{type} eq 'virtio-gl') {
1917         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1918         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1919             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1920
1921         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1922             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1923     }
1924
1925     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1926 }
1927
1928 sub parse_number_sets {
1929     my ($set) = @_;
1930     my $res = [];
1931     foreach my $part (split(/;/, $set)) {
1932         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1933             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1934             push @$res, [ $1, $2 ];
1935         } else {
1936             die "invalid range: $part\n";
1937         }
1938     }
1939     return $res;
1940 }
1941
1942 sub parse_numa {
1943     my ($data) = @_;
1944
1945     my $res = parse_property_string($numa_fmt, $data);
1946     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1947     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1948     return $res;
1949 }
1950
1951 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1952 sub parse_net {
1953     my ($data, $disable_mac_autogen) = @_;
1954
1955     my $res = eval { parse_property_string($net_fmt, $data) };
1956     if ($@) {
1957         warn $@;
1958         return;
1959     }
1960     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1961         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1962         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1963     }
1964     return $res;
1965 }
1966
1967 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1968 sub parse_ipconfig {
1969     my ($data) = @_;
1970
1971     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1972     if ($@) {
1973         warn $@;
1974         return;
1975     }
1976
1977     if ($res->{gw} && !$res->{ip}) {
1978         warn 'gateway specified without specifying an IP address';
1979         return;
1980     }
1981     if ($res->{gw6} && !$res->{ip6}) {
1982         warn 'IPv6 gateway specified without specifying an IPv6 address';
1983         return;
1984     }
1985     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1986         warn 'gateway specified together with DHCP';
1987         return;
1988     }
1989     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1990         # gw6 + auto/dhcp
1991         warn "IPv6 gateway specified together with $res->{ip6} address";
1992         return;
1993     }
1994
1995     if (!$res->{ip} && !$res->{ip6}) {
1996         return { ip => 'dhcp', ip6 => 'dhcp' };
1997     }
1998
1999     return $res;
2000 }
2001
2002 sub print_net {
2003     my $net = shift;
2004
2005     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2006 }
2007
2008 sub add_random_macs {
2009     my ($settings) = @_;
2010
2011     foreach my $opt (keys %$settings) {
2012         next if $opt !~ m/^net(\d+)$/;
2013         my $net = parse_net($settings->{$opt});
2014         next if !$net;
2015         $settings->{$opt} = print_net($net);
2016     }
2017 }
2018
2019 sub vm_is_volid_owner {
2020     my ($storecfg, $vmid, $volid) = @_;
2021
2022     if ($volid !~  m|^/|) {
2023         my ($path, $owner);
2024         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2025         if ($owner && ($owner == $vmid)) {
2026             return 1;
2027         }
2028     }
2029
2030     return;
2031 }
2032
2033 sub vmconfig_register_unused_drive {
2034     my ($storecfg, $vmid, $conf, $drive) = @_;
2035
2036     if (drive_is_cloudinit($drive)) {
2037         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2038         warn $@ if $@;
2039         delete $conf->{cloudinit};
2040     } elsif (!drive_is_cdrom($drive)) {
2041         my $volid = $drive->{file};
2042         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2043             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2044         }
2045     }
2046 }
2047
2048 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2049 my $smbios1_fmt = {
2050     uuid => {
2051         type => 'string',
2052         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2053         format_description => 'UUID',
2054         description => "Set SMBIOS1 UUID.",
2055         optional => 1,
2056     },
2057     version => {
2058         type => 'string',
2059         pattern => '[A-Za-z0-9+\/]+={0,2}',
2060         format_description => 'Base64 encoded string',
2061         description => "Set SMBIOS1 version.",
2062         optional => 1,
2063     },
2064     serial => {
2065         type => 'string',
2066         pattern => '[A-Za-z0-9+\/]+={0,2}',
2067         format_description => 'Base64 encoded string',
2068         description => "Set SMBIOS1 serial number.",
2069         optional => 1,
2070     },
2071     manufacturer => {
2072         type => 'string',
2073         pattern => '[A-Za-z0-9+\/]+={0,2}',
2074         format_description => 'Base64 encoded string',
2075         description => "Set SMBIOS1 manufacturer.",
2076         optional => 1,
2077     },
2078     product => {
2079         type => 'string',
2080         pattern => '[A-Za-z0-9+\/]+={0,2}',
2081         format_description => 'Base64 encoded string',
2082         description => "Set SMBIOS1 product ID.",
2083         optional => 1,
2084     },
2085     sku => {
2086         type => 'string',
2087         pattern => '[A-Za-z0-9+\/]+={0,2}',
2088         format_description => 'Base64 encoded string',
2089         description => "Set SMBIOS1 SKU string.",
2090         optional => 1,
2091     },
2092     family => {
2093         type => 'string',
2094         pattern => '[A-Za-z0-9+\/]+={0,2}',
2095         format_description => 'Base64 encoded string',
2096         description => "Set SMBIOS1 family string.",
2097         optional => 1,
2098     },
2099     base64 => {
2100         type => 'boolean',
2101         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2102         optional => 1,
2103     },
2104 };
2105
2106 sub parse_smbios1 {
2107     my ($data) = @_;
2108
2109     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2110     warn $@ if $@;
2111     return $res;
2112 }
2113
2114 sub print_smbios1 {
2115     my ($smbios1) = @_;
2116     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2117 }
2118
2119 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2120
2121 sub parse_watchdog {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub parse_guest_agent {
2132     my ($conf) = @_;
2133
2134     return {} if !defined($conf->{agent});
2135
2136     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2137     warn $@ if $@;
2138
2139     # if the agent is disabled ignore the other potentially set properties
2140     return {} if !$res->{enabled};
2141     return $res;
2142 }
2143
2144 sub get_qga_key {
2145     my ($conf, $key) = @_;
2146     return undef if !defined($conf->{agent});
2147
2148     my $agent = parse_guest_agent($conf);
2149     return $agent->{$key};
2150 }
2151
2152 sub parse_vga {
2153     my ($value) = @_;
2154
2155     return {} if !$value;
2156     my $res = eval { parse_property_string($vga_fmt, $value) };
2157     warn $@ if $@;
2158     return $res;
2159 }
2160
2161 sub parse_rng {
2162     my ($value) = @_;
2163
2164     return if !$value;
2165
2166     my $res = eval { parse_property_string($rng_fmt, $value) };
2167     warn $@ if $@;
2168     return $res;
2169 }
2170
2171 sub parse_meta_info {
2172     my ($value) = @_;
2173
2174     return if !$value;
2175
2176     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2177     warn $@ if $@;
2178     return $res;
2179 }
2180
2181 sub new_meta_info_string {
2182     my () = @_; # for now do not allow to override any value
2183
2184     return PVE::JSONSchema::print_property_string(
2185         {
2186             'creation-qemu' => kvm_user_version(),
2187             ctime => "". int(time()),
2188         },
2189         $meta_info_fmt
2190     );
2191 }
2192
2193 sub qemu_created_version_fixups {
2194     my ($conf, $forcemachine, $kvmver) = @_;
2195
2196     my $meta = parse_meta_info($conf->{meta}) // {};
2197     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2198
2199     # check if we need to apply some handling for VMs that always use the latest machine version but
2200     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2201     # would be required (we do not want to pin machine version for non-windows OS type)
2202     if (
2203         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2204         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2205         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2206         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2207     ) {
2208         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2209         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2210             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2211             # and thus with the predictable interface naming of systemd
2212             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2213         }
2214     }
2215     return;
2216 }
2217
2218 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2219 sub verify_usb_device {
2220     my ($value, $noerr) = @_;
2221
2222     return $value if parse_usb_device($value);
2223
2224     return if $noerr;
2225
2226     die "unable to parse usb device\n";
2227 }
2228
2229 # add JSON properties for create and set function
2230 sub json_config_properties {
2231     my ($prop, $with_disk_alloc) = @_;
2232
2233     my $skip_json_config_opts = {
2234         parent => 1,
2235         snaptime => 1,
2236         vmstate => 1,
2237         runningmachine => 1,
2238         runningcpu => 1,
2239         meta => 1,
2240     };
2241
2242     foreach my $opt (keys %$confdesc) {
2243         next if $skip_json_config_opts->{$opt};
2244
2245         if ($with_disk_alloc && is_valid_drivename($opt)) {
2246             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2247         } else {
2248             $prop->{$opt} = $confdesc->{$opt};
2249         }
2250     }
2251
2252     return $prop;
2253 }
2254
2255 # Properties that we can read from an OVF file
2256 sub json_ovf_properties {
2257     my $prop = {};
2258
2259     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2260         $prop->{$device} = {
2261             type => 'string',
2262             format => 'pve-volume-id-or-absolute-path',
2263             description => "Disk image that gets imported to $device",
2264             optional => 1,
2265         };
2266     }
2267
2268     $prop->{cores} = {
2269         type => 'integer',
2270         description => "The number of CPU cores.",
2271         optional => 1,
2272     };
2273     $prop->{memory} = {
2274         type => 'integer',
2275         description => "Amount of RAM for the VM in MB.",
2276         optional => 1,
2277     };
2278     $prop->{name} = {
2279         type => 'string',
2280         description => "Name of the VM.",
2281         optional => 1,
2282     };
2283
2284     return $prop;
2285 }
2286
2287 # return copy of $confdesc_cloudinit to generate documentation
2288 sub cloudinit_config_properties {
2289
2290     return dclone($confdesc_cloudinit);
2291 }
2292
2293 sub cloudinit_pending_properties {
2294     my $p = {
2295         map { $_ => 1 } keys $confdesc_cloudinit->%*,
2296         name => 1,
2297     };
2298     $p->{"net$_"} = 1 for 0..($MAX_NETS-1);
2299     return $p;
2300 }
2301
2302 sub check_type {
2303     my ($key, $value) = @_;
2304
2305     die "unknown setting '$key'\n" if !$confdesc->{$key};
2306
2307     my $type = $confdesc->{$key}->{type};
2308
2309     if (!defined($value)) {
2310         die "got undefined value\n";
2311     }
2312
2313     if ($value =~ m/[\n\r]/) {
2314         die "property contains a line feed\n";
2315     }
2316
2317     if ($type eq 'boolean') {
2318         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2319         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2320         die "type check ('boolean') failed - got '$value'\n";
2321     } elsif ($type eq 'integer') {
2322         return int($1) if $value =~ m/^(\d+)$/;
2323         die "type check ('integer') failed - got '$value'\n";
2324     } elsif ($type eq 'number') {
2325         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2326         die "type check ('number') failed - got '$value'\n";
2327     } elsif ($type eq 'string') {
2328         if (my $fmt = $confdesc->{$key}->{format}) {
2329             PVE::JSONSchema::check_format($fmt, $value);
2330             return $value;
2331         }
2332         $value =~ s/^\"(.*)\"$/$1/;
2333         return $value;
2334     } else {
2335         die "internal error"
2336     }
2337 }
2338
2339 sub destroy_vm {
2340     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2341
2342     my $conf = PVE::QemuConfig->load_config($vmid);
2343
2344     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2345
2346     if ($conf->{template}) {
2347         # check if any base image is still used by a linked clone
2348         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2349                 my ($ds, $drive) = @_;
2350                 return if drive_is_cdrom($drive);
2351
2352                 my $volid = $drive->{file};
2353                 return if !$volid || $volid =~ m|^/|;
2354
2355                 die "base volume '$volid' is still in use by linked cloned\n"
2356                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2357
2358         });
2359     }
2360
2361     my $volids = {};
2362     my $remove_owned_drive = sub {
2363         my ($ds, $drive) = @_;
2364         return if drive_is_cdrom($drive, 1);
2365
2366         my $volid = $drive->{file};
2367         return if !$volid || $volid =~ m|^/|;
2368         return if $volids->{$volid};
2369
2370         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2371         return if !$path || !$owner || ($owner != $vmid);
2372
2373         $volids->{$volid} = 1;
2374         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2375         warn "Could not remove disk '$volid', check manually: $@" if $@;
2376     };
2377
2378     # only remove disks owned by this VM (referenced in the config)
2379     my $include_opts = {
2380         include_unused => 1,
2381         extra_keys => ['vmstate'],
2382     };
2383     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2384
2385     for my $snap (values %{$conf->{snapshots}}) {
2386         next if !defined($snap->{vmstate});
2387         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2388         next if !defined($drive);
2389         $remove_owned_drive->('vmstate', $drive);
2390     }
2391
2392     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2393
2394     if ($purge_unreferenced) { # also remove unreferenced disk
2395         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2396         PVE::Storage::foreach_volid($vmdisks, sub {
2397             my ($volid, $sid, $volname, $d) = @_;
2398             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2399             warn $@ if $@;
2400         });
2401     }
2402
2403     if (defined $replacement_conf) {
2404         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2405     } else {
2406         PVE::QemuConfig->destroy_config($vmid);
2407     }
2408 }
2409
2410 sub parse_vm_config {
2411     my ($filename, $raw, $strict) = @_;
2412
2413     return if !defined($raw);
2414
2415     my $res = {
2416         digest => Digest::SHA::sha1_hex($raw),
2417         snapshots => {},
2418         pending => {},
2419         cloudinit => {},
2420     };
2421
2422     my $handle_error = sub {
2423         my ($msg) = @_;
2424
2425         if ($strict) {
2426             die $msg;
2427         } else {
2428             warn $msg;
2429         }
2430     };
2431
2432     $filename =~ m|/qemu-server/(\d+)\.conf$|
2433         || die "got strange filename '$filename'";
2434
2435     my $vmid = $1;
2436
2437     my $conf = $res;
2438     my $descr;
2439     my $finish_description = sub {
2440         if (defined($descr)) {
2441             $descr =~ s/\s+$//;
2442             $conf->{description} = $descr;
2443         }
2444         $descr = undef;
2445     };
2446     my $section = '';
2447
2448     my @lines = split(/\n/, $raw);
2449     foreach my $line (@lines) {
2450         next if $line =~ m/^\s*$/;
2451
2452         if ($line =~ m/^\[PENDING\]\s*$/i) {
2453             $section = 'pending';
2454             $finish_description->();
2455             $conf = $res->{$section} = {};
2456             next;
2457         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2458             $section = 'cloudinit';
2459             $finish_description->();
2460             $conf = $res->{$section} = {};
2461             next;
2462
2463         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2464             $section = $1;
2465             $finish_description->();
2466             $conf = $res->{snapshots}->{$section} = {};
2467             next;
2468         }
2469
2470         if ($line =~ m/^\#(.*)$/) {
2471             $descr = '' if !defined($descr);
2472             $descr .= PVE::Tools::decode_text($1) . "\n";
2473             next;
2474         }
2475
2476         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2477             $descr = '' if !defined($descr);
2478             $descr .= PVE::Tools::decode_text($2);
2479         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2480             $conf->{snapstate} = $1;
2481         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2482             my $key = $1;
2483             my $value = $2;
2484             $conf->{$key} = $value;
2485         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2486             my $value = $1;
2487             if ($section eq 'pending') {
2488                 $conf->{delete} = $value; # we parse this later
2489             } else {
2490                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2491             }
2492         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2493             my $key = $1;
2494             my $value = $2;
2495             if ($section eq 'cloudinit') {
2496                 # ignore validation only used for informative purpose
2497                 $conf->{$key} = $value;
2498                 next;
2499             }
2500             eval { $value = check_type($key, $value); };
2501             if ($@) {
2502                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2503             } else {
2504                 $key = 'ide2' if $key eq 'cdrom';
2505                 my $fmt = $confdesc->{$key}->{format};
2506                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2507                     my $v = parse_drive($key, $value);
2508                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2509                         $v->{file} = $volid;
2510                         $value = print_drive($v);
2511                     } else {
2512                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2513                         next;
2514                     }
2515                 }
2516
2517                 $conf->{$key} = $value;
2518             }
2519         } else {
2520             $handle_error->("vm $vmid - unable to parse config: $line\n");
2521         }
2522     }
2523
2524     $finish_description->();
2525     delete $res->{snapstate}; # just to be sure
2526
2527     return $res;
2528 }
2529
2530 sub write_vm_config {
2531     my ($filename, $conf) = @_;
2532
2533     delete $conf->{snapstate}; # just to be sure
2534
2535     if ($conf->{cdrom}) {
2536         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2537         $conf->{ide2} = $conf->{cdrom};
2538         delete $conf->{cdrom};
2539     }
2540
2541     # we do not use 'smp' any longer
2542     if ($conf->{sockets}) {
2543         delete $conf->{smp};
2544     } elsif ($conf->{smp}) {
2545         $conf->{sockets} = $conf->{smp};
2546         delete $conf->{cores};
2547         delete $conf->{smp};
2548     }
2549
2550     my $used_volids = {};
2551
2552     my $cleanup_config = sub {
2553         my ($cref, $pending, $snapname) = @_;
2554
2555         foreach my $key (keys %$cref) {
2556             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2557                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2558             my $value = $cref->{$key};
2559             if ($key eq 'delete') {
2560                 die "propertry 'delete' is only allowed in [PENDING]\n"
2561                     if !$pending;
2562                 # fixme: check syntax?
2563                 next;
2564             }
2565             eval { $value = check_type($key, $value); };
2566             die "unable to parse value of '$key' - $@" if $@;
2567
2568             $cref->{$key} = $value;
2569
2570             if (!$snapname && is_valid_drivename($key)) {
2571                 my $drive = parse_drive($key, $value);
2572                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2573             }
2574         }
2575     };
2576
2577     &$cleanup_config($conf);
2578
2579     &$cleanup_config($conf->{pending}, 1);
2580
2581     foreach my $snapname (keys %{$conf->{snapshots}}) {
2582         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2583         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2584     }
2585
2586     # remove 'unusedX' settings if we re-add a volume
2587     foreach my $key (keys %$conf) {
2588         my $value = $conf->{$key};
2589         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2590             delete $conf->{$key};
2591         }
2592     }
2593
2594     my $generate_raw_config = sub {
2595         my ($conf, $pending) = @_;
2596
2597         my $raw = '';
2598
2599         # add description as comment to top of file
2600         if (defined(my $descr = $conf->{description})) {
2601             if ($descr) {
2602                 foreach my $cl (split(/\n/, $descr)) {
2603                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2604                 }
2605             } else {
2606                 $raw .= "#\n" if $pending;
2607             }
2608         }
2609
2610         foreach my $key (sort keys %$conf) {
2611             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2612             $raw .= "$key: $conf->{$key}\n";
2613         }
2614         return $raw;
2615     };
2616
2617     my $raw = &$generate_raw_config($conf);
2618
2619     if (scalar(keys %{$conf->{pending}})){
2620         $raw .= "\n[PENDING]\n";
2621         $raw .= &$generate_raw_config($conf->{pending}, 1);
2622     }
2623
2624     if (scalar(keys %{$conf->{cloudinit}}) && PVE::QemuConfig->has_cloudinit($conf)){
2625         $raw .= "\n[special:cloudinit]\n";
2626         $raw .= &$generate_raw_config($conf->{cloudinit});
2627     }
2628
2629     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2630         $raw .= "\n[$snapname]\n";
2631         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2632     }
2633
2634     return $raw;
2635 }
2636
2637 sub load_defaults {
2638
2639     my $res = {};
2640
2641     # we use static defaults from our JSON schema configuration
2642     foreach my $key (keys %$confdesc) {
2643         if (defined(my $default = $confdesc->{$key}->{default})) {
2644             $res->{$key} = $default;
2645         }
2646     }
2647
2648     return $res;
2649 }
2650
2651 sub config_list {
2652     my $vmlist = PVE::Cluster::get_vmlist();
2653     my $res = {};
2654     return $res if !$vmlist || !$vmlist->{ids};
2655     my $ids = $vmlist->{ids};
2656     my $nodename = nodename();
2657
2658     foreach my $vmid (keys %$ids) {
2659         my $d = $ids->{$vmid};
2660         next if !$d->{node} || $d->{node} ne $nodename;
2661         next if !$d->{type} || $d->{type} ne 'qemu';
2662         $res->{$vmid}->{exists} = 1;
2663     }
2664     return $res;
2665 }
2666
2667 # test if VM uses local resources (to prevent migration)
2668 sub check_local_resources {
2669     my ($conf, $noerr) = @_;
2670
2671     my @loc_res = ();
2672
2673     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2674     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2675
2676     push @loc_res, "ivshmem" if $conf->{ivshmem};
2677
2678     foreach my $k (keys %$conf) {
2679         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2680         # sockets are safe: they will recreated be on the target side post-migrate
2681         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2682         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2683     }
2684
2685     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2686
2687     return \@loc_res;
2688 }
2689
2690 # check if used storages are available on all nodes (use by migrate)
2691 sub check_storage_availability {
2692     my ($storecfg, $conf, $node) = @_;
2693
2694     PVE::QemuConfig->foreach_volume($conf, sub {
2695         my ($ds, $drive) = @_;
2696
2697         my $volid = $drive->{file};
2698         return if !$volid;
2699
2700         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2701         return if !$sid;
2702
2703         # check if storage is available on both nodes
2704         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2705         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2706
2707         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2708
2709         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2710             if !$scfg->{content}->{$vtype};
2711    });
2712 }
2713
2714 # list nodes where all VM images are available (used by has_feature API)
2715 sub shared_nodes {
2716     my ($conf, $storecfg) = @_;
2717
2718     my $nodelist = PVE::Cluster::get_nodelist();
2719     my $nodehash = { map { $_ => 1 } @$nodelist };
2720     my $nodename = nodename();
2721
2722     PVE::QemuConfig->foreach_volume($conf, sub {
2723         my ($ds, $drive) = @_;
2724
2725         my $volid = $drive->{file};
2726         return if !$volid;
2727
2728         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2729         if ($storeid) {
2730             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2731             if ($scfg->{disable}) {
2732                 $nodehash = {};
2733             } elsif (my $avail = $scfg->{nodes}) {
2734                 foreach my $node (keys %$nodehash) {
2735                     delete $nodehash->{$node} if !$avail->{$node};
2736                 }
2737             } elsif (!$scfg->{shared}) {
2738                 foreach my $node (keys %$nodehash) {
2739                     delete $nodehash->{$node} if $node ne $nodename
2740                 }
2741             }
2742         }
2743     });
2744
2745     return $nodehash
2746 }
2747
2748 sub check_local_storage_availability {
2749     my ($conf, $storecfg) = @_;
2750
2751     my $nodelist = PVE::Cluster::get_nodelist();
2752     my $nodehash = { map { $_ => {} } @$nodelist };
2753
2754     PVE::QemuConfig->foreach_volume($conf, sub {
2755         my ($ds, $drive) = @_;
2756
2757         my $volid = $drive->{file};
2758         return if !$volid;
2759
2760         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2761         if ($storeid) {
2762             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2763
2764             if ($scfg->{disable}) {
2765                 foreach my $node (keys %$nodehash) {
2766                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2767                 }
2768             } elsif (my $avail = $scfg->{nodes}) {
2769                 foreach my $node (keys %$nodehash) {
2770                     if (!$avail->{$node}) {
2771                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2772                     }
2773                 }
2774             }
2775         }
2776     });
2777
2778     foreach my $node (values %$nodehash) {
2779         if (my $unavail = $node->{unavailable_storages}) {
2780             $node->{unavailable_storages} = [ sort keys %$unavail ];
2781         }
2782     }
2783
2784     return $nodehash
2785 }
2786
2787 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2788 sub check_running {
2789     my ($vmid, $nocheck, $node) = @_;
2790
2791     # $nocheck is set when called during a migration, in which case the config
2792     # file might still or already reside on the *other* node
2793     # - because rename has already happened, and current node is source
2794     # - because rename hasn't happened yet, and current node is target
2795     # - because rename has happened, current node is target, but hasn't yet
2796     # processed it yet
2797     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2798     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2799 }
2800
2801 sub vzlist {
2802
2803     my $vzlist = config_list();
2804
2805     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2806
2807     while (defined(my $de = $fd->read)) {
2808         next if $de !~ m/^(\d+)\.pid$/;
2809         my $vmid = $1;
2810         next if !defined($vzlist->{$vmid});
2811         if (my $pid = check_running($vmid)) {
2812             $vzlist->{$vmid}->{pid} = $pid;
2813         }
2814     }
2815
2816     return $vzlist;
2817 }
2818
2819 our $vmstatus_return_properties = {
2820     vmid => get_standard_option('pve-vmid'),
2821     status => {
2822         description => "Qemu process status.",
2823         type => 'string',
2824         enum => ['stopped', 'running'],
2825     },
2826     maxmem => {
2827         description => "Maximum memory in bytes.",
2828         type => 'integer',
2829         optional => 1,
2830         renderer => 'bytes',
2831     },
2832     maxdisk => {
2833         description => "Root disk size in bytes.",
2834         type => 'integer',
2835         optional => 1,
2836         renderer => 'bytes',
2837     },
2838     name => {
2839         description => "VM name.",
2840         type => 'string',
2841         optional => 1,
2842     },
2843     qmpstatus => {
2844         description => "Qemu QMP agent status.",
2845         type => 'string',
2846         optional => 1,
2847     },
2848     pid => {
2849         description => "PID of running qemu process.",
2850         type => 'integer',
2851         optional => 1,
2852     },
2853     uptime => {
2854         description => "Uptime.",
2855         type => 'integer',
2856         optional => 1,
2857         renderer => 'duration',
2858     },
2859     cpus => {
2860         description => "Maximum usable CPUs.",
2861         type => 'number',
2862         optional => 1,
2863     },
2864     lock => {
2865         description => "The current config lock, if any.",
2866         type => 'string',
2867         optional => 1,
2868     },
2869     tags => {
2870         description  => "The current configured tags, if any",
2871         type => 'string',
2872         optional => 1,
2873     },
2874     'running-machine' => {
2875         description => "The currently running machine type (if running).",
2876         type => 'string',
2877         optional => 1,
2878     },
2879     'running-qemu' => {
2880         description => "The currently running QEMU version (if running).",
2881         type => 'string',
2882         optional => 1,
2883     },
2884 };
2885
2886 my $last_proc_pid_stat;
2887
2888 # get VM status information
2889 # This must be fast and should not block ($full == false)
2890 # We only query KVM using QMP if $full == true (this can be slow)
2891 sub vmstatus {
2892     my ($opt_vmid, $full) = @_;
2893
2894     my $res = {};
2895
2896     my $storecfg = PVE::Storage::config();
2897
2898     my $list = vzlist();
2899     my $defaults = load_defaults();
2900
2901     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2902
2903     my $cpucount = $cpuinfo->{cpus} || 1;
2904
2905     foreach my $vmid (keys %$list) {
2906         next if $opt_vmid && ($vmid ne $opt_vmid);
2907
2908         my $conf = PVE::QemuConfig->load_config($vmid);
2909
2910         my $d = { vmid => int($vmid) };
2911         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2912
2913         # fixme: better status?
2914         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2915
2916         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2917         if (defined($size)) {
2918             $d->{disk} = 0; # no info available
2919             $d->{maxdisk} = $size;
2920         } else {
2921             $d->{disk} = 0;
2922             $d->{maxdisk} = 0;
2923         }
2924
2925         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2926             * ($conf->{cores} || $defaults->{cores});
2927         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2928         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2929
2930         $d->{name} = $conf->{name} || "VM $vmid";
2931         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2932             : $defaults->{memory}*(1024*1024);
2933
2934         if ($conf->{balloon}) {
2935             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2936             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2937                 : $defaults->{shares};
2938         }
2939
2940         $d->{uptime} = 0;
2941         $d->{cpu} = 0;
2942         $d->{mem} = 0;
2943
2944         $d->{netout} = 0;
2945         $d->{netin} = 0;
2946
2947         $d->{diskread} = 0;
2948         $d->{diskwrite} = 0;
2949
2950         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2951
2952         $d->{serial} = 1 if conf_has_serial($conf);
2953         $d->{lock} = $conf->{lock} if $conf->{lock};
2954         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2955
2956         $res->{$vmid} = $d;
2957     }
2958
2959     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2960     foreach my $dev (keys %$netdev) {
2961         next if $dev !~ m/^tap([1-9]\d*)i/;
2962         my $vmid = $1;
2963         my $d = $res->{$vmid};
2964         next if !$d;
2965
2966         $d->{netout} += $netdev->{$dev}->{receive};
2967         $d->{netin} += $netdev->{$dev}->{transmit};
2968
2969         if ($full) {
2970             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2971             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2972         }
2973
2974     }
2975
2976     my $ctime = gettimeofday;
2977
2978     foreach my $vmid (keys %$list) {
2979
2980         my $d = $res->{$vmid};
2981         my $pid = $d->{pid};
2982         next if !$pid;
2983
2984         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2985         next if !$pstat; # not running
2986
2987         my $used = $pstat->{utime} + $pstat->{stime};
2988
2989         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2990
2991         if ($pstat->{vsize}) {
2992             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2993         }
2994
2995         my $old = $last_proc_pid_stat->{$pid};
2996         if (!$old) {
2997             $last_proc_pid_stat->{$pid} = {
2998                 time => $ctime,
2999                 used => $used,
3000                 cpu => 0,
3001             };
3002             next;
3003         }
3004
3005         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3006
3007         if ($dtime > 1000) {
3008             my $dutime = $used -  $old->{used};
3009
3010             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3011             $last_proc_pid_stat->{$pid} = {
3012                 time => $ctime,
3013                 used => $used,
3014                 cpu => $d->{cpu},
3015             };
3016         } else {
3017             $d->{cpu} = $old->{cpu};
3018         }
3019     }
3020
3021     return $res if !$full;
3022
3023     my $qmpclient = PVE::QMPClient->new();
3024
3025     my $ballooncb = sub {
3026         my ($vmid, $resp) = @_;
3027
3028         my $info = $resp->{'return'};
3029         return if !$info->{max_mem};
3030
3031         my $d = $res->{$vmid};
3032
3033         # use memory assigned to VM
3034         $d->{maxmem} = $info->{max_mem};
3035         $d->{balloon} = $info->{actual};
3036
3037         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3038             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3039             $d->{freemem} = $info->{free_mem};
3040         }
3041
3042         $d->{ballooninfo} = $info;
3043     };
3044
3045     my $blockstatscb = sub {
3046         my ($vmid, $resp) = @_;
3047         my $data = $resp->{'return'} || [];
3048         my $totalrdbytes = 0;
3049         my $totalwrbytes = 0;
3050
3051         for my $blockstat (@$data) {
3052             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3053             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3054
3055             $blockstat->{device} =~ s/drive-//;
3056             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3057         }
3058         $res->{$vmid}->{diskread} = $totalrdbytes;
3059         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3060     };
3061
3062     my $machinecb = sub {
3063         my ($vmid, $resp) = @_;
3064         my $data = $resp->{'return'} || [];
3065
3066         $res->{$vmid}->{'running-machine'} =
3067             PVE::QemuServer::Machine::current_from_query_machines($data);
3068     };
3069
3070     my $versioncb = sub {
3071         my ($vmid, $resp) = @_;
3072         my $data = $resp->{'return'} // {};
3073         my $version = 'unknown';
3074
3075         if (my $v = $data->{qemu}) {
3076             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3077         }
3078
3079         $res->{$vmid}->{'running-qemu'} = $version;
3080     };
3081
3082     my $statuscb = sub {
3083         my ($vmid, $resp) = @_;
3084
3085         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3086         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3087         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3088         # this fails if ballon driver is not loaded, so this must be
3089         # the last commnand (following command are aborted if this fails).
3090         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3091
3092         my $status = 'unknown';
3093         if (!defined($status = $resp->{'return'}->{status})) {
3094             warn "unable to get VM status\n";
3095             return;
3096         }
3097
3098         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3099     };
3100
3101     foreach my $vmid (keys %$list) {
3102         next if $opt_vmid && ($vmid ne $opt_vmid);
3103         next if !$res->{$vmid}->{pid}; # not running
3104         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3105     }
3106
3107     $qmpclient->queue_execute(undef, 2);
3108
3109     foreach my $vmid (keys %$list) {
3110         next if $opt_vmid && ($vmid ne $opt_vmid);
3111         next if !$res->{$vmid}->{pid}; #not running
3112
3113         # we can't use the $qmpclient since it might have already aborted on
3114         # 'query-balloon', but this might also fail for older versions...
3115         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3116         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3117     }
3118
3119     foreach my $vmid (keys %$list) {
3120         next if $opt_vmid && ($vmid ne $opt_vmid);
3121         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3122     }
3123
3124     return $res;
3125 }
3126
3127 sub conf_has_serial {
3128     my ($conf) = @_;
3129
3130     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3131         if ($conf->{"serial$i"}) {
3132             return 1;
3133         }
3134     }
3135
3136     return 0;
3137 }
3138
3139 sub conf_has_audio {
3140     my ($conf, $id) = @_;
3141
3142     $id //= 0;
3143     my $audio = $conf->{"audio$id"};
3144     return if !defined($audio);
3145
3146     my $audioproperties = parse_property_string($audio_fmt, $audio);
3147     my $audiodriver = $audioproperties->{driver} // 'spice';
3148
3149     return {
3150         dev => $audioproperties->{device},
3151         dev_id => "audiodev$id",
3152         backend => $audiodriver,
3153         backend_id => "$audiodriver-backend${id}",
3154     };
3155 }
3156
3157 sub audio_devs {
3158     my ($audio, $audiopciaddr, $machine_version) = @_;
3159
3160     my $devs = [];
3161
3162     my $id = $audio->{dev_id};
3163     my $audiodev = "";
3164     if (min_version($machine_version, 4, 2)) {
3165         $audiodev = ",audiodev=$audio->{backend_id}";
3166     }
3167
3168     if ($audio->{dev} eq 'AC97') {
3169         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3170     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3171         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3172         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3173         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3174     } else {
3175         die "unkown audio device '$audio->{dev}', implement me!";
3176     }
3177
3178     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3179
3180     return $devs;
3181 }
3182
3183 sub get_tpm_paths {
3184     my ($vmid) = @_;
3185     return {
3186         socket => "/var/run/qemu-server/$vmid.swtpm",
3187         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3188     };
3189 }
3190
3191 sub add_tpm_device {
3192     my ($vmid, $devices, $conf) = @_;
3193
3194     return if !$conf->{tpmstate0};
3195
3196     my $paths = get_tpm_paths($vmid);
3197
3198     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3199     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3200     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3201 }
3202
3203 sub start_swtpm {
3204     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3205
3206     return if !$tpmdrive;
3207
3208     my $state;
3209     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3210     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3211     if ($storeid) {
3212         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3213     } else {
3214         $state = $tpm->{file};
3215     }
3216
3217     my $paths = get_tpm_paths($vmid);
3218
3219     # during migration, we will get state from remote
3220     #
3221     if (!$migration) {
3222         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3223         my $setup_cmd = [
3224             "swtpm_setup",
3225             "--tpmstate",
3226             "file://$state",
3227             "--createek",
3228             "--create-ek-cert",
3229             "--create-platform-cert",
3230             "--lock-nvram",
3231             "--config",
3232             "/etc/swtpm_setup.conf", # do not use XDG configs
3233             "--runas",
3234             "0", # force creation as root, error if not possible
3235             "--not-overwrite", # ignore existing state, do not modify
3236         ];
3237
3238         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3239         # TPM 2.0 supports ECC crypto, use if possible
3240         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3241
3242         run_command($setup_cmd, outfunc => sub {
3243             print "swtpm_setup: $1\n";
3244         });
3245     }
3246
3247     my $emulator_cmd = [
3248         "swtpm",
3249         "socket",
3250         "--tpmstate",
3251         "backend-uri=file://$state,mode=0600",
3252         "--ctrl",
3253         "type=unixio,path=$paths->{socket},mode=0600",
3254         "--pid",
3255         "file=$paths->{pid}",
3256         "--terminate", # terminate on QEMU disconnect
3257         "--daemon",
3258     ];
3259     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3260     run_command($emulator_cmd, outfunc => sub { print $1; });
3261
3262     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3263     while (! -e $paths->{pid}) {
3264         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3265         usleep(50_000);
3266     }
3267
3268     # return untainted PID of swtpm daemon so it can be killed on error
3269     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3270     return $1;
3271 }
3272
3273 sub vga_conf_has_spice {
3274     my ($vga) = @_;
3275
3276     my $vgaconf = parse_vga($vga);
3277     my $vgatype = $vgaconf->{type};
3278     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3279
3280     return $1 || 1;
3281 }
3282
3283 sub is_native($) {
3284     my ($arch) = @_;
3285     return get_host_arch() eq $arch;
3286 }
3287
3288 sub get_vm_arch {
3289     my ($conf) = @_;
3290     return $conf->{arch} // get_host_arch();
3291 }
3292
3293 my $default_machines = {
3294     x86_64 => 'pc',
3295     aarch64 => 'virt',
3296 };
3297
3298 sub get_installed_machine_version {
3299     my ($kvmversion) = @_;
3300     $kvmversion = kvm_user_version() if !defined($kvmversion);
3301     $kvmversion =~ m/^(\d+\.\d+)/;
3302     return $1;
3303 }
3304
3305 sub windows_get_pinned_machine_version {
3306     my ($machine, $base_version, $kvmversion) = @_;
3307
3308     my $pin_version = $base_version;
3309     if (!defined($base_version) ||
3310         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3311     ) {
3312         $pin_version = get_installed_machine_version($kvmversion);
3313     }
3314     if (!$machine || $machine eq 'pc') {
3315         $machine = "pc-i440fx-$pin_version";
3316     } elsif ($machine eq 'q35') {
3317         $machine = "pc-q35-$pin_version";
3318     } elsif ($machine eq 'virt') {
3319         $machine = "virt-$pin_version";
3320     } else {
3321         warn "unknown machine type '$machine', not touching that!\n";
3322     }
3323
3324     return $machine;
3325 }
3326
3327 sub get_vm_machine {
3328     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3329
3330     my $machine = $forcemachine || $conf->{machine};
3331
3332     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3333         $kvmversion //= kvm_user_version();
3334         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3335         # layout which confuses windows quite a bit and may result in various regressions..
3336         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3337         if (windows_version($conf->{ostype})) {
3338             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3339         }
3340         $arch //= 'x86_64';
3341         $machine ||= $default_machines->{$arch};
3342         if ($add_pve_version) {
3343             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3344             $machine .= "+pve$pvever";
3345         }
3346     }
3347
3348     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3349         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3350         $machine = $1 if $is_pxe;
3351
3352         # for version-pinned machines that do not include a pve-version (e.g.
3353         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3354         $machine .= '+pve0';
3355
3356         $machine .= '.pxe' if $is_pxe;
3357     }
3358
3359     return $machine;
3360 }
3361
3362 sub get_ovmf_files($$$) {
3363     my ($arch, $efidisk, $smm) = @_;
3364
3365     my $types = $OVMF->{$arch}
3366         or die "no OVMF images known for architecture '$arch'\n";
3367
3368     my $type = 'default';
3369     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3370         $type = $smm ? "4m" : "4m-no-smm";
3371         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3372     }
3373
3374     return $types->{$type}->@*;
3375 }
3376
3377 my $Arch2Qemu = {
3378     aarch64 => '/usr/bin/qemu-system-aarch64',
3379     x86_64 => '/usr/bin/qemu-system-x86_64',
3380 };
3381 sub get_command_for_arch($) {
3382     my ($arch) = @_;
3383     return '/usr/bin/kvm' if is_native($arch);
3384
3385     my $cmd = $Arch2Qemu->{$arch}
3386         or die "don't know how to emulate architecture '$arch'\n";
3387     return $cmd;
3388 }
3389
3390 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3391 # to use in a QEMU command line (-cpu element), first array_intersect the result
3392 # of query_supported_ with query_understood_. This is necessary because:
3393 #
3394 # a) query_understood_ returns flags the host cannot use and
3395 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3396 #    flags, but CPU settings - with most of them being flags. Those settings
3397 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3398 #
3399 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3400 # expensive. If you need the value returned from this, you can get it much
3401 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3402 # $accel being 'kvm' or 'tcg'.
3403 #
3404 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3405 # changes, automatically populating pmxcfs.
3406 #
3407 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3408 # since kvm and tcg machines support different flags
3409 #
3410 sub query_supported_cpu_flags {
3411     my ($arch) = @_;
3412
3413     $arch //= get_host_arch();
3414     my $default_machine = $default_machines->{$arch};
3415
3416     my $flags = {};
3417
3418     # FIXME: Once this is merged, the code below should work for ARM as well:
3419     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3420     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3421         $arch eq "aarch64";
3422
3423     my $kvm_supported = defined(kvm_version());
3424     my $qemu_cmd = get_command_for_arch($arch);
3425     my $fakevmid = -1;
3426     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3427
3428     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3429     my $query_supported_run_qemu = sub {
3430         my ($kvm) = @_;
3431
3432         my $flags = {};
3433         my $cmd = [
3434             $qemu_cmd,
3435             '-machine', $default_machine,
3436             '-display', 'none',
3437             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3438             '-mon', 'chardev=qmp,mode=control',
3439             '-pidfile', $pidfile,
3440             '-S', '-daemonize'
3441         ];
3442
3443         if (!$kvm) {
3444             push @$cmd, '-accel', 'tcg';
3445         }
3446
3447         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3448         die "QEMU flag querying VM exited with code " . $rc if $rc;
3449
3450         eval {
3451             my $cmd_result = mon_cmd(
3452                 $fakevmid,
3453                 'query-cpu-model-expansion',
3454                 type => 'full',
3455                 model => { name => 'host' }
3456             );
3457
3458             my $props = $cmd_result->{model}->{props};
3459             foreach my $prop (keys %$props) {
3460                 next if $props->{$prop} ne '1';
3461                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3462                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3463                 # We only keep those with underscores, to match /proc/cpuinfo
3464                 $prop =~ s/\.|-/_/g;
3465                 $flags->{$prop} = 1;
3466             }
3467         };
3468         my $err = $@;
3469
3470         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3471         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3472
3473         die $err if $err;
3474
3475         return [ sort keys %$flags ];
3476     };
3477
3478     # We need to query QEMU twice, since KVM and TCG have different supported flags
3479     PVE::QemuConfig->lock_config($fakevmid, sub {
3480         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3481         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3482
3483         if ($kvm_supported) {
3484             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3485             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3486         }
3487     });
3488
3489     return $flags;
3490 }
3491
3492 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3493 my $understood_cpu_flag_dir = "/usr/share/kvm";
3494 sub query_understood_cpu_flags {
3495     my $arch = get_host_arch();
3496     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3497
3498     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3499         if ! -e $filepath;
3500
3501     my $raw = file_get_contents($filepath);
3502     $raw =~ s/^\s+|\s+$//g;
3503     my @flags = split(/\s+/, $raw);
3504
3505     return \@flags;
3506 }
3507
3508 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3509 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3510 my sub should_disable_smm {
3511     my ($conf, $vga) = @_;
3512
3513     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3514         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3515 }
3516
3517 my sub print_ovmf_drive_commandlines {
3518     my ($conf, $storecfg, $vmid, $arch, $q35, $version_guard) = @_;
3519
3520     my $d = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
3521
3522     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3523     die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3524
3525     my $var_drive_str = "if=pflash,unit=1,id=drive-efidisk0";
3526     if ($d) {
3527         my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3528         my ($path, $format) = $d->@{'file', 'format'};
3529         if ($storeid) {
3530             $path = PVE::Storage::path($storecfg, $d->{file});
3531             if (!defined($format)) {
3532                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3533                 $format = qemu_img_format($scfg, $volname);
3534             }
3535         } elsif (!defined($format)) {
3536             die "efidisk format must be specified\n";
3537         }
3538         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3539         if ($path =~ m/^rbd:/) {
3540             $var_drive_str .= ',cache=writeback';
3541             $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3542         }
3543         $var_drive_str .= ",format=$format,file=$path";
3544
3545         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $format eq 'raw' && $version_guard->(4, 1, 2);
3546         $var_drive_str .= ',readonly=on' if drive_is_read_only($conf, $d);
3547     } else {
3548         log_warn("no efidisk configured! Using temporary efivars disk.");
3549         my $path = "/tmp/$vmid-ovmf.fd";
3550         PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3551         $var_drive_str .= ",format=raw,file=$path";
3552         $var_drive_str .= ",size=" . (-s $ovmf_vars) if $version_guard->(4, 1, 2);
3553     }
3554
3555     return ("if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code", $var_drive_str);
3556 }
3557
3558 sub config_to_command {
3559     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3560         $pbs_backing) = @_;
3561
3562     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3563     my $devices = [];
3564     my $bridges = {};
3565     my $ostype = $conf->{ostype};
3566     my $winversion = windows_version($ostype);
3567     my $kvm = $conf->{kvm};
3568     my $nodename = nodename();
3569
3570     my $arch = get_vm_arch($conf);
3571     my $kvm_binary = get_command_for_arch($arch);
3572     my $kvmver = kvm_user_version($kvm_binary);
3573
3574     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3575         $kvmver //= "undefined";
3576         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3577     }
3578
3579     my $add_pve_version = min_version($kvmver, 4, 1);
3580
3581     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3582     my $machine_version = extract_version($machine_type, $kvmver);
3583     $kvm //= 1 if is_native($arch);
3584
3585     $machine_version =~ m/(\d+)\.(\d+)/;
3586     my ($machine_major, $machine_minor) = ($1, $2);
3587
3588     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3589         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3590     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3591         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3592             ." please upgrade node '$nodename'\n"
3593     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3594         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3595         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3596             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3597             ." node '$nodename'\n";
3598     }
3599
3600     # if a specific +pve version is required for a feature, use $version_guard
3601     # instead of min_version to allow machines to be run with the minimum
3602     # required version
3603     my $required_pve_version = 0;
3604     my $version_guard = sub {
3605         my ($major, $minor, $pve) = @_;
3606         return 0 if !min_version($machine_version, $major, $minor, $pve);
3607         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3608         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3609         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3610         return 1;
3611     };
3612
3613     if ($kvm && !defined kvm_version()) {
3614         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3615             ." or enable in BIOS.\n";
3616     }
3617
3618     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3619     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3620     my $use_old_bios_files = undef;
3621     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3622
3623     my $cmd = [];
3624     if ($conf->{affinity}) {
3625         push @$cmd, '/usr/bin/taskset', '--cpu-list', '--all-tasks', $conf->{affinity};
3626     }
3627
3628     push @$cmd, $kvm_binary;
3629
3630     push @$cmd, '-id', $vmid;
3631
3632     my $vmname = $conf->{name} || "vm$vmid";
3633
3634     push @$cmd, '-name', "$vmname,debug-threads=on";
3635
3636     push @$cmd, '-no-shutdown';
3637
3638     my $use_virtio = 0;
3639
3640     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3641     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3642     push @$cmd, '-mon', "chardev=qmp,mode=control";
3643
3644     if (min_version($machine_version, 2, 12)) {
3645         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3646         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3647     }
3648
3649     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3650
3651     push @$cmd, '-daemonize';
3652
3653     if ($conf->{smbios1}) {
3654         my $smbios_conf = parse_smbios1($conf->{smbios1});
3655         if ($smbios_conf->{base64}) {
3656             # Do not pass base64 flag to qemu
3657             delete $smbios_conf->{base64};
3658             my $smbios_string = "";
3659             foreach my $key (keys %$smbios_conf) {
3660                 my $value;
3661                 if ($key eq "uuid") {
3662                     $value = $smbios_conf->{uuid}
3663                 } else {
3664                     $value = decode_base64($smbios_conf->{$key});
3665                 }
3666                 # qemu accepts any binary data, only commas need escaping by double comma
3667                 $value =~ s/,/,,/g;
3668                 $smbios_string .= "," . $key . "=" . $value if $value;
3669             }
3670             push @$cmd, '-smbios', "type=1" . $smbios_string;
3671         } else {
3672             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3673         }
3674     }
3675
3676     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3677         my ($code_drive_str, $var_drive_str) =
3678             print_ovmf_drive_commandlines($conf, $storecfg, $vmid, $arch, $q35, $version_guard);
3679         push $cmd->@*, '-drive', $code_drive_str;
3680         push $cmd->@*, '-drive', $var_drive_str;
3681     }
3682
3683     if ($q35) { # tell QEMU to load q35 config early
3684         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3685         if (min_version($machine_version, 4, 0)) {
3686             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3687         } else {
3688             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3689         }
3690     }
3691
3692     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3693         push @$cmd, $fixups->@*;
3694     }
3695
3696     if ($conf->{vmgenid}) {
3697         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3698     }
3699
3700     # add usb controllers
3701     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3702         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3703     push @$devices, @usbcontrollers if @usbcontrollers;
3704     my $vga = parse_vga($conf->{vga});
3705
3706     my $qxlnum = vga_conf_has_spice($conf->{vga});
3707     $vga->{type} = 'qxl' if $qxlnum;
3708
3709     if (!$vga->{type}) {
3710         if ($arch eq 'aarch64') {
3711             $vga->{type} = 'virtio';
3712         } elsif (min_version($machine_version, 2, 9)) {
3713             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3714         } else {
3715             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3716         }
3717     }
3718
3719     # enable absolute mouse coordinates (needed by vnc)
3720     my $tablet = $conf->{tablet};
3721     if (!defined($tablet)) {
3722         $tablet = $defaults->{tablet};
3723         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3724         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3725     }
3726
3727     if ($tablet) {
3728         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3729         my $kbd = print_keyboarddevice_full($conf, $arch);
3730         push @$devices, '-device', $kbd if defined($kbd);
3731     }
3732
3733     my $bootorder = device_bootorder($conf);
3734
3735     # host pci device passthrough
3736     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3737         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3738
3739     # usb devices
3740     my $usb_dev_features = {};
3741     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3742
3743     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3744         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3745     push @$devices, @usbdevices if @usbdevices;
3746
3747     # serial devices
3748     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3749         my $path = $conf->{"serial$i"} or next;
3750         if ($path eq 'socket') {
3751             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3752             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3753             # On aarch64, serial0 is the UART device. Qemu only allows
3754             # connecting UART devices via the '-serial' command line, as
3755             # the device has a fixed slot on the hardware...
3756             if ($arch eq 'aarch64' && $i == 0) {
3757                 push @$devices, '-serial', "chardev:serial$i";
3758             } else {
3759                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3760             }
3761         } else {
3762             die "no such serial device\n" if ! -c $path;
3763             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3764             push @$devices, '-device', "isa-serial,chardev=serial$i";
3765         }
3766     }
3767
3768     # parallel devices
3769     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3770         if (my $path = $conf->{"parallel$i"}) {
3771             die "no such parallel device\n" if ! -c $path;
3772             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3773             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3774             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3775         }
3776     }
3777
3778     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3779         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3780         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3781         push @$devices, @$audio_devs;
3782     }
3783
3784     add_tpm_device($vmid, $devices, $conf);
3785
3786     my $sockets = 1;
3787     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3788     $sockets = $conf->{sockets} if  $conf->{sockets};
3789
3790     my $cores = $conf->{cores} || 1;
3791
3792     my $maxcpus = $sockets * $cores;
3793
3794     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3795
3796     my $allowed_vcpus = $cpuinfo->{cpus};
3797
3798     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3799
3800     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3801         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3802         for (my $i = 2; $i <= $vcpus; $i++)  {
3803             my $cpustr = print_cpu_device($conf,$i);
3804             push @$cmd, '-device', $cpustr;
3805         }
3806
3807     } else {
3808
3809         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3810     }
3811     push @$cmd, '-nodefaults';
3812
3813     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3814
3815     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3816
3817     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3818
3819     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3820         push @$devices, '-device', print_vga_device(
3821             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3822
3823         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3824
3825         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3826         push @$cmd,  '-vnc', "unix:$socket,password=on";
3827     } else {
3828         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3829         push @$cmd, '-nographic';
3830     }
3831
3832     # time drift fix
3833     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3834     my $useLocaltime = $conf->{localtime};
3835
3836     if ($winversion >= 5) { # windows
3837         $useLocaltime = 1 if !defined($conf->{localtime});
3838
3839         # use time drift fix when acpi is enabled
3840         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3841             $tdf = 1 if !defined($conf->{tdf});
3842         }
3843     }
3844
3845     if ($winversion >= 6) {
3846         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3847         push @$cmd, '-no-hpet';
3848     }
3849
3850     push @$rtcFlags, 'driftfix=slew' if $tdf;
3851
3852     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3853         push @$rtcFlags, "base=$conf->{startdate}";
3854     } elsif ($useLocaltime) {
3855         push @$rtcFlags, 'base=localtime';
3856     }
3857
3858     if ($forcecpu) {
3859         push @$cmd, '-cpu', $forcecpu;
3860     } else {
3861         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3862     }
3863
3864     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3865
3866     push @$cmd, '-S' if $conf->{freeze};
3867
3868     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3869
3870     my $guest_agent = parse_guest_agent($conf);
3871
3872     if ($guest_agent->{enabled}) {
3873         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3874         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3875
3876         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3877             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3878             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3879             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3880         } elsif ($guest_agent->{type} eq 'isa') {
3881             push @$devices, '-device', "isa-serial,chardev=qga0";
3882         }
3883     }
3884
3885     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3886     if ($rng && $version_guard->(4, 1, 2)) {
3887         check_rng_source($rng->{source});
3888
3889         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3890         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3891         my $limiter_str = "";
3892         if ($max_bytes) {
3893             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3894         }
3895
3896         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3897         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3898         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3899     }
3900
3901     my $spice_port;
3902
3903     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3904         if ($qxlnum > 1) {
3905             if ($winversion){
3906                 for (my $i = 1; $i < $qxlnum; $i++){
3907                     push @$devices, '-device', print_vga_device(
3908                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3909                 }
3910             } else {
3911                 # assume other OS works like Linux
3912                 my ($ram, $vram) = ("134217728", "67108864");
3913                 if ($vga->{memory}) {
3914                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3915                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3916                 }
3917                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3918                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3919             }
3920         }
3921
3922         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3923
3924         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3925         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3926         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3927
3928         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3929         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3930         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3931
3932         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3933         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3934
3935         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3936         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3937         if ($spice_enhancement->{foldersharing}) {
3938             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3939             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3940         }
3941
3942         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3943         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3944             if $spice_enhancement->{videostreaming};
3945
3946         push @$devices, '-spice', "$spice_opts";
3947     }
3948
3949     # enable balloon by default, unless explicitly disabled
3950     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3951         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3952         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3953         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3954         push @$devices, '-device', $ballooncmd;
3955     }
3956
3957     if ($conf->{watchdog}) {
3958         my $wdopts = parse_watchdog($conf->{watchdog});
3959         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3960         my $watchdog = $wdopts->{model} || 'i6300esb';
3961         push @$devices, '-device', "$watchdog$pciaddr";
3962         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3963     }
3964
3965     my $vollist = [];
3966     my $scsicontroller = {};
3967     my $ahcicontroller = {};
3968     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3969
3970     # Add iscsi initiator name if available
3971     if (my $initiator = get_initiator_name()) {
3972         push @$devices, '-iscsi', "initiator-name=$initiator";
3973     }
3974
3975     PVE::QemuConfig->foreach_volume($conf, sub {
3976         my ($ds, $drive) = @_;
3977
3978         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3979             check_volume_storage_type($storecfg, $drive->{file});
3980             push @$vollist, $drive->{file};
3981         }
3982
3983         # ignore efidisk here, already added in bios/fw handling code above
3984         return if $drive->{interface} eq 'efidisk';
3985         # similar for TPM
3986         return if $drive->{interface} eq 'tpmstate';
3987
3988         $use_virtio = 1 if $ds =~ m/^virtio/;
3989
3990         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3991
3992         if ($drive->{interface} eq 'virtio'){
3993            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3994         }
3995
3996         if ($drive->{interface} eq 'scsi') {
3997
3998             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3999
4000             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
4001                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
4002
4003             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4004             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4005
4006             my $iothread = '';
4007             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4008                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4009                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4010             } elsif ($drive->{iothread}) {
4011                 log_warn(
4012                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4013                 );
4014             }
4015
4016             my $queues = '';
4017             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4018                 $queues = ",num_queues=$drive->{queues}";
4019             }
4020
4021             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4022                 if !$scsicontroller->{$controller};
4023             $scsicontroller->{$controller}=1;
4024         }
4025
4026         if ($drive->{interface} eq 'sata') {
4027             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4028             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4029             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4030                 if !$ahcicontroller->{$controller};
4031             $ahcicontroller->{$controller}=1;
4032         }
4033
4034         my $pbs_conf = $pbs_backing->{$ds};
4035         my $pbs_name = undef;
4036         if ($pbs_conf) {
4037             $pbs_name = "drive-$ds-pbs";
4038             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4039         }
4040
4041         my $drive_cmd = print_drive_commandline_full(
4042             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4043
4044         # extra protection for templates, but SATA and IDE don't support it..
4045         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4046
4047         push @$devices, '-drive',$drive_cmd;
4048         push @$devices, '-device', print_drivedevice_full(
4049             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4050     });
4051
4052     for (my $i = 0; $i < $MAX_NETS; $i++) {
4053         my $netname = "net$i";
4054
4055         next if !$conf->{$netname};
4056         my $d = parse_net($conf->{$netname});
4057         next if !$d;
4058         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4059
4060         $use_virtio = 1 if $d->{model} eq 'virtio';
4061
4062         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4063
4064         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4065         push @$devices, '-netdev', $netdevfull;
4066
4067         my $netdevicefull = print_netdevice_full(
4068             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4069
4070         push @$devices, '-device', $netdevicefull;
4071     }
4072
4073     if ($conf->{ivshmem}) {
4074         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4075
4076         my $bus;
4077         if ($q35) {
4078             $bus = print_pcie_addr("ivshmem");
4079         } else {
4080             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4081         }
4082
4083         my $ivshmem_name = $ivshmem->{name} // $vmid;
4084         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4085
4086         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4087         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4088             .",size=$ivshmem->{size}M";
4089     }
4090
4091     # pci.4 is nested in pci.1
4092     $bridges->{1} = 1 if $bridges->{4};
4093
4094     if (!$q35) { # add pci bridges
4095         if (min_version($machine_version, 2, 3)) {
4096            $bridges->{1} = 1;
4097            $bridges->{2} = 1;
4098         }
4099         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4100     }
4101
4102     for my $k (sort {$b cmp $a} keys %$bridges) {
4103         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4104
4105         my $k_name = $k;
4106         if ($k == 2 && $legacy_igd) {
4107             $k_name = "$k-igd";
4108         }
4109         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4110         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4111
4112         if ($q35) { # add after -readconfig pve-q35.cfg
4113             splice @$devices, 2, 0, '-device', $devstr;
4114         } else {
4115             unshift @$devices, '-device', $devstr if $k > 0;
4116         }
4117     }
4118
4119     if (!$kvm) {
4120         push @$machineFlags, 'accel=tcg';
4121     }
4122
4123     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4124
4125     my $machine_type_min = $machine_type;
4126     if ($add_pve_version) {
4127         $machine_type_min =~ s/\+pve\d+$//;
4128         $machine_type_min .= "+pve$required_pve_version";
4129     }
4130     push @$machineFlags, "type=${machine_type_min}";
4131
4132     push @$cmd, @$devices;
4133     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4134     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4135     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4136
4137     if (my $vmstate = $conf->{vmstate}) {
4138         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4139         push @$vollist, $vmstate;
4140         push @$cmd, '-loadstate', $statepath;
4141         print "activating and using '$vmstate' as vmstate\n";
4142     }
4143
4144     if (PVE::QemuConfig->is_template($conf)) {
4145         # needed to workaround base volumes being read-only
4146         push @$cmd, '-snapshot';
4147     }
4148
4149     # add custom args
4150     if ($conf->{args}) {
4151         my $aa = PVE::Tools::split_args($conf->{args});
4152         push @$cmd, @$aa;
4153     }
4154
4155     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4156 }
4157
4158 sub check_rng_source {
4159     my ($source) = @_;
4160
4161     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4162     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4163         if ! -e $source;
4164
4165     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4166     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4167         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4168         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4169         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4170             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4171             ." to the host.\n";
4172     }
4173 }
4174
4175 sub spice_port {
4176     my ($vmid) = @_;
4177
4178     my $res = mon_cmd($vmid, 'query-spice');
4179
4180     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4181 }
4182
4183 sub vm_devices_list {
4184     my ($vmid) = @_;
4185
4186     my $res = mon_cmd($vmid, 'query-pci');
4187     my $devices_to_check = [];
4188     my $devices = {};
4189     foreach my $pcibus (@$res) {
4190         push @$devices_to_check, @{$pcibus->{devices}},
4191     }
4192
4193     while (@$devices_to_check) {
4194         my $to_check = [];
4195         for my $d (@$devices_to_check) {
4196             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4197             next if !$d->{'pci_bridge'};
4198
4199             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4200             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4201         }
4202         $devices_to_check = $to_check;
4203     }
4204
4205     my $resblock = mon_cmd($vmid, 'query-block');
4206     foreach my $block (@$resblock) {
4207         if($block->{device} =~ m/^drive-(\S+)/){
4208                 $devices->{$1} = 1;
4209         }
4210     }
4211
4212     my $resmice = mon_cmd($vmid, 'query-mice');
4213     foreach my $mice (@$resmice) {
4214         if ($mice->{name} eq 'QEMU HID Tablet') {
4215             $devices->{tablet} = 1;
4216             last;
4217         }
4218     }
4219
4220     # for usb devices there is no query-usb
4221     # but we can iterate over the entries in
4222     # qom-list path=/machine/peripheral
4223     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4224     foreach my $per (@$resperipheral) {
4225         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4226             $devices->{$per->{name}} = 1;
4227         }
4228     }
4229
4230     return $devices;
4231 }
4232
4233 sub vm_deviceplug {
4234     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4235
4236     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4237
4238     my $devices_list = vm_devices_list($vmid);
4239     return 1 if defined($devices_list->{$deviceid});
4240
4241     # add PCI bridge if we need it for the device
4242     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4243
4244     if ($deviceid eq 'tablet') {
4245         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4246     } elsif ($deviceid eq 'keyboard') {
4247         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4248     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4249         my $id = $1;
4250         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4251         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4252     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4253         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4254     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4255         qemu_iothread_add($vmid, $deviceid, $device);
4256
4257         qemu_driveadd($storecfg, $vmid, $device);
4258         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4259
4260         qemu_deviceadd($vmid, $devicefull);
4261         eval { qemu_deviceaddverify($vmid, $deviceid); };
4262         if (my $err = $@) {
4263             eval { qemu_drivedel($vmid, $deviceid); };
4264             warn $@ if $@;
4265             die $err;
4266         }
4267     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4268         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4269         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4270         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4271
4272         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4273
4274         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4275             qemu_iothread_add($vmid, $deviceid, $device);
4276             $devicefull .= ",iothread=iothread-$deviceid";
4277         }
4278
4279         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4280             $devicefull .= ",num_queues=$device->{queues}";
4281         }
4282
4283         qemu_deviceadd($vmid, $devicefull);
4284         qemu_deviceaddverify($vmid, $deviceid);
4285     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4286         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4287         qemu_driveadd($storecfg, $vmid, $device);
4288
4289         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4290         eval { qemu_deviceadd($vmid, $devicefull); };
4291         if (my $err = $@) {
4292             eval { qemu_drivedel($vmid, $deviceid); };
4293             warn $@ if $@;
4294             die $err;
4295         }
4296     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4297         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4298
4299         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4300         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4301         my $use_old_bios_files = undef;
4302         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4303
4304         my $netdevicefull = print_netdevice_full(
4305             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4306         qemu_deviceadd($vmid, $netdevicefull);
4307         eval {
4308             qemu_deviceaddverify($vmid, $deviceid);
4309             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4310         };
4311         if (my $err = $@) {
4312             eval { qemu_netdevdel($vmid, $deviceid); };
4313             warn $@ if $@;
4314             die $err;
4315         }
4316     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4317         my $bridgeid = $2;
4318         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4319         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4320
4321         qemu_deviceadd($vmid, $devicefull);
4322         qemu_deviceaddverify($vmid, $deviceid);
4323     } else {
4324         die "can't hotplug device '$deviceid'\n";
4325     }
4326
4327     return 1;
4328 }
4329
4330 # fixme: this should raise exceptions on error!
4331 sub vm_deviceunplug {
4332     my ($vmid, $conf, $deviceid) = @_;
4333
4334     my $devices_list = vm_devices_list($vmid);
4335     return 1 if !defined($devices_list->{$deviceid});
4336
4337     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4338     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4339
4340     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4341         qemu_devicedel($vmid, $deviceid);
4342     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4343         qemu_devicedel($vmid, $deviceid);
4344         qemu_devicedelverify($vmid, $deviceid);
4345     } elsif ($deviceid =~ m/^usb\d+$/) {
4346         qemu_devicedel($vmid, $deviceid);
4347         qemu_devicedelverify($vmid, $deviceid);
4348     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4349         my $device = parse_drive($deviceid, $conf->{$deviceid});
4350
4351         qemu_devicedel($vmid, $deviceid);
4352         qemu_devicedelverify($vmid, $deviceid);
4353         qemu_drivedel($vmid, $deviceid);
4354         qemu_iothread_del($vmid, $deviceid, $device);
4355     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4356         qemu_devicedel($vmid, $deviceid);
4357         qemu_devicedelverify($vmid, $deviceid);
4358     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4359         my $device = parse_drive($deviceid, $conf->{$deviceid});
4360
4361         qemu_devicedel($vmid, $deviceid);
4362         qemu_devicedelverify($vmid, $deviceid);
4363         qemu_drivedel($vmid, $deviceid);
4364         qemu_deletescsihw($conf, $vmid, $deviceid);
4365
4366         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4367             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4368     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4369         qemu_devicedel($vmid, $deviceid);
4370         qemu_devicedelverify($vmid, $deviceid);
4371         qemu_netdevdel($vmid, $deviceid);
4372     } else {
4373         die "can't unplug device '$deviceid'\n";
4374     }
4375
4376     return 1;
4377 }
4378
4379 sub qemu_spice_usbredir_chardev_add {
4380     my ($vmid, $id) = @_;
4381
4382     mon_cmd($vmid, "chardev-add" , (
4383         id => $id,
4384         backend => {
4385             type => 'spicevmc',
4386             data => {
4387                 type => "usbredir",
4388             },
4389         },
4390     ));
4391 }
4392
4393 sub qemu_deviceadd {
4394     my ($vmid, $devicefull) = @_;
4395
4396     $devicefull = "driver=".$devicefull;
4397     my %options =  split(/[=,]/, $devicefull);
4398
4399     mon_cmd($vmid, "device_add" , %options);
4400 }
4401
4402 sub qemu_devicedel {
4403     my ($vmid, $deviceid) = @_;
4404
4405     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4406 }
4407
4408 sub qemu_iothread_add {
4409     my ($vmid, $deviceid, $device) = @_;
4410
4411     if ($device->{iothread}) {
4412         my $iothreads = vm_iothreads_list($vmid);
4413         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4414     }
4415 }
4416
4417 sub qemu_iothread_del {
4418     my ($vmid, $deviceid, $device) = @_;
4419
4420     if ($device->{iothread}) {
4421         my $iothreads = vm_iothreads_list($vmid);
4422         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4423     }
4424 }
4425
4426 sub qemu_objectadd {
4427     my ($vmid, $objectid, $qomtype) = @_;
4428
4429     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4430
4431     return 1;
4432 }
4433
4434 sub qemu_objectdel {
4435     my ($vmid, $objectid) = @_;
4436
4437     mon_cmd($vmid, "object-del", id => $objectid);
4438
4439     return 1;
4440 }
4441
4442 sub qemu_driveadd {
4443     my ($storecfg, $vmid, $device) = @_;
4444
4445     my $kvmver = get_running_qemu_version($vmid);
4446     my $io_uring = min_version($kvmver, 6, 0);
4447     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4448     $drive =~ s/\\/\\\\/g;
4449     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4450
4451     # If the command succeeds qemu prints: "OK"
4452     return 1 if $ret =~ m/OK/s;
4453
4454     die "adding drive failed: $ret\n";
4455 }
4456
4457 sub qemu_drivedel {
4458     my ($vmid, $deviceid) = @_;
4459
4460     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4461     $ret =~ s/^\s+//;
4462
4463     return 1 if $ret eq "";
4464
4465     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4466     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4467
4468     die "deleting drive $deviceid failed : $ret\n";
4469 }
4470
4471 sub qemu_deviceaddverify {
4472     my ($vmid, $deviceid) = @_;
4473
4474     for (my $i = 0; $i <= 5; $i++) {
4475          my $devices_list = vm_devices_list($vmid);
4476          return 1 if defined($devices_list->{$deviceid});
4477          sleep 1;
4478     }
4479
4480     die "error on hotplug device '$deviceid'\n";
4481 }
4482
4483
4484 sub qemu_devicedelverify {
4485     my ($vmid, $deviceid) = @_;
4486
4487     # need to verify that the device is correctly removed as device_del
4488     # is async and empty return is not reliable
4489
4490     for (my $i = 0; $i <= 5; $i++) {
4491          my $devices_list = vm_devices_list($vmid);
4492          return 1 if !defined($devices_list->{$deviceid});
4493          sleep 1;
4494     }
4495
4496     die "error on hot-unplugging device '$deviceid'\n";
4497 }
4498
4499 sub qemu_findorcreatescsihw {
4500     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4501
4502     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4503
4504     my $scsihwid="$controller_prefix$controller";
4505     my $devices_list = vm_devices_list($vmid);
4506
4507     if (!defined($devices_list->{$scsihwid})) {
4508         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4509     }
4510
4511     return 1;
4512 }
4513
4514 sub qemu_deletescsihw {
4515     my ($conf, $vmid, $opt) = @_;
4516
4517     my $device = parse_drive($opt, $conf->{$opt});
4518
4519     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4520         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4521         return 1;
4522     }
4523
4524     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4525
4526     my $devices_list = vm_devices_list($vmid);
4527     foreach my $opt (keys %{$devices_list}) {
4528         if (is_valid_drivename($opt)) {
4529             my $drive = parse_drive($opt, $conf->{$opt});
4530             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4531                 return 1;
4532             }
4533         }
4534     }
4535
4536     my $scsihwid="scsihw$controller";
4537
4538     vm_deviceunplug($vmid, $conf, $scsihwid);
4539
4540     return 1;
4541 }
4542
4543 sub qemu_add_pci_bridge {
4544     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4545
4546     my $bridges = {};
4547
4548     my $bridgeid;
4549
4550     print_pci_addr($device, $bridges, $arch, $machine_type);
4551
4552     while (my ($k, $v) = each %$bridges) {
4553         $bridgeid = $k;
4554     }
4555     return 1 if !defined($bridgeid) || $bridgeid < 1;
4556
4557     my $bridge = "pci.$bridgeid";
4558     my $devices_list = vm_devices_list($vmid);
4559
4560     if (!defined($devices_list->{$bridge})) {
4561         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4562     }
4563
4564     return 1;
4565 }
4566
4567 sub qemu_set_link_status {
4568     my ($vmid, $device, $up) = @_;
4569
4570     mon_cmd($vmid, "set_link", name => $device,
4571                up => $up ? JSON::true : JSON::false);
4572 }
4573
4574 sub qemu_netdevadd {
4575     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4576
4577     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4578     my %options =  split(/[=,]/, $netdev);
4579
4580     if (defined(my $vhost = $options{vhost})) {
4581         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4582     }
4583
4584     if (defined(my $queues = $options{queues})) {
4585         $options{queues} = $queues + 0;
4586     }
4587
4588     mon_cmd($vmid, "netdev_add",  %options);
4589     return 1;
4590 }
4591
4592 sub qemu_netdevdel {
4593     my ($vmid, $deviceid) = @_;
4594
4595     mon_cmd($vmid, "netdev_del", id => $deviceid);
4596 }
4597
4598 sub qemu_usb_hotplug {
4599     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4600
4601     return if !$device;
4602
4603     # remove the old one first
4604     vm_deviceunplug($vmid, $conf, $deviceid);
4605
4606     # check if xhci controller is necessary and available
4607     my $devicelist = vm_devices_list($vmid);
4608
4609     if (!$devicelist->{xhci}) {
4610         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4611         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4612     }
4613
4614     # print_usbdevice_full expects the parsed device
4615     my $d = parse_usb_device($device->{host});
4616     $d->{usb3} = $device->{usb3};
4617
4618     # add the new one
4619     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4620 }
4621
4622 sub qemu_cpu_hotplug {
4623     my ($vmid, $conf, $vcpus) = @_;
4624
4625     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4626
4627     my $sockets = 1;
4628     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4629     $sockets = $conf->{sockets} if  $conf->{sockets};
4630     my $cores = $conf->{cores} || 1;
4631     my $maxcpus = $sockets * $cores;
4632
4633     $vcpus = $maxcpus if !$vcpus;
4634
4635     die "you can't add more vcpus than maxcpus\n"
4636         if $vcpus > $maxcpus;
4637
4638     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4639
4640     if ($vcpus < $currentvcpus) {
4641
4642         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4643
4644             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4645                 qemu_devicedel($vmid, "cpu$i");
4646                 my $retry = 0;
4647                 my $currentrunningvcpus = undef;
4648                 while (1) {
4649                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4650                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4651                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4652                     $retry++;
4653                     sleep 1;
4654                 }
4655                 #update conf after each succesfull cpu unplug
4656                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4657                 PVE::QemuConfig->write_config($vmid, $conf);
4658             }
4659         } else {
4660             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4661         }
4662
4663         return;
4664     }
4665
4666     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4667     die "vcpus in running vm does not match its configuration\n"
4668         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4669
4670     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4671
4672         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4673             my $cpustr = print_cpu_device($conf, $i);
4674             qemu_deviceadd($vmid, $cpustr);
4675
4676             my $retry = 0;
4677             my $currentrunningvcpus = undef;
4678             while (1) {
4679                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4680                 last if scalar(@{$currentrunningvcpus}) == $i;
4681                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4682                 sleep 1;
4683                 $retry++;
4684             }
4685             #update conf after each succesfull cpu hotplug
4686             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4687             PVE::QemuConfig->write_config($vmid, $conf);
4688         }
4689     } else {
4690
4691         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4692             mon_cmd($vmid, "cpu-add", id => int($i));
4693         }
4694     }
4695 }
4696
4697 sub qemu_block_set_io_throttle {
4698     my ($vmid, $deviceid,
4699         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4700         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4701         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4702         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4703
4704     return if !check_running($vmid) ;
4705
4706     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4707         bps => int($bps),
4708         bps_rd => int($bps_rd),
4709         bps_wr => int($bps_wr),
4710         iops => int($iops),
4711         iops_rd => int($iops_rd),
4712         iops_wr => int($iops_wr),
4713         bps_max => int($bps_max),
4714         bps_rd_max => int($bps_rd_max),
4715         bps_wr_max => int($bps_wr_max),
4716         iops_max => int($iops_max),
4717         iops_rd_max => int($iops_rd_max),
4718         iops_wr_max => int($iops_wr_max),
4719         bps_max_length => int($bps_max_length),
4720         bps_rd_max_length => int($bps_rd_max_length),
4721         bps_wr_max_length => int($bps_wr_max_length),
4722         iops_max_length => int($iops_max_length),
4723         iops_rd_max_length => int($iops_rd_max_length),
4724         iops_wr_max_length => int($iops_wr_max_length),
4725     );
4726
4727 }
4728
4729 sub qemu_block_resize {
4730     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4731
4732     my $running = check_running($vmid);
4733
4734     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4735
4736     return if !$running;
4737
4738     my $padding = (1024 - $size % 1024) % 1024;
4739     $size = $size + $padding;
4740
4741     mon_cmd(
4742         $vmid,
4743         "block_resize",
4744         device => $deviceid,
4745         size => int($size),
4746         timeout => 60,
4747     );
4748 }
4749
4750 sub qemu_volume_snapshot {
4751     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4752
4753     my $running = check_running($vmid);
4754
4755     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4756         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4757     } else {
4758         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4759     }
4760 }
4761
4762 sub qemu_volume_snapshot_delete {
4763     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4764
4765     my $running = check_running($vmid);
4766
4767     if($running) {
4768
4769         $running = undef;
4770         my $conf = PVE::QemuConfig->load_config($vmid);
4771         PVE::QemuConfig->foreach_volume($conf, sub {
4772             my ($ds, $drive) = @_;
4773             $running = 1 if $drive->{file} eq $volid;
4774         });
4775     }
4776
4777     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4778         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4779     } else {
4780         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4781     }
4782 }
4783
4784 sub set_migration_caps {
4785     my ($vmid, $savevm) = @_;
4786
4787     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4788
4789     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4790     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4791
4792     my $cap_ref = [];
4793
4794     my $enabled_cap = {
4795         "auto-converge" => 1,
4796         "xbzrle" => 1,
4797         "x-rdma-pin-all" => 0,
4798         "zero-blocks" => 0,
4799         "compress" => 0,
4800         "dirty-bitmaps" => $dirty_bitmaps,
4801     };
4802
4803     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4804
4805     for my $supported_capability (@$supported_capabilities) {
4806         push @$cap_ref, {
4807             capability => $supported_capability->{capability},
4808             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4809         };
4810     }
4811
4812     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4813 }
4814
4815 sub foreach_volid {
4816     my ($conf, $func, @param) = @_;
4817
4818     my $volhash = {};
4819
4820     my $test_volid = sub {
4821         my ($key, $drive, $snapname) = @_;
4822
4823         my $volid = $drive->{file};
4824         return if !$volid;
4825
4826         $volhash->{$volid}->{cdrom} //= 1;
4827         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4828
4829         my $replicate = $drive->{replicate} // 1;
4830         $volhash->{$volid}->{replicate} //= 0;
4831         $volhash->{$volid}->{replicate} = 1 if $replicate;
4832
4833         $volhash->{$volid}->{shared} //= 0;
4834         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4835
4836         $volhash->{$volid}->{referenced_in_config} //= 0;
4837         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4838
4839         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4840             if defined($snapname);
4841
4842         my $size = $drive->{size};
4843         $volhash->{$volid}->{size} //= $size if $size;
4844
4845         $volhash->{$volid}->{is_vmstate} //= 0;
4846         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4847
4848         $volhash->{$volid}->{is_tpmstate} //= 0;
4849         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4850
4851         $volhash->{$volid}->{is_unused} //= 0;
4852         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4853
4854         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4855     };
4856
4857     my $include_opts = {
4858         extra_keys => ['vmstate'],
4859         include_unused => 1,
4860     };
4861
4862     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4863     foreach my $snapname (keys %{$conf->{snapshots}}) {
4864         my $snap = $conf->{snapshots}->{$snapname};
4865         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4866     }
4867
4868     foreach my $volid (keys %$volhash) {
4869         &$func($volid, $volhash->{$volid}, @param);
4870     }
4871 }
4872
4873 my $fast_plug_option = {
4874     'lock' => 1,
4875     'name' => 1,
4876     'onboot' => 1,
4877     'shares' => 1,
4878     'startup' => 1,
4879     'description' => 1,
4880     'protection' => 1,
4881     'vmstatestorage' => 1,
4882     'hookscript' => 1,
4883     'tags' => 1,
4884 };
4885
4886 for my $opt (keys %$confdesc_cloudinit) {
4887     $fast_plug_option->{$opt} = 1;
4888 };
4889
4890 # hotplug changes in [PENDING]
4891 # $selection hash can be used to only apply specified options, for
4892 # example: { cores => 1 } (only apply changed 'cores')
4893 # $errors ref is used to return error messages
4894 sub vmconfig_hotplug_pending {
4895     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4896
4897     my $defaults = load_defaults();
4898     my $arch = get_vm_arch($conf);
4899     my $machine_type = get_vm_machine($conf, undef, $arch);
4900
4901     # commit values which do not have any impact on running VM first
4902     # Note: those option cannot raise errors, we we do not care about
4903     # $selection and always apply them.
4904
4905     my $add_error = sub {
4906         my ($opt, $msg) = @_;
4907         $errors->{$opt} = "hotplug problem - $msg";
4908     };
4909
4910     my $cloudinit_pending_properties = PVE::QemuServer::cloudinit_pending_properties();
4911
4912     my $cloudinit_record_changed = sub {
4913         my ($conf, $opt, $old, $new) = @_;
4914         return if !$cloudinit_pending_properties->{$opt};
4915
4916         my $ci = ($conf->{cloudinit} //= {});
4917
4918         my $recorded = $ci->{$opt};
4919         my %added = map { $_ => 1 } PVE::Tools::split_list(delete($ci->{added}) // '');
4920
4921         if (defined($new)) {
4922             if (defined($old)) {
4923                 # an existing value is being modified
4924                 if (defined($recorded)) {
4925                     # the value was already not in sync
4926                     if ($new eq $recorded) {
4927                         # a value is being reverted to the cloud-init state:
4928                         delete $ci->{$opt};
4929                         delete $added{$opt};
4930                     } else {
4931                         # the value was changed multiple times, do nothing
4932                     }
4933                 } elsif ($added{$opt}) {
4934                     # the value had been marked as added and is being changed, do nothing
4935                 } else {
4936                     # the value is new, record it:
4937                     $ci->{$opt} = $old;
4938                 }
4939             } else {
4940                 # a new value is being added
4941                 if (defined($recorded)) {
4942                     # it was already not in sync
4943                     if ($new eq $recorded) {
4944                         # a value is being reverted to the cloud-init state:
4945                         delete $ci->{$opt};
4946                         delete $added{$opt};
4947                     } else {
4948                         # the value had temporarily been removed, do nothing
4949                     }
4950                 } elsif ($added{$opt}) {
4951                     # the value had been marked as added already, do nothing
4952                 } else {
4953                     # the value is new, add it
4954                     $added{$opt} = 1;
4955                 }
4956             }
4957         } elsif (!defined($old)) {
4958             # a non-existent value is being removed? ignore...
4959         } else {
4960             # a value is being deleted
4961             if (defined($recorded)) {
4962                 # a value was already recorded, just keep it
4963             } elsif ($added{$opt}) {
4964                 # the value was marked as added, remove it
4965                 delete $added{$opt};
4966             } else {
4967                 # a previously unrecorded value is being removed, record the old value:
4968                 $ci->{$opt} = $old;
4969             }
4970         }
4971
4972         my $added = join(',', sort keys %added);
4973         $ci->{added} = $added if length($added);
4974     };
4975
4976     my $changes = 0;
4977     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4978         if ($fast_plug_option->{$opt}) {
4979             my $new = delete $conf->{pending}->{$opt};
4980             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $new);
4981             $conf->{$opt} = $new;
4982             $changes = 1;
4983         }
4984     }
4985
4986     if ($changes) {
4987         PVE::QemuConfig->write_config($vmid, $conf);
4988     }
4989
4990     my $ostype = $conf->{ostype};
4991     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4992     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4993     my $usb_hotplug = $hotplug_features->{usb}
4994         && min_version($version, 7, 1)
4995         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4996
4997     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4998     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4999
5000     foreach my $opt (sort keys %$pending_delete_hash) {
5001         next if $selection && !$selection->{$opt};
5002         my $force = $pending_delete_hash->{$opt}->{force};
5003         eval {
5004             if ($opt eq 'hotplug') {
5005                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
5006             } elsif ($opt eq 'tablet') {
5007                 die "skip\n" if !$hotplug_features->{usb};
5008                 if ($defaults->{tablet}) {
5009                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5010                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5011                         if $arch eq 'aarch64';
5012                 } else {
5013                     vm_deviceunplug($vmid, $conf, 'tablet');
5014                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5015                 }
5016             } elsif ($opt =~ m/^usb(\d+)$/) {
5017                 my $index = $1;
5018                 die "skip\n" if !$usb_hotplug;
5019                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
5020                 vm_deviceunplug($vmid, $conf, $opt);
5021             } elsif ($opt eq 'vcpus') {
5022                 die "skip\n" if !$hotplug_features->{cpu};
5023                 qemu_cpu_hotplug($vmid, $conf, undef);
5024             } elsif ($opt eq 'balloon') {
5025                 # enable balloon device is not hotpluggable
5026                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
5027                 # here we reset the ballooning value to memory
5028                 my $balloon = $conf->{memory} || $defaults->{memory};
5029                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5030             } elsif ($fast_plug_option->{$opt}) {
5031                 # do nothing
5032             } elsif ($opt =~ m/^net(\d+)$/) {
5033                 die "skip\n" if !$hotplug_features->{network};
5034                 vm_deviceunplug($vmid, $conf, $opt);
5035             } elsif (is_valid_drivename($opt)) {
5036                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
5037                 vm_deviceunplug($vmid, $conf, $opt);
5038                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5039             } elsif ($opt =~ m/^memory$/) {
5040                 die "skip\n" if !$hotplug_features->{memory};
5041                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
5042             } elsif ($opt eq 'cpuunits') {
5043                 $cgroup->change_cpu_shares(undef);
5044             } elsif ($opt eq 'cpulimit') {
5045                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
5046             } else {
5047                 die "skip\n";
5048             }
5049         };
5050         if (my $err = $@) {
5051             &$add_error($opt, $err) if $err ne "skip\n";
5052         } else {
5053             my $old = delete $conf->{$opt};
5054             $cloudinit_record_changed->($conf, $opt, $old, undef);
5055             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5056         }
5057     }
5058
5059     my $cloudinit_opt;
5060     foreach my $opt (keys %{$conf->{pending}}) {
5061         next if $selection && !$selection->{$opt};
5062         my $value = $conf->{pending}->{$opt};
5063         eval {
5064             if ($opt eq 'hotplug') {
5065                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5066             } elsif ($opt eq 'tablet') {
5067                 die "skip\n" if !$hotplug_features->{usb};
5068                 if ($value == 1) {
5069                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5070                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5071                         if $arch eq 'aarch64';
5072                 } elsif ($value == 0) {
5073                     vm_deviceunplug($vmid, $conf, 'tablet');
5074                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5075                 }
5076             } elsif ($opt =~ m/^usb(\d+)$/) {
5077                 my $index = $1;
5078                 die "skip\n" if !$usb_hotplug;
5079                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5080                 my $id = $opt;
5081                 if ($d->{host} eq 'spice')  {
5082                     $id = "usbredirdev$index";
5083                 }
5084                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5085             } elsif ($opt eq 'vcpus') {
5086                 die "skip\n" if !$hotplug_features->{cpu};
5087                 qemu_cpu_hotplug($vmid, $conf, $value);
5088             } elsif ($opt eq 'balloon') {
5089                 # enable/disable balloning device is not hotpluggable
5090                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5091                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5092                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5093
5094                 # allow manual ballooning if shares is set to zero
5095                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5096                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5097                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5098                 }
5099             } elsif ($opt =~ m/^net(\d+)$/) {
5100                 # some changes can be done without hotplug
5101                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5102                                     $vmid, $opt, $value, $arch, $machine_type);
5103             } elsif (is_valid_drivename($opt)) {
5104                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5105                 # some changes can be done without hotplug
5106                 my $drive = parse_drive($opt, $value);
5107                 if (drive_is_cloudinit($drive)) {
5108                     $cloudinit_opt = [$opt, $drive];
5109                     # apply all the other changes first, then generate the cloudinit disk
5110                     die "skip\n";
5111                 }
5112                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5113                                      $vmid, $opt, $value, $arch, $machine_type);
5114             } elsif ($opt =~ m/^memory$/) { #dimms
5115                 die "skip\n" if !$hotplug_features->{memory};
5116                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5117             } elsif ($opt eq 'cpuunits') {
5118                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5119                 $cgroup->change_cpu_shares($new_cpuunits);
5120             } elsif ($opt eq 'cpulimit') {
5121                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5122                 $cgroup->change_cpu_quota($cpulimit, 100000);
5123             } elsif ($opt eq 'agent') {
5124                 vmconfig_update_agent($conf, $opt, $value);
5125             } else {
5126                 die "skip\n";  # skip non-hot-pluggable options
5127             }
5128         };
5129         if (my $err = $@) {
5130             &$add_error($opt, $err) if $err ne "skip\n";
5131         } else {
5132             $cloudinit_record_changed->($conf, $opt, $conf->{$opt}, $value);
5133             $conf->{$opt} = $value;
5134             delete $conf->{pending}->{$opt};
5135         }
5136     }
5137
5138     if (defined($cloudinit_opt)) {
5139         my ($opt, $drive) = @$cloudinit_opt;
5140         my $value = $conf->{pending}->{$opt};
5141         eval {
5142             my $temp = {%$conf, $opt => $value};
5143             PVE::QemuServer::Cloudinit::apply_cloudinit_config($temp, $vmid);
5144             vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5145                                  $vmid, $opt, $value, $arch, $machine_type);
5146         };
5147         if (my $err = $@) {
5148             &$add_error($opt, $err) if $err ne "skip\n";
5149         } else {
5150             $conf->{$opt} = $value;
5151             delete $conf->{pending}->{$opt};
5152         }
5153     }
5154
5155     # unplug xhci controller if no usb device is left
5156     if ($usb_hotplug) {
5157         my $has_usb = 0;
5158         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5159             next if !defined($conf->{"usb$i"});
5160             $has_usb = 1;
5161             last;
5162         }
5163         if (!$has_usb) {
5164             vm_deviceunplug($vmid, $conf, 'xhci');
5165         }
5166     }
5167
5168     PVE::QemuConfig->write_config($vmid, $conf);
5169
5170     if ($hotplug_features->{cloudinit} && PVE::QemuServer::Cloudinit::has_changes($conf)) {
5171         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid);
5172     }
5173 }
5174
5175 sub try_deallocate_drive {
5176     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5177
5178     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5179         my $volid = $drive->{file};
5180         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5181             my $sid = PVE::Storage::parse_volume_id($volid);
5182             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5183
5184             # check if the disk is really unused
5185             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5186                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5187             PVE::Storage::vdisk_free($storecfg, $volid);
5188             return 1;
5189         } else {
5190             # If vm is not owner of this disk remove from config
5191             return 1;
5192         }
5193     }
5194
5195     return;
5196 }
5197
5198 sub vmconfig_delete_or_detach_drive {
5199     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5200
5201     my $drive = parse_drive($opt, $conf->{$opt});
5202
5203     my $rpcenv = PVE::RPCEnvironment::get();
5204     my $authuser = $rpcenv->get_user();
5205
5206     if ($force) {
5207         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5208         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5209     } else {
5210         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5211     }
5212 }
5213
5214
5215
5216 sub vmconfig_apply_pending {
5217     my ($vmid, $conf, $storecfg, $errors, $skip_cloud_init) = @_;
5218
5219     return if !scalar(keys %{$conf->{pending}});
5220
5221     my $add_apply_error = sub {
5222         my ($opt, $msg) = @_;
5223         my $err_msg = "unable to apply pending change $opt : $msg";
5224         $errors->{$opt} = $err_msg;
5225         warn $err_msg;
5226     };
5227
5228     # cold plug
5229
5230     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5231     foreach my $opt (sort keys %$pending_delete_hash) {
5232         my $force = $pending_delete_hash->{$opt}->{force};
5233         eval {
5234             if ($opt =~ m/^unused/) {
5235                 die "internal error";
5236             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5237                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5238             }
5239         };
5240         if (my $err = $@) {
5241             $add_apply_error->($opt, $err);
5242         } else {
5243             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5244             delete $conf->{$opt};
5245         }
5246     }
5247
5248     PVE::QemuConfig->cleanup_pending($conf);
5249
5250     my $generate_cloudinit = $skip_cloud_init ? 0 : undef;
5251
5252     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5253         next if $opt eq 'delete'; # just to be sure
5254         eval {
5255             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5256                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5257             }
5258         };
5259         if (my $err = $@) {
5260             $add_apply_error->($opt, $err);
5261         } else {
5262
5263             if (is_valid_drivename($opt)) {
5264                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5265                 $generate_cloudinit //= 1 if drive_is_cloudinit($drive);
5266             }
5267
5268             $conf->{$opt} = delete $conf->{pending}->{$opt};
5269         }
5270     }
5271
5272     # write all changes at once to avoid unnecessary i/o
5273     PVE::QemuConfig->write_config($vmid, $conf);
5274     if ($generate_cloudinit) {
5275         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5276             # After successful generation and if there were changes to be applied, update the
5277             # config to drop the {cloudinit} entry.
5278             PVE::QemuConfig->write_config($vmid, $conf);
5279         }
5280     }
5281 }
5282
5283 sub vmconfig_update_net {
5284     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5285
5286     my $newnet = parse_net($value);
5287
5288     if ($conf->{$opt}) {
5289         my $oldnet = parse_net($conf->{$opt});
5290
5291         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5292             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5293             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5294             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5295
5296             # for non online change, we try to hot-unplug
5297             die "skip\n" if !$hotplug;
5298             vm_deviceunplug($vmid, $conf, $opt);
5299         } else {
5300
5301             die "internal error" if $opt !~ m/net(\d+)/;
5302             my $iface = "tap${vmid}i$1";
5303
5304             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5305                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5306                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5307                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5308                 PVE::Network::tap_unplug($iface);
5309
5310                 if ($have_sdn) {
5311                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5312                 } else {
5313                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5314                 }
5315             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5316                 # Rate can be applied on its own but any change above needs to
5317                 # include the rate in tap_plug since OVS resets everything.
5318                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5319             }
5320
5321             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5322                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5323             }
5324
5325             return 1;
5326         }
5327     }
5328
5329     if ($hotplug) {
5330         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5331     } else {
5332         die "skip\n";
5333     }
5334 }
5335
5336 sub vmconfig_update_agent {
5337     my ($conf, $opt, $value) = @_;
5338
5339     die "skip\n" if !$conf->{$opt};
5340
5341     my $hotplug_options = { fstrim_cloned_disks => 1 };
5342
5343     my $old_agent = parse_guest_agent($conf);
5344     my $agent = parse_guest_agent({$opt => $value});
5345
5346     for my $option (keys %$agent) { # added/changed options
5347         next if defined($hotplug_options->{$option});
5348         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5349     }
5350
5351     for my $option (keys %$old_agent) { # removed options
5352         next if defined($hotplug_options->{$option});
5353         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5354     }
5355
5356     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5357 }
5358
5359 sub vmconfig_update_disk {
5360     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5361
5362     my $drive = parse_drive($opt, $value);
5363
5364     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5365         my $media = $drive->{media} || 'disk';
5366         my $oldmedia = $old_drive->{media} || 'disk';
5367         die "unable to change media type\n" if $media ne $oldmedia;
5368
5369         if (!drive_is_cdrom($old_drive)) {
5370
5371             if ($drive->{file} ne $old_drive->{file}) {
5372
5373                 die "skip\n" if !$hotplug;
5374
5375                 # unplug and register as unused
5376                 vm_deviceunplug($vmid, $conf, $opt);
5377                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5378
5379             } else {
5380                 # update existing disk
5381
5382                 # skip non hotpluggable value
5383                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5384                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5385                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5386                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5387                     safe_string_ne($drive->{ssd}, $old_drive->{ssd}) ||
5388                     safe_string_ne($drive->{ro}, $old_drive->{ro})) {
5389                     die "skip\n";
5390                 }
5391
5392                 # apply throttle
5393                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5394                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5395                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5396                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5397                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5398                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5399                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5400                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5401                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5402                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5403                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5404                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5405                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5406                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5407                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5408                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5409                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5410                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5411
5412                     qemu_block_set_io_throttle(
5413                         $vmid,"drive-$opt",
5414                         ($drive->{mbps} || 0)*1024*1024,
5415                         ($drive->{mbps_rd} || 0)*1024*1024,
5416                         ($drive->{mbps_wr} || 0)*1024*1024,
5417                          $drive->{iops} || 0,
5418                          $drive->{iops_rd} || 0,
5419                          $drive->{iops_wr} || 0,
5420                         ($drive->{mbps_max} || 0)*1024*1024,
5421                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5422                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5423                          $drive->{iops_max} || 0,
5424                          $drive->{iops_rd_max} || 0,
5425                          $drive->{iops_wr_max} || 0,
5426                          $drive->{bps_max_length} || 1,
5427                          $drive->{bps_rd_max_length} || 1,
5428                          $drive->{bps_wr_max_length} || 1,
5429                          $drive->{iops_max_length} || 1,
5430                          $drive->{iops_rd_max_length} || 1,
5431                          $drive->{iops_wr_max_length} || 1,
5432                     );
5433
5434                 }
5435
5436                 return 1;
5437             }
5438
5439         } else { # cdrom
5440
5441             if ($drive->{file} eq 'none') {
5442                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5443                 if (drive_is_cloudinit($old_drive)) {
5444                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5445                 }
5446             } else {
5447                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5448
5449                 # force eject if locked
5450                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5451
5452                 if ($path) {
5453                     mon_cmd($vmid, "blockdev-change-medium",
5454                         id => "$opt", filename => "$path");
5455                 }
5456             }
5457
5458             return 1;
5459         }
5460     }
5461
5462     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5463     # hotplug new disks
5464     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5465     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5466 }
5467
5468 sub vmconfig_update_cloudinit_drive {
5469     my ($storecfg, $conf, $vmid) = @_;
5470
5471     my $cloudinit_ds = undef;
5472     my $cloudinit_drive = undef;
5473
5474     PVE::QemuConfig->foreach_volume($conf, sub {
5475         my ($ds, $drive) = @_;
5476         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5477             $cloudinit_ds = $ds;
5478             $cloudinit_drive = $drive;
5479         }
5480     });
5481
5482     return if !$cloudinit_drive;
5483
5484     if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5485         PVE::QemuConfig->write_config($vmid, $conf);
5486     }
5487
5488     my $running = PVE::QemuServer::check_running($vmid);
5489
5490     if ($running) {
5491         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5492         if ($path) {
5493             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5494             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5495         }
5496     }
5497 }
5498
5499 # called in locked context by incoming migration
5500 sub vm_migrate_get_nbd_disks {
5501     my ($storecfg, $conf, $replicated_volumes) = @_;
5502
5503     my $local_volumes = {};
5504     PVE::QemuConfig->foreach_volume($conf, sub {
5505         my ($ds, $drive) = @_;
5506
5507         return if drive_is_cdrom($drive);
5508         return if $ds eq 'tpmstate0';
5509
5510         my $volid = $drive->{file};
5511
5512         return if !$volid;
5513
5514         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5515
5516         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5517         return if $scfg->{shared};
5518
5519         # replicated disks re-use existing state via bitmap
5520         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5521         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5522     });
5523     return $local_volumes;
5524 }
5525
5526 # called in locked context by incoming migration
5527 sub vm_migrate_alloc_nbd_disks {
5528     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5529
5530     my $nbd = {};
5531     foreach my $opt (sort keys %$source_volumes) {
5532         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5533
5534         if ($use_existing) {
5535             $nbd->{$opt}->{drivestr} = print_drive($drive);
5536             $nbd->{$opt}->{volid} = $volid;
5537             $nbd->{$opt}->{replicated} = 1;
5538             next;
5539         }
5540
5541         # storage mapping + volname = regular migration
5542         # storage mapping + format = remote migration
5543         # order of precedence, filtered by whether storage supports it:
5544         # 1. explicit requested format
5545         # 2. format of current volume
5546         # 3. default format of storage
5547         if (!$storagemap->{identity}) {
5548             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5549             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5550             if (!$format || !grep { $format eq $_ } @$validFormats) {
5551                 if ($volname) {
5552                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5553                     my $fileFormat = qemu_img_format($scfg, $volname);
5554                     $format = $fileFormat
5555                         if grep { $fileFormat eq $_ } @$validFormats;
5556                 }
5557                 $format //= $defFormat;
5558             }
5559         } else {
5560             # can't happen for remote migration, so $volname is always defined
5561             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5562             $format = qemu_img_format($scfg, $volname);
5563         }
5564
5565         my $size = $drive->{size} / 1024;
5566         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5567         my $newdrive = $drive;
5568         $newdrive->{format} = $format;
5569         $newdrive->{file} = $newvolid;
5570         my $drivestr = print_drive($newdrive);
5571         $nbd->{$opt}->{drivestr} = $drivestr;
5572         $nbd->{$opt}->{volid} = $newvolid;
5573     }
5574
5575     return $nbd;
5576 }
5577
5578 # see vm_start_nolock for parameters, additionally:
5579 # migrate_opts:
5580 #   storagemap = parsed storage map for allocating NBD disks
5581 sub vm_start {
5582     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5583
5584     return PVE::QemuConfig->lock_config($vmid, sub {
5585         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5586
5587         die "you can't start a vm if it's a template\n"
5588             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5589
5590         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5591         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5592
5593         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5594
5595         if ($has_backup_lock && $running) {
5596             # a backup is currently running, attempt to start the guest in the
5597             # existing QEMU instance
5598             return vm_resume($vmid);
5599         }
5600
5601         PVE::QemuConfig->check_lock($conf)
5602             if !($params->{skiplock} || $has_suspended_lock);
5603
5604         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5605
5606         die "VM $vmid already running\n" if $running;
5607
5608         if (my $storagemap = $migrate_opts->{storagemap}) {
5609             my $replicated = $migrate_opts->{replicated_volumes};
5610             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5611             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5612
5613             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5614                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5615             }
5616         }
5617
5618         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5619     });
5620 }
5621
5622
5623 # params:
5624 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5625 #   skiplock => 0/1, skip checking for config lock
5626 #   skiptemplate => 0/1, skip checking whether VM is template
5627 #   forcemachine => to force Qemu machine (rollback/migration)
5628 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5629 #   timeout => in seconds
5630 #   paused => start VM in paused state (backup)
5631 #   resume => resume from hibernation
5632 #   pbs-backing => {
5633 #      sata0 => {
5634 #         repository
5635 #         snapshot
5636 #         keyfile
5637 #         archive
5638 #      },
5639 #      virtio2 => ...
5640 #   }
5641 # migrate_opts:
5642 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5643 #   migratedfrom => source node
5644 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5645 #   network => CIDR of migration network
5646 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5647 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5648 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5649 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5650 #       contained in config
5651 sub vm_start_nolock {
5652     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5653
5654     my $statefile = $params->{statefile};
5655     my $resume = $params->{resume};
5656
5657     my $migratedfrom = $migrate_opts->{migratedfrom};
5658     my $migration_type = $migrate_opts->{type};
5659
5660     my $res = {};
5661
5662     # clean up leftover reboot request files
5663     eval { clear_reboot_request($vmid); };
5664     warn $@ if $@;
5665
5666     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5667         vmconfig_apply_pending($vmid, $conf, $storecfg);
5668         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5669     }
5670
5671     # don't regenerate the ISO if the VM is started as part of a live migration
5672     # this way we can reuse the old ISO with the correct config
5673     if (!$migratedfrom) {
5674         if (PVE::QemuServer::Cloudinit::apply_cloudinit_config($conf, $vmid)) {
5675             # FIXME: apply_cloudinit_config updates $conf in this case, and it would only drop
5676             # $conf->{cloudinit}, so we could just not do this?
5677             # But we do it above, so for now let's be consistent.
5678             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5679         }
5680     }
5681
5682     # override offline migrated volumes, conf is out of date still
5683     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5684         for my $key (sort keys $offline_volumes->%*) {
5685             my $parsed = parse_drive($key, $conf->{$key});
5686             $parsed->{file} = $offline_volumes->{$key};
5687             $conf->{$key} = print_drive($parsed);
5688         }
5689     }
5690
5691     my $defaults = load_defaults();
5692
5693     # set environment variable useful inside network script
5694     # for remote migration the config is available on the target node!
5695     if (!$migrate_opts->{remote_node}) {
5696         $ENV{PVE_MIGRATED_FROM} = $migratedfrom;
5697     }
5698
5699     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5700
5701     my $forcemachine = $params->{forcemachine};
5702     my $forcecpu = $params->{forcecpu};
5703     if ($resume) {
5704         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5705         $forcemachine = $conf->{runningmachine};
5706         $forcecpu = $conf->{runningcpu};
5707         print "Resuming suspended VM\n";
5708     }
5709
5710     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5711         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5712
5713     my $migration_ip;
5714     my $get_migration_ip = sub {
5715         my ($nodename) = @_;
5716
5717         return $migration_ip if defined($migration_ip);
5718
5719         my $cidr = $migrate_opts->{network};
5720
5721         if (!defined($cidr)) {
5722             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5723             $cidr = $dc_conf->{migration}->{network};
5724         }
5725
5726         if (defined($cidr)) {
5727             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5728
5729             die "could not get IP: no address configured on local " .
5730                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5731
5732             die "could not get IP: multiple addresses configured on local " .
5733                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5734
5735             $migration_ip = @$ips[0];
5736         }
5737
5738         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5739             if !defined($migration_ip);
5740
5741         return $migration_ip;
5742     };
5743
5744     if ($statefile) {
5745         if ($statefile eq 'tcp') {
5746             my $migrate = $res->{migrate} = { proto => 'tcp' };
5747             $migrate->{addr} = "localhost";
5748             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5749             my $nodename = nodename();
5750
5751             if (!defined($migration_type)) {
5752                 if (defined($datacenterconf->{migration}->{type})) {
5753                     $migration_type = $datacenterconf->{migration}->{type};
5754                 } else {
5755                     $migration_type = 'secure';
5756                 }
5757             }
5758
5759             if ($migration_type eq 'insecure') {
5760                 $migrate->{addr} = $get_migration_ip->($nodename);
5761                 $migrate->{addr} = "[$migrate->{addr}]" if Net::IP::ip_is_ipv6($migrate->{addr});
5762             }
5763
5764             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5765             $migrate->{port} = PVE::Tools::next_migrate_port($pfamily);
5766             $migrate->{uri} = "tcp:$migrate->{addr}:$migrate->{port}";
5767             push @$cmd, '-incoming', $migrate->{uri};
5768             push @$cmd, '-S';
5769
5770         } elsif ($statefile eq 'unix') {
5771             # should be default for secure migrations as a ssh TCP forward
5772             # tunnel is not deterministic reliable ready and fails regurarly
5773             # to set up in time, so use UNIX socket forwards
5774             my $migrate = $res->{migrate} = { proto => 'unix' };
5775             $migrate->{addr} = "/run/qemu-server/$vmid.migrate";
5776             unlink $migrate->{addr};
5777
5778             $migrate->{uri} = "unix:$migrate->{addr}";
5779             push @$cmd, '-incoming', $migrate->{uri};
5780             push @$cmd, '-S';
5781
5782         } elsif (-e $statefile) {
5783             push @$cmd, '-loadstate', $statefile;
5784         } else {
5785             my $statepath = PVE::Storage::path($storecfg, $statefile);
5786             push @$vollist, $statefile;
5787             push @$cmd, '-loadstate', $statepath;
5788         }
5789     } elsif ($params->{paused}) {
5790         push @$cmd, '-S';
5791     }
5792
5793     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5794
5795     my $pci_devices = {}; # host pci devices
5796     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5797         my $dev = $conf->{"hostpci$i"} or next;
5798         $pci_devices->{$i} = parse_hostpci($dev);
5799     }
5800
5801     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5802     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5803
5804     # map to a flat list of pci ids
5805     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5806
5807     # reserve all PCI IDs before actually doing anything with them
5808     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5809
5810     eval {
5811         my $uuid;
5812         for my $id (sort keys %$pci_devices) {
5813             my $d = $pci_devices->{$id};
5814             for my $dev ($d->{pciid}->@*) {
5815                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5816
5817                 # nvidia grid needs the uuid of the mdev as qemu parameter
5818                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5819                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5820                 }
5821             }
5822         }
5823         push @$cmd, '-uuid', $uuid if defined($uuid);
5824     };
5825     if (my $err = $@) {
5826         eval { cleanup_pci_devices($vmid, $conf) };
5827         warn $@ if $@;
5828         die $err;
5829     }
5830
5831     PVE::Storage::activate_volumes($storecfg, $vollist);
5832
5833     eval {
5834         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5835     };
5836     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5837     # timeout should be more than enough here...
5838     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5839
5840     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5841
5842     my %run_params = (
5843         timeout => $statefile ? undef : $start_timeout,
5844         umask => 0077,
5845         noerr => 1,
5846     );
5847
5848     # when migrating, prefix QEMU output so other side can pick up any
5849     # errors that might occur and show the user
5850     if ($migratedfrom) {
5851         $run_params{quiet} = 1;
5852         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5853     }
5854
5855     my %systemd_properties = (
5856         Slice => 'qemu.slice',
5857         KillMode => 'process',
5858         SendSIGKILL => 0,
5859         TimeoutStopUSec => ULONG_MAX, # infinity
5860     );
5861
5862     if (PVE::CGroup::cgroup_mode() == 2) {
5863         $systemd_properties{CPUWeight} = $cpuunits;
5864     } else {
5865         $systemd_properties{CPUShares} = $cpuunits;
5866     }
5867
5868     if (my $cpulimit = $conf->{cpulimit}) {
5869         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5870     }
5871     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5872
5873     my $run_qemu = sub {
5874         PVE::Tools::run_fork sub {
5875             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5876
5877             my $tpmpid;
5878             if (my $tpm = $conf->{tpmstate0}) {
5879                 # start the TPM emulator so QEMU can connect on start
5880                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5881             }
5882
5883             my $exitcode = run_command($cmd, %run_params);
5884             if ($exitcode) {
5885                 if ($tpmpid) {
5886                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5887                     kill 'TERM', $tpmpid;
5888                 }
5889                 die "QEMU exited with code $exitcode\n";
5890             }
5891         };
5892     };
5893
5894     if ($conf->{hugepages}) {
5895
5896         my $code = sub {
5897             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5898             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5899
5900             PVE::QemuServer::Memory::hugepages_mount();
5901             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5902
5903             eval { $run_qemu->() };
5904             if (my $err = $@) {
5905                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5906                     if !$conf->{keephugepages};
5907                 die $err;
5908             }
5909
5910             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5911                 if !$conf->{keephugepages};
5912         };
5913         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5914
5915     } else {
5916         eval { $run_qemu->() };
5917     }
5918
5919     if (my $err = $@) {
5920         # deactivate volumes if start fails
5921         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5922         warn $@ if $@;
5923         eval { cleanup_pci_devices($vmid, $conf) };
5924         warn $@ if $@;
5925
5926         die "start failed: $err";
5927     }
5928
5929     # re-reserve all PCI IDs now that we can know the actual VM PID
5930     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5931     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5932     warn $@ if $@;
5933
5934     if (defined($res->{migrate})) {
5935         print "migration listens on $res->{migrate}->{uri}\n";
5936     } elsif ($statefile) {
5937         eval { mon_cmd($vmid, "cont"); };
5938         warn $@ if $@;
5939     }
5940
5941     #start nbd server for storage migration
5942     if (my $nbd = $migrate_opts->{nbd}) {
5943         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5944
5945         my $migrate_storage_uri;
5946         # nbd_protocol_version > 0 for unix socket support
5947         if ($nbd_protocol_version > 0 && ($migration_type eq 'secure' || $migration_type eq 'websocket')) {
5948             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5949             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5950             $migrate_storage_uri = "nbd:unix:$socket_path";
5951             $res->{migrate}->{unix_sockets} = [$socket_path];
5952         } else {
5953             my $nodename = nodename();
5954             my $localip = $get_migration_ip->($nodename);
5955             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5956             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5957
5958             mon_cmd($vmid, "nbd-server-start", addr => {
5959                 type => 'inet',
5960                 data => {
5961                     host => "${localip}",
5962                     port => "${storage_migrate_port}",
5963                 },
5964             });
5965             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5966             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5967         }
5968
5969         foreach my $opt (sort keys %$nbd) {
5970             my $drivestr = $nbd->{$opt}->{drivestr};
5971             my $volid = $nbd->{$opt}->{volid};
5972             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5973             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5974             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5975             print "re-using replicated volume: $opt - $volid\n"
5976                 if $nbd->{$opt}->{replicated};
5977
5978             $res->{drives}->{$opt} = $nbd->{$opt};
5979             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5980         }
5981     }
5982
5983     if ($migratedfrom) {
5984         eval {
5985             set_migration_caps($vmid);
5986         };
5987         warn $@ if $@;
5988
5989         if ($spice_port) {
5990             print "spice listens on port $spice_port\n";
5991             $res->{spice_port} = $spice_port;
5992             if ($migrate_opts->{spice_ticket}) {
5993                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5994                     $migrate_opts->{spice_ticket});
5995                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5996             }
5997         }
5998
5999     } else {
6000         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
6001             if !$statefile && $conf->{balloon};
6002
6003         foreach my $opt (keys %$conf) {
6004             next if $opt !~  m/^net\d+$/;
6005             my $nicconf = parse_net($conf->{$opt});
6006             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
6007         }
6008         add_nets_bridge_fdb($conf, $vmid);
6009     }
6010
6011     mon_cmd($vmid, 'qom-set',
6012                 path => "machine/peripheral/balloon0",
6013                 property => "guest-stats-polling-interval",
6014                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
6015
6016     if ($resume) {
6017         print "Resumed VM, removing state\n";
6018         if (my $vmstate = $conf->{vmstate}) {
6019             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6020             PVE::Storage::vdisk_free($storecfg, $vmstate);
6021         }
6022         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
6023         PVE::QemuConfig->write_config($vmid, $conf);
6024     }
6025
6026     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
6027
6028     return $res;
6029 }
6030
6031 sub vm_commandline {
6032     my ($storecfg, $vmid, $snapname) = @_;
6033
6034     my $conf = PVE::QemuConfig->load_config($vmid);
6035
6036     my ($forcemachine, $forcecpu);
6037     if ($snapname) {
6038         my $snapshot = $conf->{snapshots}->{$snapname};
6039         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
6040
6041         # check for machine or CPU overrides in snapshot
6042         $forcemachine = $snapshot->{runningmachine};
6043         $forcecpu = $snapshot->{runningcpu};
6044
6045         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
6046
6047         $conf = $snapshot;
6048     }
6049
6050     my $defaults = load_defaults();
6051
6052     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
6053
6054     return PVE::Tools::cmd2string($cmd);
6055 }
6056
6057 sub vm_reset {
6058     my ($vmid, $skiplock) = @_;
6059
6060     PVE::QemuConfig->lock_config($vmid, sub {
6061
6062         my $conf = PVE::QemuConfig->load_config($vmid);
6063
6064         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6065
6066         mon_cmd($vmid, "system_reset");
6067     });
6068 }
6069
6070 sub get_vm_volumes {
6071     my ($conf) = @_;
6072
6073     my $vollist = [];
6074     foreach_volid($conf, sub {
6075         my ($volid, $attr) = @_;
6076
6077         return if $volid =~ m|^/|;
6078
6079         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6080         return if !$sid;
6081
6082         push @$vollist, $volid;
6083     });
6084
6085     return $vollist;
6086 }
6087
6088 sub cleanup_pci_devices {
6089     my ($vmid, $conf) = @_;
6090
6091     foreach my $key (keys %$conf) {
6092         next if $key !~ m/^hostpci(\d+)$/;
6093         my $hostpciindex = $1;
6094         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
6095         my $d = parse_hostpci($conf->{$key});
6096         if ($d->{mdev}) {
6097             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
6098             # don't want to break ABI just for this two liner
6099             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
6100             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
6101         }
6102     }
6103     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
6104 }
6105
6106 sub vm_stop_cleanup {
6107     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
6108
6109     eval {
6110
6111         if (!$keepActive) {
6112             my $vollist = get_vm_volumes($conf);
6113             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6114
6115             if (my $tpmdrive = $conf->{tpmstate0}) {
6116                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6117                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6118                 if ($storeid) {
6119                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6120                 }
6121             }
6122         }
6123
6124         foreach my $ext (qw(mon qmp pid vnc qga)) {
6125             unlink "/var/run/qemu-server/${vmid}.$ext";
6126         }
6127
6128         if ($conf->{ivshmem}) {
6129             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6130             # just delete it for now, VMs which have this already open do not
6131             # are affected, but new VMs will get a separated one. If this
6132             # becomes an issue we either add some sort of ref-counting or just
6133             # add a "don't delete on stop" flag to the ivshmem format.
6134             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6135         }
6136
6137         cleanup_pci_devices($vmid, $conf);
6138
6139         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6140     };
6141     warn $@ if $@; # avoid errors - just warn
6142 }
6143
6144 # call only in locked context
6145 sub _do_vm_stop {
6146     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6147
6148     my $pid = check_running($vmid, $nocheck);
6149     return if !$pid;
6150
6151     my $conf;
6152     if (!$nocheck) {
6153         $conf = PVE::QemuConfig->load_config($vmid);
6154         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6155         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6156             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6157             $timeout = $opts->{down} if $opts->{down};
6158         }
6159         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6160     }
6161
6162     eval {
6163         if ($shutdown) {
6164             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6165                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6166             } else {
6167                 mon_cmd($vmid, "system_powerdown");
6168             }
6169         } else {
6170             mon_cmd($vmid, "quit");
6171         }
6172     };
6173     my $err = $@;
6174
6175     if (!$err) {
6176         $timeout = 60 if !defined($timeout);
6177
6178         my $count = 0;
6179         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6180             $count++;
6181             sleep 1;
6182         }
6183
6184         if ($count >= $timeout) {
6185             if ($force) {
6186                 warn "VM still running - terminating now with SIGTERM\n";
6187                 kill 15, $pid;
6188             } else {
6189                 die "VM quit/powerdown failed - got timeout\n";
6190             }
6191         } else {
6192             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6193             return;
6194         }
6195     } else {
6196        if (!check_running($vmid, $nocheck)) {
6197             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6198             return;
6199        }
6200        if ($force) {
6201             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6202             kill 15, $pid;
6203         } else {
6204             die "VM quit/powerdown failed\n";
6205         }
6206     }
6207
6208     # wait again
6209     $timeout = 10;
6210
6211     my $count = 0;
6212     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6213         $count++;
6214         sleep 1;
6215     }
6216
6217     if ($count >= $timeout) {
6218         warn "VM still running - terminating now with SIGKILL\n";
6219         kill 9, $pid;
6220         sleep 1;
6221     }
6222
6223     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6224 }
6225
6226 # Note: use $nocheck to skip tests if VM configuration file exists.
6227 # We need that when migration VMs to other nodes (files already moved)
6228 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6229 sub vm_stop {
6230     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6231
6232     $force = 1 if !defined($force) && !$shutdown;
6233
6234     if ($migratedfrom){
6235         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6236         kill 15, $pid if $pid;
6237         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6238         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6239         return;
6240     }
6241
6242     PVE::QemuConfig->lock_config($vmid, sub {
6243         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6244    });
6245 }
6246
6247 sub vm_reboot {
6248     my ($vmid, $timeout) = @_;
6249
6250     PVE::QemuConfig->lock_config($vmid, sub {
6251         eval {
6252
6253             # only reboot if running, as qmeventd starts it again on a stop event
6254             return if !check_running($vmid);
6255
6256             create_reboot_request($vmid);
6257
6258             my $storecfg = PVE::Storage::config();
6259             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6260
6261         };
6262         if (my $err = $@) {
6263             # avoid that the next normal shutdown will be confused for a reboot
6264             clear_reboot_request($vmid);
6265             die $err;
6266         }
6267    });
6268 }
6269
6270 # note: if using the statestorage parameter, the caller has to check privileges
6271 sub vm_suspend {
6272     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6273
6274     my $conf;
6275     my $path;
6276     my $storecfg;
6277     my $vmstate;
6278
6279     PVE::QemuConfig->lock_config($vmid, sub {
6280
6281         $conf = PVE::QemuConfig->load_config($vmid);
6282
6283         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6284         PVE::QemuConfig->check_lock($conf)
6285             if !($skiplock || $is_backing_up);
6286
6287         die "cannot suspend to disk during backup\n"
6288             if $is_backing_up && $includestate;
6289
6290         if ($includestate) {
6291             $conf->{lock} = 'suspending';
6292             my $date = strftime("%Y-%m-%d", localtime(time()));
6293             $storecfg = PVE::Storage::config();
6294             if (!$statestorage) {
6295                 $statestorage = find_vmstate_storage($conf, $storecfg);
6296                 # check permissions for the storage
6297                 my $rpcenv = PVE::RPCEnvironment::get();
6298                 if ($rpcenv->{type} ne 'cli') {
6299                     my $authuser = $rpcenv->get_user();
6300                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6301                 }
6302             }
6303
6304
6305             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6306                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6307             $path = PVE::Storage::path($storecfg, $vmstate);
6308             PVE::QemuConfig->write_config($vmid, $conf);
6309         } else {
6310             mon_cmd($vmid, "stop");
6311         }
6312     });
6313
6314     if ($includestate) {
6315         # save vm state
6316         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6317
6318         eval {
6319             set_migration_caps($vmid, 1);
6320             mon_cmd($vmid, "savevm-start", statefile => $path);
6321             for(;;) {
6322                 my $state = mon_cmd($vmid, "query-savevm");
6323                 if (!$state->{status}) {
6324                     die "savevm not active\n";
6325                 } elsif ($state->{status} eq 'active') {
6326                     sleep(1);
6327                     next;
6328                 } elsif ($state->{status} eq 'completed') {
6329                     print "State saved, quitting\n";
6330                     last;
6331                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6332                     die "query-savevm failed with error '$state->{error}'\n"
6333                 } else {
6334                     die "query-savevm returned status '$state->{status}'\n";
6335                 }
6336             }
6337         };
6338         my $err = $@;
6339
6340         PVE::QemuConfig->lock_config($vmid, sub {
6341             $conf = PVE::QemuConfig->load_config($vmid);
6342             if ($err) {
6343                 # cleanup, but leave suspending lock, to indicate something went wrong
6344                 eval {
6345                     mon_cmd($vmid, "savevm-end");
6346                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6347                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6348                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6349                     PVE::QemuConfig->write_config($vmid, $conf);
6350                 };
6351                 warn $@ if $@;
6352                 die $err;
6353             }
6354
6355             die "lock changed unexpectedly\n"
6356                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6357
6358             mon_cmd($vmid, "quit");
6359             $conf->{lock} = 'suspended';
6360             PVE::QemuConfig->write_config($vmid, $conf);
6361         });
6362     }
6363 }
6364
6365 # $nocheck is set when called as part of a migration - in this context the
6366 # location of the config file (source or target node) is not deterministic,
6367 # since migration cannot wait for pmxcfs to process the rename
6368 sub vm_resume {
6369     my ($vmid, $skiplock, $nocheck) = @_;
6370
6371     PVE::QemuConfig->lock_config($vmid, sub {
6372         my $res = mon_cmd($vmid, 'query-status');
6373         my $resume_cmd = 'cont';
6374         my $reset = 0;
6375         my $conf;
6376         if ($nocheck) {
6377             $conf = eval { PVE::QemuConfig->load_config($vmid) }; # try on target node
6378             if ($@) {
6379                 my $vmlist = PVE::Cluster::get_vmlist();
6380                 if (exists($vmlist->{ids}->{$vmid})) {
6381                     my $node = $vmlist->{ids}->{$vmid}->{node};
6382                     $conf = eval { PVE::QemuConfig->load_config($vmid, $node) }; # try on source node
6383                 }
6384                 if (!$conf) {
6385                     PVE::Cluster::cfs_update(); # vmlist was wrong, invalidate cache
6386                     $conf = PVE::QemuConfig->load_config($vmid); # last try on target node again
6387                 }
6388             }
6389         } else {
6390             $conf = PVE::QemuConfig->load_config($vmid);
6391         }
6392
6393         if ($res->{status}) {
6394             return if $res->{status} eq 'running'; # job done, go home
6395             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6396             $reset = 1 if $res->{status} eq 'shutdown';
6397         }
6398
6399         if (!$nocheck) {
6400             PVE::QemuConfig->check_lock($conf)
6401                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6402         }
6403
6404         if ($reset) {
6405             # required if a VM shuts down during a backup and we get a resume
6406             # request before the backup finishes for example
6407             mon_cmd($vmid, "system_reset");
6408         }
6409
6410         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6411
6412         mon_cmd($vmid, $resume_cmd);
6413     });
6414 }
6415
6416 sub vm_sendkey {
6417     my ($vmid, $skiplock, $key) = @_;
6418
6419     PVE::QemuConfig->lock_config($vmid, sub {
6420
6421         my $conf = PVE::QemuConfig->load_config($vmid);
6422
6423         # there is no qmp command, so we use the human monitor command
6424         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6425         die $res if $res ne '';
6426     });
6427 }
6428
6429 # vzdump restore implementaion
6430
6431 sub tar_archive_read_firstfile {
6432     my $archive = shift;
6433
6434     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6435
6436     # try to detect archive type first
6437     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6438         die "unable to open file '$archive'\n";
6439     my $firstfile = <$fh>;
6440     kill 15, $pid;
6441     close $fh;
6442
6443     die "ERROR: archive contaions no data\n" if !$firstfile;
6444     chomp $firstfile;
6445
6446     return $firstfile;
6447 }
6448
6449 sub tar_restore_cleanup {
6450     my ($storecfg, $statfile) = @_;
6451
6452     print STDERR "starting cleanup\n";
6453
6454     if (my $fd = IO::File->new($statfile, "r")) {
6455         while (defined(my $line = <$fd>)) {
6456             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6457                 my $volid = $2;
6458                 eval {
6459                     if ($volid =~ m|^/|) {
6460                         unlink $volid || die 'unlink failed\n';
6461                     } else {
6462                         PVE::Storage::vdisk_free($storecfg, $volid);
6463                     }
6464                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6465                 };
6466                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6467             } else {
6468                 print STDERR "unable to parse line in statfile - $line";
6469             }
6470         }
6471         $fd->close();
6472     }
6473 }
6474
6475 sub restore_file_archive {
6476     my ($archive, $vmid, $user, $opts) = @_;
6477
6478     return restore_vma_archive($archive, $vmid, $user, $opts)
6479         if $archive eq '-';
6480
6481     my $info = PVE::Storage::archive_info($archive);
6482     my $format = $opts->{format} // $info->{format};
6483     my $comp = $info->{compression};
6484
6485     # try to detect archive format
6486     if ($format eq 'tar') {
6487         return restore_tar_archive($archive, $vmid, $user, $opts);
6488     } else {
6489         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6490     }
6491 }
6492
6493 # hepler to remove disks that will not be used after restore
6494 my $restore_cleanup_oldconf = sub {
6495     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6496
6497     my $kept_disks = {};
6498
6499     PVE::QemuConfig->foreach_volume($oldconf, sub {
6500         my ($ds, $drive) = @_;
6501
6502         return if drive_is_cdrom($drive, 1);
6503
6504         my $volid = $drive->{file};
6505         return if !$volid || $volid =~ m|^/|;
6506
6507         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6508         return if !$path || !$owner || ($owner != $vmid);
6509
6510         # Note: only delete disk we want to restore
6511         # other volumes will become unused
6512         if ($virtdev_hash->{$ds}) {
6513             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6514             if (my $err = $@) {
6515                 warn $err;
6516             }
6517         } else {
6518             $kept_disks->{$volid} = 1;
6519         }
6520     });
6521
6522     # after the restore we have no snapshots anymore
6523     for my $snapname (keys $oldconf->{snapshots}->%*) {
6524         my $snap = $oldconf->{snapshots}->{$snapname};
6525         if ($snap->{vmstate}) {
6526             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6527             if (my $err = $@) {
6528                 warn $err;
6529             }
6530         }
6531
6532         for my $volid (keys $kept_disks->%*) {
6533             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6534             warn $@ if $@;
6535         }
6536     }
6537 };
6538
6539 # Helper to parse vzdump backup device hints
6540 #
6541 # $rpcenv: Environment, used to ckeck storage permissions
6542 # $user: User ID, to check storage permissions
6543 # $storecfg: Storage configuration
6544 # $fh: the file handle for reading the configuration
6545 # $devinfo: should contain device sizes for all backu-up'ed devices
6546 # $options: backup options (pool, default storage)
6547 #
6548 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6549 my $parse_backup_hints = sub {
6550     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6551
6552     my $check_storage = sub { # assert if an image can be allocate
6553         my ($storeid, $scfg) = @_;
6554         die "Content type 'images' is not available on storage '$storeid'\n"
6555             if !$scfg->{content}->{images};
6556         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6557             if $user ne 'root@pam';
6558     };
6559
6560     my $virtdev_hash = {};
6561     while (defined(my $line = <$fh>)) {
6562         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6563             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6564             die "archive does not contain data for drive '$virtdev'\n"
6565                 if !$devinfo->{$devname};
6566
6567             if (defined($options->{storage})) {
6568                 $storeid = $options->{storage} || 'local';
6569             } elsif (!$storeid) {
6570                 $storeid = 'local';
6571             }
6572             $format = 'raw' if !$format;
6573             $devinfo->{$devname}->{devname} = $devname;
6574             $devinfo->{$devname}->{virtdev} = $virtdev;
6575             $devinfo->{$devname}->{format} = $format;
6576             $devinfo->{$devname}->{storeid} = $storeid;
6577
6578             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6579             $check_storage->($storeid, $scfg); # permission and content type check
6580
6581             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6582         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6583             my $virtdev = $1;
6584             my $drive = parse_drive($virtdev, $2);
6585
6586             if (drive_is_cloudinit($drive)) {
6587                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6588                 $storeid = $options->{storage} if defined ($options->{storage});
6589                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6590                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6591
6592                 $check_storage->($storeid, $scfg); # permission and content type check
6593
6594                 $virtdev_hash->{$virtdev} = {
6595                     format => $format,
6596                     storeid => $storeid,
6597                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6598                     is_cloudinit => 1,
6599                 };
6600             }
6601         }
6602     }
6603
6604     return $virtdev_hash;
6605 };
6606
6607 # Helper to allocate and activate all volumes required for a restore
6608 #
6609 # $storecfg: Storage configuration
6610 # $virtdev_hash: as returned by parse_backup_hints()
6611 #
6612 # Returns: { $virtdev => $volid }
6613 my $restore_allocate_devices = sub {
6614     my ($storecfg, $virtdev_hash, $vmid) = @_;
6615
6616     my $map = {};
6617     foreach my $virtdev (sort keys %$virtdev_hash) {
6618         my $d = $virtdev_hash->{$virtdev};
6619         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6620         my $storeid = $d->{storeid};
6621         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6622
6623         # test if requested format is supported
6624         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6625         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6626         $d->{format} = $defFormat if !$supported;
6627
6628         my $name;
6629         if ($d->{is_cloudinit}) {
6630             $name = "vm-$vmid-cloudinit";
6631             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6632             if ($scfg->{path}) {
6633                 $name .= ".$d->{format}";
6634             }
6635         }
6636
6637         my $volid = PVE::Storage::vdisk_alloc(
6638             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6639
6640         print STDERR "new volume ID is '$volid'\n";
6641         $d->{volid} = $volid;
6642
6643         PVE::Storage::activate_volumes($storecfg, [$volid]);
6644
6645         $map->{$virtdev} = $volid;
6646     }
6647
6648     return $map;
6649 };
6650
6651 sub restore_update_config_line {
6652     my ($cookie, $map, $line, $unique) = @_;
6653
6654     return '' if $line =~ m/^\#qmdump\#/;
6655     return '' if $line =~ m/^\#vzdump\#/;
6656     return '' if $line =~ m/^lock:/;
6657     return '' if $line =~ m/^unused\d+:/;
6658     return '' if $line =~ m/^parent:/;
6659
6660     my $res = '';
6661
6662     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6663     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6664         # try to convert old 1.X settings
6665         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6666         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6667             my ($model, $macaddr) = split(/\=/, $devconfig);
6668             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6669             my $net = {
6670                 model => $model,
6671                 bridge => "vmbr$ind",
6672                 macaddr => $macaddr,
6673             };
6674             my $netstr = print_net($net);
6675
6676             $res .= "net$cookie->{netcount}: $netstr\n";
6677             $cookie->{netcount}++;
6678         }
6679     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6680         my ($id, $netstr) = ($1, $2);
6681         my $net = parse_net($netstr);
6682         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6683         $netstr = print_net($net);
6684         $res .= "$id: $netstr\n";
6685     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6686         my $virtdev = $1;
6687         my $value = $3;
6688         my $di = parse_drive($virtdev, $value);
6689         if (defined($di->{backup}) && !$di->{backup}) {
6690             $res .= "#$line";
6691         } elsif ($map->{$virtdev}) {
6692             delete $di->{format}; # format can change on restore
6693             $di->{file} = $map->{$virtdev};
6694             $value = print_drive($di);
6695             $res .= "$virtdev: $value\n";
6696         } else {
6697             $res .= $line;
6698         }
6699     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6700         my $vmgenid = $1;
6701         if ($vmgenid ne '0') {
6702             # always generate a new vmgenid if there was a valid one setup
6703             $vmgenid = generate_uuid();
6704         }
6705         $res .= "vmgenid: $vmgenid\n";
6706     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6707         my ($uuid, $uuid_str);
6708         UUID::generate($uuid);
6709         UUID::unparse($uuid, $uuid_str);
6710         my $smbios1 = parse_smbios1($2);
6711         $smbios1->{uuid} = $uuid_str;
6712         $res .= $1.print_smbios1($smbios1)."\n";
6713     } else {
6714         $res .= $line;
6715     }
6716
6717     return $res;
6718 }
6719
6720 my $restore_deactivate_volumes = sub {
6721     my ($storecfg, $virtdev_hash) = @_;
6722
6723     my $vollist = [];
6724     for my $dev (values $virtdev_hash->%*) {
6725         push $vollist->@*, $dev->{volid} if $dev->{volid};
6726     }
6727
6728     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6729     print STDERR $@ if $@;
6730 };
6731
6732 my $restore_destroy_volumes = sub {
6733     my ($storecfg, $virtdev_hash) = @_;
6734
6735     for my $dev (values $virtdev_hash->%*) {
6736         my $volid = $dev->{volid} or next;
6737         eval {
6738             PVE::Storage::vdisk_free($storecfg, $volid);
6739             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6740         };
6741         print STDERR "unable to cleanup '$volid' - $@" if $@;
6742     }
6743 };
6744
6745 my $restore_merge_config = sub {
6746     my ($filename, $backup_conf_raw, $override_conf) = @_;
6747
6748     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6749     for my $key (keys $override_conf->%*) {
6750         $backup_conf->{$key} = $override_conf->{$key};
6751     }
6752
6753     return $backup_conf;
6754 };
6755
6756 sub scan_volids {
6757     my ($cfg, $vmid) = @_;
6758
6759     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6760
6761     my $volid_hash = {};
6762     foreach my $storeid (keys %$info) {
6763         foreach my $item (@{$info->{$storeid}}) {
6764             next if !($item->{volid} && $item->{size});
6765             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6766             $volid_hash->{$item->{volid}} = $item;
6767         }
6768     }
6769
6770     return $volid_hash;
6771 }
6772
6773 sub update_disk_config {
6774     my ($vmid, $conf, $volid_hash) = @_;
6775
6776     my $changes;
6777     my $prefix = "VM $vmid";
6778
6779     # used and unused disks
6780     my $referenced = {};
6781
6782     # Note: it is allowed to define multiple storages with same path (alias), so
6783     # we need to check both 'volid' and real 'path' (two different volid can point
6784     # to the same path).
6785
6786     my $referencedpath = {};
6787
6788     # update size info
6789     PVE::QemuConfig->foreach_volume($conf, sub {
6790         my ($opt, $drive) = @_;
6791
6792         my $volid = $drive->{file};
6793         return if !$volid;
6794         my $volume = $volid_hash->{$volid};
6795
6796         # mark volid as "in-use" for next step
6797         $referenced->{$volid} = 1;
6798         if ($volume && (my $path = $volume->{path})) {
6799             $referencedpath->{$path} = 1;
6800         }
6801
6802         return if drive_is_cdrom($drive);
6803         return if !$volume;
6804
6805         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6806         if (defined($updated)) {
6807             $changes = 1;
6808             $conf->{$opt} = print_drive($updated);
6809             print "$prefix ($opt): $msg\n";
6810         }
6811     });
6812
6813     # remove 'unusedX' entry if volume is used
6814     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6815         my ($opt, $drive) = @_;
6816
6817         my $volid = $drive->{file};
6818         return if !$volid;
6819
6820         my $path;
6821         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6822         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6823             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6824             $changes = 1;
6825             delete $conf->{$opt};
6826         }
6827
6828         $referenced->{$volid} = 1;
6829         $referencedpath->{$path} = 1 if $path;
6830     });
6831
6832     foreach my $volid (sort keys %$volid_hash) {
6833         next if $volid =~ m/vm-$vmid-state-/;
6834         next if $referenced->{$volid};
6835         my $path = $volid_hash->{$volid}->{path};
6836         next if !$path; # just to be sure
6837         next if $referencedpath->{$path};
6838         $changes = 1;
6839         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6840         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6841         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6842     }
6843
6844     return $changes;
6845 }
6846
6847 sub rescan {
6848     my ($vmid, $nolock, $dryrun) = @_;
6849
6850     my $cfg = PVE::Storage::config();
6851
6852     print "rescan volumes...\n";
6853     my $volid_hash = scan_volids($cfg, $vmid);
6854
6855     my $updatefn =  sub {
6856         my ($vmid) = @_;
6857
6858         my $conf = PVE::QemuConfig->load_config($vmid);
6859
6860         PVE::QemuConfig->check_lock($conf);
6861
6862         my $vm_volids = {};
6863         foreach my $volid (keys %$volid_hash) {
6864             my $info = $volid_hash->{$volid};
6865             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6866         }
6867
6868         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6869
6870         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6871     };
6872
6873     if (defined($vmid)) {
6874         if ($nolock) {
6875             &$updatefn($vmid);
6876         } else {
6877             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6878         }
6879     } else {
6880         my $vmlist = config_list();
6881         foreach my $vmid (keys %$vmlist) {
6882             if ($nolock) {
6883                 &$updatefn($vmid);
6884             } else {
6885                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6886             }
6887         }
6888     }
6889 }
6890
6891 sub restore_proxmox_backup_archive {
6892     my ($archive, $vmid, $user, $options) = @_;
6893
6894     my $storecfg = PVE::Storage::config();
6895
6896     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6897     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6898
6899     my $fingerprint = $scfg->{fingerprint};
6900     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6901
6902     my $repo = PVE::PBSClient::get_repository($scfg);
6903     my $namespace = $scfg->{namespace};
6904
6905     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6906     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6907     local $ENV{PBS_PASSWORD} = $password;
6908     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6909
6910     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6911         PVE::Storage::parse_volname($storecfg, $archive);
6912
6913     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6914
6915     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6916
6917     my $tmpdir = "/var/tmp/vzdumptmp$$";
6918     rmtree $tmpdir;
6919     mkpath $tmpdir;
6920
6921     my $conffile = PVE::QemuConfig->config_file($vmid);
6922      # disable interrupts (always do cleanups)
6923     local $SIG{INT} =
6924         local $SIG{TERM} =
6925         local $SIG{QUIT} =
6926         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6927
6928     # Note: $oldconf is undef if VM does not exists
6929     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6930     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6931     my $new_conf_raw = '';
6932
6933     my $rpcenv = PVE::RPCEnvironment::get();
6934     my $devinfo = {}; # info about drives included in backup
6935     my $virtdev_hash = {}; # info about allocated drives
6936
6937     eval {
6938         # enable interrupts
6939         local $SIG{INT} =
6940             local $SIG{TERM} =
6941             local $SIG{QUIT} =
6942             local $SIG{HUP} =
6943             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6944
6945         my $cfgfn = "$tmpdir/qemu-server.conf";
6946         my $firewall_config_fn = "$tmpdir/fw.conf";
6947         my $index_fn = "$tmpdir/index.json";
6948
6949         my $cmd = "restore";
6950
6951         my $param = [$pbs_backup_name, "index.json", $index_fn];
6952         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6953         my $index = PVE::Tools::file_get_contents($index_fn);
6954         $index = decode_json($index);
6955
6956         foreach my $info (@{$index->{files}}) {
6957             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6958                 my $devname = $1;
6959                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6960                     $devinfo->{$devname}->{size} = $1;
6961                 } else {
6962                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6963                 }
6964             }
6965         }
6966
6967         my $is_qemu_server_backup = scalar(
6968             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6969         );
6970         if (!$is_qemu_server_backup) {
6971             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6972         }
6973         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6974
6975         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6976         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6977
6978         if ($has_firewall_config) {
6979             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6980             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6981
6982             my $pve_firewall_dir = '/etc/pve/firewall';
6983             mkdir $pve_firewall_dir; # make sure the dir exists
6984             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6985         }
6986
6987         my $fh = IO::File->new($cfgfn, "r") ||
6988             die "unable to read qemu-server.conf - $!\n";
6989
6990         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6991
6992         # fixme: rate limit?
6993
6994         # create empty/temp config
6995         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6996
6997         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6998
6999         # allocate volumes
7000         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
7001
7002         foreach my $virtdev (sort keys %$virtdev_hash) {
7003             my $d = $virtdev_hash->{$virtdev};
7004             next if $d->{is_cloudinit}; # no need to restore cloudinit
7005
7006             # this fails if storage is unavailable
7007             my $volid = $d->{volid};
7008             my $path = PVE::Storage::path($storecfg, $volid);
7009
7010             # for live-restore we only want to preload the efidisk and TPM state
7011             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
7012
7013             my @ns_arg;
7014             if (defined(my $ns = $scfg->{namespace})) {
7015                 @ns_arg = ('--ns', $ns);
7016             }
7017
7018             my $pbs_restore_cmd = [
7019                 '/usr/bin/pbs-restore',
7020                 '--repository', $repo,
7021                 @ns_arg,
7022                 $pbs_backup_name,
7023                 "$d->{devname}.img.fidx",
7024                 $path,
7025                 '--verbose',
7026                 ];
7027
7028             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
7029             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
7030
7031             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
7032                 push @$pbs_restore_cmd, '--skip-zero';
7033             }
7034
7035             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
7036             print "restore proxmox backup image: $dbg_cmdstring\n";
7037             run_command($pbs_restore_cmd);
7038         }
7039
7040         $fh->seek(0, 0) || die "seek failed - $!\n";
7041
7042         my $cookie = { netcount => 0 };
7043         while (defined(my $line = <$fh>)) {
7044             $new_conf_raw .= restore_update_config_line(
7045                 $cookie,
7046                 $map,
7047                 $line,
7048                 $options->{unique},
7049             );
7050         }
7051
7052         $fh->close();
7053     };
7054     my $err = $@;
7055
7056     if ($err || !$options->{live}) {
7057         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
7058     }
7059
7060     rmtree $tmpdir;
7061
7062     if ($err) {
7063         $restore_destroy_volumes->($storecfg, $virtdev_hash);
7064         die $err;
7065     }
7066
7067     if ($options->{live}) {
7068         # keep lock during live-restore
7069         $new_conf_raw .= "\nlock: create";
7070     }
7071
7072     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
7073     PVE::QemuConfig->write_config($vmid, $new_conf);
7074
7075     eval { rescan($vmid, 1); };
7076     warn $@ if $@;
7077
7078     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
7079
7080     if ($options->{live}) {
7081         # enable interrupts
7082         local $SIG{INT} =
7083             local $SIG{TERM} =
7084             local $SIG{QUIT} =
7085             local $SIG{HUP} =
7086             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
7087
7088         my $conf = PVE::QemuConfig->load_config($vmid);
7089         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
7090
7091         # these special drives are already restored before start
7092         delete $devinfo->{'drive-efidisk0'};
7093         delete $devinfo->{'drive-tpmstate0-backup'};
7094
7095         my $pbs_opts = {
7096             repo => $repo,
7097             keyfile => $keyfile,
7098             snapshot => $pbs_backup_name,
7099             namespace => $namespace,
7100         };
7101         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
7102
7103         PVE::QemuConfig->remove_lock($vmid, "create");
7104     }
7105 }
7106
7107 sub pbs_live_restore {
7108     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
7109
7110     print "starting VM for live-restore\n";
7111     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
7112
7113     my $pbs_backing = {};
7114     for my $ds (keys %$restored_disks) {
7115         $ds =~ m/^drive-(.*)$/;
7116         my $confname = $1;
7117         $pbs_backing->{$confname} = {
7118             repository => $opts->{repo},
7119             snapshot => $opts->{snapshot},
7120             archive => "$ds.img.fidx",
7121         };
7122         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
7123         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
7124
7125         my $drive = parse_drive($confname, $conf->{$confname});
7126         print "restoring '$ds' to '$drive->{file}'\n";
7127     }
7128
7129     my $drives_streamed = 0;
7130     eval {
7131         # make sure HA doesn't interrupt our restore by stopping the VM
7132         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7133             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7134         }
7135
7136         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7137         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7138         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7139
7140         my $qmeventd_fd = register_qmeventd_handle($vmid);
7141
7142         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7143         # this will effectively collapse the backing image chain consisting of
7144         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7145         # removes itself once all backing images vanish with 'auto-remove=on')
7146         my $jobs = {};
7147         for my $ds (sort keys %$restored_disks) {
7148             my $job_id = "restore-$ds";
7149             mon_cmd($vmid, 'block-stream',
7150                 'job-id' => $job_id,
7151                 device => "$ds",
7152             );
7153             $jobs->{$job_id} = {};
7154         }
7155
7156         mon_cmd($vmid, 'cont');
7157         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7158
7159         print "restore-drive jobs finished successfully, removing all tracking block devices"
7160             ." to disconnect from Proxmox Backup Server\n";
7161
7162         for my $ds (sort keys %$restored_disks) {
7163             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7164         }
7165
7166         close($qmeventd_fd);
7167     };
7168
7169     my $err = $@;
7170
7171     if ($err) {
7172         warn "An error occurred during live-restore: $err\n";
7173         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7174         die "live-restore failed\n";
7175     }
7176 }
7177
7178 sub restore_vma_archive {
7179     my ($archive, $vmid, $user, $opts, $comp) = @_;
7180
7181     my $readfrom = $archive;
7182
7183     my $cfg = PVE::Storage::config();
7184     my $commands = [];
7185     my $bwlimit = $opts->{bwlimit};
7186
7187     my $dbg_cmdstring = '';
7188     my $add_pipe = sub {
7189         my ($cmd) = @_;
7190         push @$commands, $cmd;
7191         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7192         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7193         $readfrom = '-';
7194     };
7195
7196     my $input = undef;
7197     if ($archive eq '-') {
7198         $input = '<&STDIN';
7199     } else {
7200         # If we use a backup from a PVE defined storage we also consider that
7201         # storage's rate limit:
7202         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7203         if (defined($volid)) {
7204             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7205             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7206             if ($readlimit) {
7207                 print STDERR "applying read rate limit: $readlimit\n";
7208                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7209                 $add_pipe->($cstream);
7210             }
7211         }
7212     }
7213
7214     if ($comp) {
7215         my $info = PVE::Storage::decompressor_info('vma', $comp);
7216         my $cmd = $info->{decompressor};
7217         push @$cmd, $readfrom;
7218         $add_pipe->($cmd);
7219     }
7220
7221     my $tmpdir = "/var/tmp/vzdumptmp$$";
7222     rmtree $tmpdir;
7223
7224     # disable interrupts (always do cleanups)
7225     local $SIG{INT} =
7226         local $SIG{TERM} =
7227         local $SIG{QUIT} =
7228         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7229
7230     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7231     POSIX::mkfifo($mapfifo, 0600);
7232     my $fifofh;
7233     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7234
7235     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7236
7237     my $oldtimeout;
7238     my $timeout = 5;
7239
7240     my $devinfo = {}; # info about drives included in backup
7241     my $virtdev_hash = {}; # info about allocated drives
7242
7243     my $rpcenv = PVE::RPCEnvironment::get();
7244
7245     my $conffile = PVE::QemuConfig->config_file($vmid);
7246
7247     # Note: $oldconf is undef if VM does not exist
7248     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7249     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7250     my $new_conf_raw = '';
7251
7252     my %storage_limits;
7253
7254     my $print_devmap = sub {
7255         my $cfgfn = "$tmpdir/qemu-server.conf";
7256
7257         # we can read the config - that is already extracted
7258         my $fh = IO::File->new($cfgfn, "r") ||
7259             die "unable to read qemu-server.conf - $!\n";
7260
7261         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7262         if (-f $fwcfgfn) {
7263             my $pve_firewall_dir = '/etc/pve/firewall';
7264             mkdir $pve_firewall_dir; # make sure the dir exists
7265             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7266         }
7267
7268         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7269
7270         foreach my $info (values %{$virtdev_hash}) {
7271             my $storeid = $info->{storeid};
7272             next if defined($storage_limits{$storeid});
7273
7274             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7275             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7276             $storage_limits{$storeid} = $limit * 1024;
7277         }
7278
7279         foreach my $devname (keys %$devinfo) {
7280             die "found no device mapping information for device '$devname'\n"
7281                 if !$devinfo->{$devname}->{virtdev};
7282         }
7283
7284         # create empty/temp config
7285         if ($oldconf) {
7286             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7287             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7288         }
7289
7290         # allocate volumes
7291         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7292
7293         # print restore information to $fifofh
7294         foreach my $virtdev (sort keys %$virtdev_hash) {
7295             my $d = $virtdev_hash->{$virtdev};
7296             next if $d->{is_cloudinit}; # no need to restore cloudinit
7297
7298             my $storeid = $d->{storeid};
7299             my $volid = $d->{volid};
7300
7301             my $map_opts = '';
7302             if (my $limit = $storage_limits{$storeid}) {
7303                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7304             }
7305
7306             my $write_zeros = 1;
7307             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7308                 $write_zeros = 0;
7309             }
7310
7311             my $path = PVE::Storage::path($cfg, $volid);
7312
7313             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7314
7315             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7316         }
7317
7318         $fh->seek(0, 0) || die "seek failed - $!\n";
7319
7320         my $cookie = { netcount => 0 };
7321         while (defined(my $line = <$fh>)) {
7322             $new_conf_raw .= restore_update_config_line(
7323                 $cookie,
7324                 $map,
7325                 $line,
7326                 $opts->{unique},
7327             );
7328         }
7329
7330         $fh->close();
7331     };
7332
7333     eval {
7334         # enable interrupts
7335         local $SIG{INT} =
7336             local $SIG{TERM} =
7337             local $SIG{QUIT} =
7338             local $SIG{HUP} =
7339             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7340         local $SIG{ALRM} = sub { die "got timeout\n"; };
7341
7342         $oldtimeout = alarm($timeout);
7343
7344         my $parser = sub {
7345             my $line = shift;
7346
7347             print "$line\n";
7348
7349             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7350                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7351                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7352             } elsif ($line =~ m/^CTIME: /) {
7353                 # we correctly received the vma config, so we can disable
7354                 # the timeout now for disk allocation (set to 10 minutes, so
7355                 # that we always timeout if something goes wrong)
7356                 alarm(600);
7357                 &$print_devmap();
7358                 print $fifofh "done\n";
7359                 my $tmp = $oldtimeout || 0;
7360                 $oldtimeout = undef;
7361                 alarm($tmp);
7362                 close($fifofh);
7363                 $fifofh = undef;
7364             }
7365         };
7366
7367         print "restore vma archive: $dbg_cmdstring\n";
7368         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7369     };
7370     my $err = $@;
7371
7372     alarm($oldtimeout) if $oldtimeout;
7373
7374     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7375
7376     close($fifofh) if $fifofh;
7377     unlink $mapfifo;
7378     rmtree $tmpdir;
7379
7380     if ($err) {
7381         $restore_destroy_volumes->($cfg, $virtdev_hash);
7382         die $err;
7383     }
7384
7385     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7386     PVE::QemuConfig->write_config($vmid, $new_conf);
7387
7388     eval { rescan($vmid, 1); };
7389     warn $@ if $@;
7390
7391     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7392 }
7393
7394 sub restore_tar_archive {
7395     my ($archive, $vmid, $user, $opts) = @_;
7396
7397     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7398         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7399         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7400     }
7401
7402     if ($archive ne '-') {
7403         my $firstfile = tar_archive_read_firstfile($archive);
7404         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7405             if $firstfile ne 'qemu-server.conf';
7406     }
7407
7408     my $storecfg = PVE::Storage::config();
7409
7410     # avoid zombie disks when restoring over an existing VM -> cleanup first
7411     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7412     # skiplock=1 because qmrestore has set the 'create' lock itself already
7413     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7414     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7415
7416     my $tocmd = "/usr/lib/qemu-server/qmextract";
7417
7418     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7419     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7420     $tocmd .= ' --prealloc' if $opts->{prealloc};
7421     $tocmd .= ' --info' if $opts->{info};
7422
7423     # tar option "xf" does not autodetect compression when read from STDIN,
7424     # so we pipe to zcat
7425     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7426         PVE::Tools::shellquote("--to-command=$tocmd");
7427
7428     my $tmpdir = "/var/tmp/vzdumptmp$$";
7429     mkpath $tmpdir;
7430
7431     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7432     local $ENV{VZDUMP_VMID} = $vmid;
7433     local $ENV{VZDUMP_USER} = $user;
7434
7435     my $conffile = PVE::QemuConfig->config_file($vmid);
7436     my $new_conf_raw = '';
7437
7438     # disable interrupts (always do cleanups)
7439     local $SIG{INT} =
7440         local $SIG{TERM} =
7441         local $SIG{QUIT} =
7442         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7443
7444     eval {
7445         # enable interrupts
7446         local $SIG{INT} =
7447             local $SIG{TERM} =
7448             local $SIG{QUIT} =
7449             local $SIG{HUP} =
7450             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7451
7452         if ($archive eq '-') {
7453             print "extracting archive from STDIN\n";
7454             run_command($cmd, input => "<&STDIN");
7455         } else {
7456             print "extracting archive '$archive'\n";
7457             run_command($cmd);
7458         }
7459
7460         return if $opts->{info};
7461
7462         # read new mapping
7463         my $map = {};
7464         my $statfile = "$tmpdir/qmrestore.stat";
7465         if (my $fd = IO::File->new($statfile, "r")) {
7466             while (defined (my $line = <$fd>)) {
7467                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7468                     $map->{$1} = $2 if $1;
7469                 } else {
7470                     print STDERR "unable to parse line in statfile - $line\n";
7471                 }
7472             }
7473             $fd->close();
7474         }
7475
7476         my $confsrc = "$tmpdir/qemu-server.conf";
7477
7478         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7479
7480         my $cookie = { netcount => 0 };
7481         while (defined (my $line = <$srcfd>)) {
7482             $new_conf_raw .= restore_update_config_line(
7483                 $cookie,
7484                 $map,
7485                 $line,
7486                 $opts->{unique},
7487             );
7488         }
7489
7490         $srcfd->close();
7491     };
7492     if (my $err = $@) {
7493         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7494         die $err;
7495     }
7496
7497     rmtree $tmpdir;
7498
7499     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7500
7501     PVE::Cluster::cfs_update(); # make sure we read new file
7502
7503     eval { rescan($vmid, 1); };
7504     warn $@ if $@;
7505 };
7506
7507 sub foreach_storage_used_by_vm {
7508     my ($conf, $func) = @_;
7509
7510     my $sidhash = {};
7511
7512     PVE::QemuConfig->foreach_volume($conf, sub {
7513         my ($ds, $drive) = @_;
7514         return if drive_is_cdrom($drive);
7515
7516         my $volid = $drive->{file};
7517
7518         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7519         $sidhash->{$sid} = $sid if $sid;
7520     });
7521
7522     foreach my $sid (sort keys %$sidhash) {
7523         &$func($sid);
7524     }
7525 }
7526
7527 my $qemu_snap_storage = {
7528     rbd => 1,
7529 };
7530 sub do_snapshots_with_qemu {
7531     my ($storecfg, $volid, $deviceid) = @_;
7532
7533     return if $deviceid =~ m/tpmstate0/;
7534
7535     my $storage_name = PVE::Storage::parse_volume_id($volid);
7536     my $scfg = $storecfg->{ids}->{$storage_name};
7537     die "could not find storage '$storage_name'\n" if !defined($scfg);
7538
7539     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7540         return 1;
7541     }
7542
7543     if ($volid =~ m/\.(qcow2|qed)$/){
7544         return 1;
7545     }
7546
7547     return;
7548 }
7549
7550 sub qga_check_running {
7551     my ($vmid, $nowarn) = @_;
7552
7553     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7554     if ($@) {
7555         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7556         return 0;
7557     }
7558     return 1;
7559 }
7560
7561 sub template_create {
7562     my ($vmid, $conf, $disk) = @_;
7563
7564     my $storecfg = PVE::Storage::config();
7565
7566     PVE::QemuConfig->foreach_volume($conf, sub {
7567         my ($ds, $drive) = @_;
7568
7569         return if drive_is_cdrom($drive);
7570         return if $disk && $ds ne $disk;
7571
7572         my $volid = $drive->{file};
7573         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7574
7575         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7576         $drive->{file} = $voliddst;
7577         $conf->{$ds} = print_drive($drive);
7578         PVE::QemuConfig->write_config($vmid, $conf);
7579     });
7580 }
7581
7582 sub convert_iscsi_path {
7583     my ($path) = @_;
7584
7585     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7586         my $portal = $1;
7587         my $target = $2;
7588         my $lun = $3;
7589
7590         my $initiator_name = get_initiator_name();
7591
7592         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7593                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7594     }
7595
7596     die "cannot convert iscsi path '$path', unkown format\n";
7597 }
7598
7599 sub qemu_img_convert {
7600     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7601
7602     my $storecfg = PVE::Storage::config();
7603     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7604     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7605
7606     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7607
7608     my $cachemode;
7609     my $src_path;
7610     my $src_is_iscsi = 0;
7611     my $src_format;
7612
7613     if ($src_storeid) {
7614         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7615         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7616         $src_format = qemu_img_format($src_scfg, $src_volname);
7617         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7618         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7619         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7620     } elsif (-f $src_volid || -b $src_volid) {
7621         $src_path = $src_volid;
7622         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7623             $src_format = $1;
7624         }
7625     }
7626
7627     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7628
7629     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7630     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7631     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7632     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7633
7634     my $cmd = [];
7635     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7636     push @$cmd, '-l', "snapshot.name=$snapname"
7637         if $snapname && $src_format && $src_format eq "qcow2";
7638     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7639     push @$cmd, '-T', $cachemode if defined($cachemode);
7640
7641     if ($src_is_iscsi) {
7642         push @$cmd, '--image-opts';
7643         $src_path = convert_iscsi_path($src_path);
7644     } elsif ($src_format) {
7645         push @$cmd, '-f', $src_format;
7646     }
7647
7648     if ($dst_is_iscsi) {
7649         push @$cmd, '--target-image-opts';
7650         $dst_path = convert_iscsi_path($dst_path);
7651     } else {
7652         push @$cmd, '-O', $dst_format;
7653     }
7654
7655     push @$cmd, $src_path;
7656
7657     if (!$dst_is_iscsi && $is_zero_initialized) {
7658         push @$cmd, "zeroinit:$dst_path";
7659     } else {
7660         push @$cmd, $dst_path;
7661     }
7662
7663     my $parser = sub {
7664         my $line = shift;
7665         if($line =~ m/\((\S+)\/100\%\)/){
7666             my $percent = $1;
7667             my $transferred = int($size * $percent / 100);
7668             my $total_h = render_bytes($size, 1);
7669             my $transferred_h = render_bytes($transferred, 1);
7670
7671             print "transferred $transferred_h of $total_h ($percent%)\n";
7672         }
7673
7674     };
7675
7676     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7677     my $err = $@;
7678     die "copy failed: $err" if $err;
7679 }
7680
7681 sub qemu_img_format {
7682     my ($scfg, $volname) = @_;
7683
7684     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7685         return $1;
7686     } else {
7687         return "raw";
7688     }
7689 }
7690
7691 sub qemu_drive_mirror {
7692     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7693
7694     $jobs = {} if !$jobs;
7695
7696     my $qemu_target;
7697     my $format;
7698     $jobs->{"drive-$drive"} = {};
7699
7700     if ($dst_volid =~ /^nbd:/) {
7701         $qemu_target = $dst_volid;
7702         $format = "nbd";
7703     } else {
7704         my $storecfg = PVE::Storage::config();
7705         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7706
7707         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7708
7709         $format = qemu_img_format($dst_scfg, $dst_volname);
7710
7711         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7712
7713         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7714     }
7715
7716     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7717     $opts->{format} = $format if $format;
7718
7719     if (defined($src_bitmap)) {
7720         $opts->{sync} = 'incremental';
7721         $opts->{bitmap} = $src_bitmap;
7722         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7723     }
7724
7725     if (defined($bwlimit)) {
7726         $opts->{speed} = $bwlimit * 1024;
7727         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7728     } else {
7729         print "drive mirror is starting for drive-$drive\n";
7730     }
7731
7732     # if a job already runs for this device we get an error, catch it for cleanup
7733     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7734     if (my $err = $@) {
7735         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7736         warn "$@\n" if $@;
7737         die "mirroring error: $err\n";
7738     }
7739
7740     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7741 }
7742
7743 # $completion can be either
7744 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7745 # 'cancel': wait until all jobs are ready, block-job-cancel them
7746 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7747 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7748 sub qemu_drive_mirror_monitor {
7749     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7750
7751     $completion //= 'complete';
7752     $op //= "mirror";
7753
7754     eval {
7755         my $err_complete = 0;
7756
7757         my $starttime = time ();
7758         while (1) {
7759             die "block job ('$op') timed out\n" if $err_complete > 300;
7760
7761             my $stats = mon_cmd($vmid, "query-block-jobs");
7762             my $ctime = time();
7763
7764             my $running_jobs = {};
7765             for my $stat (@$stats) {
7766                 next if $stat->{type} ne $op;
7767                 $running_jobs->{$stat->{device}} = $stat;
7768             }
7769
7770             my $readycounter = 0;
7771
7772             for my $job_id (sort keys %$jobs) {
7773                 my $job = $running_jobs->{$job_id};
7774
7775                 my $vanished = !defined($job);
7776                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7777                 if($complete || ($vanished && $completion eq 'auto')) {
7778                     print "$job_id: $op-job finished\n";
7779                     delete $jobs->{$job_id};
7780                     next;
7781                 }
7782
7783                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7784
7785                 my $busy = $job->{busy};
7786                 my $ready = $job->{ready};
7787                 if (my $total = $job->{len}) {
7788                     my $transferred = $job->{offset} || 0;
7789                     my $remaining = $total - $transferred;
7790                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7791
7792                     my $duration = $ctime - $starttime;
7793                     my $total_h = render_bytes($total, 1);
7794                     my $transferred_h = render_bytes($transferred, 1);
7795
7796                     my $status = sprintf(
7797                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7798                         render_duration($duration),
7799                     );
7800
7801                     if ($ready) {
7802                         if ($busy) {
7803                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7804                         } else {
7805                             $status .= ", ready";
7806                         }
7807                     }
7808                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7809                     $jobs->{$job_id}->{ready} = $ready;
7810                 }
7811
7812                 $readycounter++ if $job->{ready};
7813             }
7814
7815             last if scalar(keys %$jobs) == 0;
7816
7817             if ($readycounter == scalar(keys %$jobs)) {
7818                 print "all '$op' jobs are ready\n";
7819
7820                 # do the complete later (or has already been done)
7821                 last if $completion eq 'skip' || $completion eq 'auto';
7822
7823                 if ($vmiddst && $vmiddst != $vmid) {
7824                     my $agent_running = $qga && qga_check_running($vmid);
7825                     if ($agent_running) {
7826                         print "freeze filesystem\n";
7827                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7828                         warn $@ if $@;
7829                     } else {
7830                         print "suspend vm\n";
7831                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7832                         warn $@ if $@;
7833                     }
7834
7835                     # if we clone a disk for a new target vm, we don't switch the disk
7836                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7837
7838                     if ($agent_running) {
7839                         print "unfreeze filesystem\n";
7840                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7841                         warn $@ if $@;
7842                     } else {
7843                         print "resume vm\n";
7844                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7845                         warn $@ if $@;
7846                     }
7847
7848                     last;
7849                 } else {
7850
7851                     for my $job_id (sort keys %$jobs) {
7852                         # try to switch the disk if source and destination are on the same guest
7853                         print "$job_id: Completing block job_id...\n";
7854
7855                         my $op;
7856                         if ($completion eq 'complete') {
7857                             $op = 'block-job-complete';
7858                         } elsif ($completion eq 'cancel') {
7859                             $op = 'block-job-cancel';
7860                         } else {
7861                             die "invalid completion value: $completion\n";
7862                         }
7863                         eval { mon_cmd($vmid, $op, device => $job_id) };
7864                         if ($@ =~ m/cannot be completed/) {
7865                             print "$job_id: block job cannot be completed, trying again.\n";
7866                             $err_complete++;
7867                         }else {
7868                             print "$job_id: Completed successfully.\n";
7869                             $jobs->{$job_id}->{complete} = 1;
7870                         }
7871                     }
7872                 }
7873             }
7874             sleep 1;
7875         }
7876     };
7877     my $err = $@;
7878
7879     if ($err) {
7880         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7881         die "block job ($op) error: $err";
7882     }
7883 }
7884
7885 sub qemu_blockjobs_cancel {
7886     my ($vmid, $jobs) = @_;
7887
7888     foreach my $job (keys %$jobs) {
7889         print "$job: Cancelling block job\n";
7890         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7891         $jobs->{$job}->{cancel} = 1;
7892     }
7893
7894     while (1) {
7895         my $stats = mon_cmd($vmid, "query-block-jobs");
7896
7897         my $running_jobs = {};
7898         foreach my $stat (@$stats) {
7899             $running_jobs->{$stat->{device}} = $stat;
7900         }
7901
7902         foreach my $job (keys %$jobs) {
7903
7904             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7905                 print "$job: Done.\n";
7906                 delete $jobs->{$job};
7907             }
7908         }
7909
7910         last if scalar(keys %$jobs) == 0;
7911
7912         sleep 1;
7913     }
7914 }
7915
7916 sub clone_disk {
7917     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7918
7919     my ($vmid, $running) = $source->@{qw(vmid running)};
7920     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7921
7922     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7923     my ($storage, $format) = $dest->@{qw(storage format)};
7924
7925     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7926
7927     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7928         die "cloning from/to EFI disk requires EFI disk\n"
7929             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7930         die "cloning from/to TPM state requires TPM state\n"
7931             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7932
7933         # This would lead to two device nodes in QEMU pointing to the same backing image!
7934         die "cannot change drive name when cloning disk from/to the same VM\n"
7935             if $use_drive_mirror && $vmid == $newvmid;
7936     }
7937
7938     die "cannot move TPM state while VM is running\n"
7939         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7940
7941     my $newvolid;
7942
7943     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7944     print "$src_drivename " if $src_drivename;
7945     print "($drive->{file})\n";
7946
7947     if (!$full) {
7948         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7949         push @$newvollist, $newvolid;
7950     } else {
7951
7952         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7953         $storeid = $storage if $storage;
7954
7955         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7956
7957         my $name = undef;
7958         my $size = undef;
7959         if (drive_is_cloudinit($drive)) {
7960             $name = "vm-$newvmid-cloudinit";
7961             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7962             if ($scfg->{path}) {
7963                 $name .= ".$dst_format";
7964             }
7965             $snapname = undef;
7966             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7967         } elsif ($dst_drivename eq 'efidisk0') {
7968             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7969         } elsif ($dst_drivename eq 'tpmstate0') {
7970             $dst_format = 'raw';
7971             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7972         } else {
7973             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7974         }
7975         $newvolid = PVE::Storage::vdisk_alloc(
7976             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7977         );
7978         push @$newvollist, $newvolid;
7979
7980         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7981
7982         if (drive_is_cloudinit($drive)) {
7983             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7984             # if this is the case, we have to complete any block-jobs still there from
7985             # previous drive-mirrors
7986             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7987                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7988             }
7989             goto no_data_clone;
7990         }
7991
7992         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7993         if ($use_drive_mirror) {
7994             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7995                 $completion, $qga, $bwlimit);
7996         } else {
7997             # TODO: handle bwlimits
7998             if ($dst_drivename eq 'efidisk0') {
7999                 # the relevant data on the efidisk may be smaller than the source
8000                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
8001                 # that is given by the OVMF_VARS.fd
8002                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
8003                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
8004
8005                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
8006
8007                 # better for Ceph if block size is not too small, see bug #3324
8008                 my $bs = 1024*1024;
8009
8010                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
8011
8012                 if ($src_format eq 'qcow2' && $snapname) {
8013                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
8014                         if !min_version(kvm_user_version(), 6, 2);
8015                     push $cmd->@*, '-l', $snapname;
8016                 }
8017                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
8018                 run_command($cmd);
8019             } else {
8020                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
8021             }
8022         }
8023     }
8024
8025 no_data_clone:
8026     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
8027
8028     my $disk = dclone($drive);
8029     delete $disk->{format};
8030     $disk->{file} = $newvolid;
8031     $disk->{size} = $size if defined($size);
8032
8033     return $disk;
8034 }
8035
8036 sub get_running_qemu_version {
8037     my ($vmid) = @_;
8038     my $res = mon_cmd($vmid, "query-version");
8039     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
8040 }
8041
8042 sub qemu_use_old_bios_files {
8043     my ($machine_type) = @_;
8044
8045     return if !$machine_type;
8046
8047     my $use_old_bios_files = undef;
8048
8049     if ($machine_type =~ m/^(\S+)\.pxe$/) {
8050         $machine_type = $1;
8051         $use_old_bios_files = 1;
8052     } else {
8053         my $version = extract_version($machine_type, kvm_user_version());
8054         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
8055         # load new efi bios files on migration. So this hack is required to allow
8056         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
8057         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
8058         $use_old_bios_files = !min_version($version, 2, 4);
8059     }
8060
8061     return ($use_old_bios_files, $machine_type);
8062 }
8063
8064 sub get_efivars_size {
8065     my ($conf, $efidisk) = @_;
8066
8067     my $arch = get_vm_arch($conf);
8068     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
8069     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
8070     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8071     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
8072     return -s $ovmf_vars;
8073 }
8074
8075 sub update_efidisk_size {
8076     my ($conf) = @_;
8077
8078     return if !defined($conf->{efidisk0});
8079
8080     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
8081     $disk->{size} = get_efivars_size($conf);
8082     $conf->{efidisk0} = print_drive($disk);
8083
8084     return;
8085 }
8086
8087 sub update_tpmstate_size {
8088     my ($conf) = @_;
8089
8090     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
8091     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
8092     $conf->{tpmstate0} = print_drive($disk);
8093 }
8094
8095 sub create_efidisk($$$$$$$) {
8096     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
8097
8098     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
8099     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
8100
8101     my $vars_size_b = -s $ovmf_vars;
8102     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
8103     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
8104     PVE::Storage::activate_volumes($storecfg, [$volid]);
8105
8106     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
8107     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
8108
8109     return ($volid, $size/1024);
8110 }
8111
8112 sub vm_iothreads_list {
8113     my ($vmid) = @_;
8114
8115     my $res = mon_cmd($vmid, 'query-iothreads');
8116
8117     my $iothreads = {};
8118     foreach my $iothread (@$res) {
8119         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
8120     }
8121
8122     return $iothreads;
8123 }
8124
8125 sub scsihw_infos {
8126     my ($conf, $drive) = @_;
8127
8128     my $maxdev = 0;
8129
8130     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8131         $maxdev = 7;
8132     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8133         $maxdev = 1;
8134     } else {
8135         $maxdev = 256;
8136     }
8137
8138     my $controller = int($drive->{index} / $maxdev);
8139     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8140         ? "virtioscsi"
8141         : "scsihw";
8142
8143     return ($maxdev, $controller, $controller_prefix);
8144 }
8145
8146 sub resolve_dst_disk_format {
8147         my ($storecfg, $storeid, $src_volname, $format) = @_;
8148         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8149
8150         if (!$format) {
8151             # if no target format is specified, use the source disk format as hint
8152             if ($src_volname) {
8153                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8154                 $format = qemu_img_format($scfg, $src_volname);
8155             } else {
8156                 return $defFormat;
8157             }
8158         }
8159
8160         # test if requested format is supported - else use default
8161         my $supported = grep { $_ eq $format } @$validFormats;
8162         $format = $defFormat if !$supported;
8163         return $format;
8164 }
8165
8166 # NOTE: if this logic changes, please update docs & possibly gui logic
8167 sub find_vmstate_storage {
8168     my ($conf, $storecfg) = @_;
8169
8170     # first, return storage from conf if set
8171     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8172
8173     my ($target, $shared, $local);
8174
8175     foreach_storage_used_by_vm($conf, sub {
8176         my ($sid) = @_;
8177         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8178         my $dst = $scfg->{shared} ? \$shared : \$local;
8179         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8180     });
8181
8182     # second, use shared storage where VM has at least one disk
8183     # third, use local storage where VM has at least one disk
8184     # fall back to local storage
8185     $target = $shared // $local // 'local';
8186
8187     return $target;
8188 }
8189
8190 sub generate_uuid {
8191     my ($uuid, $uuid_str);
8192     UUID::generate($uuid);
8193     UUID::unparse($uuid, $uuid_str);
8194     return $uuid_str;
8195 }
8196
8197 sub generate_smbios1_uuid {
8198     return "uuid=".generate_uuid();
8199 }
8200
8201 sub nbd_stop {
8202     my ($vmid) = @_;
8203
8204     mon_cmd($vmid, 'nbd-server-stop');
8205 }
8206
8207 sub create_reboot_request {
8208     my ($vmid) = @_;
8209     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8210         or die "failed to create reboot trigger file: $!\n";
8211     close($fh);
8212 }
8213
8214 sub clear_reboot_request {
8215     my ($vmid) = @_;
8216     my $path = "/run/qemu-server/$vmid.reboot";
8217     my $res = 0;
8218
8219     $res = unlink($path);
8220     die "could not remove reboot request for $vmid: $!"
8221         if !$res && $! != POSIX::ENOENT;
8222
8223     return $res;
8224 }
8225
8226 sub bootorder_from_legacy {
8227     my ($conf, $bootcfg) = @_;
8228
8229     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8230     my $bootindex_hash = {};
8231     my $i = 1;
8232     foreach my $o (split(//, $boot)) {
8233         $bootindex_hash->{$o} = $i*100;
8234         $i++;
8235     }
8236
8237     my $bootorder = {};
8238
8239     PVE::QemuConfig->foreach_volume($conf, sub {
8240         my ($ds, $drive) = @_;
8241
8242         if (drive_is_cdrom ($drive, 1)) {
8243             if ($bootindex_hash->{d}) {
8244                 $bootorder->{$ds} = $bootindex_hash->{d};
8245                 $bootindex_hash->{d} += 1;
8246             }
8247         } elsif ($bootindex_hash->{c}) {
8248             $bootorder->{$ds} = $bootindex_hash->{c}
8249                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8250             $bootindex_hash->{c} += 1;
8251         }
8252     });
8253
8254     if ($bootindex_hash->{n}) {
8255         for (my $i = 0; $i < $MAX_NETS; $i++) {
8256             my $netname = "net$i";
8257             next if !$conf->{$netname};
8258             $bootorder->{$netname} = $bootindex_hash->{n};
8259             $bootindex_hash->{n} += 1;
8260         }
8261     }
8262
8263     return $bootorder;
8264 }
8265
8266 # Generate default device list for 'boot: order=' property. Matches legacy
8267 # default boot order, but with explicit device names. This is important, since
8268 # the fallback for when neither 'order' nor the old format is specified relies
8269 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8270 sub get_default_bootdevices {
8271     my ($conf) = @_;
8272
8273     my @ret = ();
8274
8275     # harddisk
8276     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8277     push @ret, $first if $first;
8278
8279     # cdrom
8280     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8281     push @ret, $first if $first;
8282
8283     # network
8284     for (my $i = 0; $i < $MAX_NETS; $i++) {
8285         my $netname = "net$i";
8286         next if !$conf->{$netname};
8287         push @ret, $netname;
8288         last;
8289     }
8290
8291     return \@ret;
8292 }
8293
8294 sub device_bootorder {
8295     my ($conf) = @_;
8296
8297     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8298
8299     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8300
8301     my $bootorder = {};
8302     if (!defined($boot) || $boot->{legacy}) {
8303         $bootorder = bootorder_from_legacy($conf, $boot);
8304     } elsif ($boot->{order}) {
8305         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8306         for my $dev (PVE::Tools::split_list($boot->{order})) {
8307             $bootorder->{$dev} = $i++;
8308         }
8309     }
8310
8311     return $bootorder;
8312 }
8313
8314 sub register_qmeventd_handle {
8315     my ($vmid) = @_;
8316
8317     my $fh;
8318     my $peer = "/var/run/qmeventd.sock";
8319     my $count = 0;
8320
8321     for (;;) {
8322         $count++;
8323         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8324         last if $fh;
8325         if ($! != EINTR && $! != EAGAIN) {
8326             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8327         }
8328         if ($count > 4) {
8329             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8330               . "after $count retries\n";
8331         }
8332         usleep(25000);
8333     }
8334
8335     # send handshake to mark VM as backing up
8336     print $fh to_json({vzdump => {vmid => "$vmid"}});
8337
8338     # return handle to be closed later when inhibit is no longer required
8339     return $fh;
8340 }
8341
8342 # bash completion helper
8343
8344 sub complete_backup_archives {
8345     my ($cmdname, $pname, $cvalue) = @_;
8346
8347     my $cfg = PVE::Storage::config();
8348
8349     my $storeid;
8350
8351     if ($cvalue =~ m/^([^:]+):/) {
8352         $storeid = $1;
8353     }
8354
8355     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8356
8357     my $res = [];
8358     foreach my $id (keys %$data) {
8359         foreach my $item (@{$data->{$id}}) {
8360             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8361             push @$res, $item->{volid} if defined($item->{volid});
8362         }
8363     }
8364
8365     return $res;
8366 }
8367
8368 my $complete_vmid_full = sub {
8369     my ($running) = @_;
8370
8371     my $idlist = vmstatus();
8372
8373     my $res = [];
8374
8375     foreach my $id (keys %$idlist) {
8376         my $d = $idlist->{$id};
8377         if (defined($running)) {
8378             next if $d->{template};
8379             next if $running && $d->{status} ne 'running';
8380             next if !$running && $d->{status} eq 'running';
8381         }
8382         push @$res, $id;
8383
8384     }
8385     return $res;
8386 };
8387
8388 sub complete_vmid {
8389     return &$complete_vmid_full();
8390 }
8391
8392 sub complete_vmid_stopped {
8393     return &$complete_vmid_full(0);
8394 }
8395
8396 sub complete_vmid_running {
8397     return &$complete_vmid_full(1);
8398 }
8399
8400 sub complete_storage {
8401
8402     my $cfg = PVE::Storage::config();
8403     my $ids = $cfg->{ids};
8404
8405     my $res = [];
8406     foreach my $sid (keys %$ids) {
8407         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8408         next if !$ids->{$sid}->{content}->{images};
8409         push @$res, $sid;
8410     }
8411
8412     return $res;
8413 }
8414
8415 sub complete_migration_storage {
8416     my ($cmd, $param, $current_value, $all_args) = @_;
8417
8418     my $targetnode = @$all_args[1];
8419
8420     my $cfg = PVE::Storage::config();
8421     my $ids = $cfg->{ids};
8422
8423     my $res = [];
8424     foreach my $sid (keys %$ids) {
8425         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8426         next if !$ids->{$sid}->{content}->{images};
8427         push @$res, $sid;
8428     }
8429
8430     return $res;
8431 }
8432
8433 sub vm_is_paused {
8434     my ($vmid) = @_;
8435     my $qmpstatus = eval {
8436         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8437         mon_cmd($vmid, "query-status");
8438     };
8439     warn "$@\n" if $@;
8440     return $qmpstatus && $qmpstatus->{status} eq "paused";
8441 }
8442
8443 sub check_volume_storage_type {
8444     my ($storecfg, $vol) = @_;
8445
8446     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8447     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8448     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8449
8450     die "storage '$storeid' does not support content-type '$vtype'\n"
8451         if !$scfg->{content}->{$vtype};
8452
8453     return 1;
8454 }
8455
8456 sub add_nets_bridge_fdb {
8457     my ($conf, $vmid) = @_;
8458
8459     for my $opt (keys %$conf) {
8460         next if $opt !~ m/^net(\d+)$/;
8461         my $iface = "tap${vmid}i$1";
8462         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8463         my $net = parse_net($conf->{$opt}, 1) or next;
8464
8465         my $mac = $net->{macaddr};
8466         if (!$mac) {
8467             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8468                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8469             next;
8470         }
8471
8472         my $bridge = $net->{bridge};
8473         if ($have_sdn) {
8474             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8475         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8476             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8477         }
8478     }
8479 }
8480
8481 sub del_nets_bridge_fdb {
8482     my ($conf, $vmid) = @_;
8483
8484     for my $opt (keys %$conf) {
8485         next if $opt !~ m/^net(\d+)$/;
8486         my $iface = "tap${vmid}i$1";
8487
8488         my $net = parse_net($conf->{$opt}) or next;
8489         my $mac = $net->{macaddr} or next;
8490
8491         my $bridge = $net->{bridge};
8492         if ($have_sdn) {
8493             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $bridge, $net->{firewall});
8494         } elsif (-d "/sys/class/net/$bridge/bridge") { # avoid fdb management with OVS for now
8495             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8496         }
8497     }
8498 }
8499
8500 1;