PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::CpuSet;
  32 use PVE::DataCenterConfig;
  33 use PVE::Exception qw(raise raise_param_exc);
  34 use PVE::Format qw(render_duration render_bytes);
  35 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  36 use PVE::INotify;
  37 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  38 use PVE::ProcFSTools;
  39 use PVE::PBSClient;
  40 use PVE::RESTEnvironment qw(log_warn);
  41 use PVE::RPCEnvironment;
  42 use PVE::Storage;
  43 use PVE::SysFSTools;
  44 use PVE::Systemd;
  45 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  46
  47 use PVE::QMPClient;
  48 use PVE::QemuConfig;
  49 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout windows_version);
  50 use PVE::QemuServer::Cloudinit;
  51 use PVE::QemuServer::CGroup;
  52 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  53 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  54 use PVE::QemuServer::Machine;
  55 use PVE::QemuServer::Memory;
  56 use PVE::QemuServer::Monitor qw(mon_cmd);
  57 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  58 use PVE::QemuServer::USB qw(parse_usb_device);
  59
  60 my $have_sdn;
  61 eval {
  62     require PVE::Network::SDN::Zones;
  63     $have_sdn = 1;
  64 };
  65
  66 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  67 my $OVMF = {
  68     x86_64 => {
  69         '4m-no-smm' => [
  70             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  71             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  72         ],
  73         '4m-no-smm-ms' => [
  74             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  75             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  76         ],
  77         '4m' => [
  78             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  79             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  80         ],
  81         '4m-ms' => [
  82             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  83             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  84         ],
  85         default => [
  86             "$EDK2_FW_BASE/OVMF_CODE.fd",
  87             "$EDK2_FW_BASE/OVMF_VARS.fd",
  88         ],
  89     },
  90     aarch64 => {
  91         default => [
  92             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  93             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  94         ],
  95     },
  96 };
  97
  98 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  99
 100 # Note about locking: we use flock on the config file protect against concurent actions.
 101 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 102 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 103 # But you can ignore this kind of lock with the --skiplock flag.
 104
 105 cfs_register_file('/qemu-server/',
 106                   \&parse_vm_config,
 107                   \&write_vm_config);
 108
 109 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 110     description => "Some command save/restore state from this location.",
 111     type => 'string',
 112     maxLength => 128,
 113     optional => 1,
 114 });
 115
 116 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 117         description => "Specifies the Qemu machine type.",
 118         type => 'string',
 119         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 120         maxLength => 40,
 121         optional => 1,
 122 });
 123
 124 #no warnings 'redefine';
 125
 126 my $nodename_cache;
 127 sub nodename {
 128     $nodename_cache //= PVE::INotify::nodename();
 129     return $nodename_cache;
 130 }
 131
 132 my $watchdog_fmt = {
 133     model => {
 134         default_key => 1,
 135         type => 'string',
 136         enum => [qw(i6300esb ib700)],
 137         description => "Watchdog type to emulate.",
 138         default => 'i6300esb',
 139         optional => 1,
 140     },
 141     action => {
 142         type => 'string',
 143         enum => [qw(reset shutdown poweroff pause debug none)],
 144         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 145         optional => 1,
 146     },
 147 };
 148 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 149
 150 my $agent_fmt = {
 151     enabled => {
 152         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 153         type => 'boolean',
 154         default => 0,
 155         default_key => 1,
 156     },
 157     fstrim_cloned_disks => {
 158         description => "Run fstrim after moving a disk or migrating the VM.",
 159         type => 'boolean',
 160         optional => 1,
 161         default => 0
 162     },
 163     type => {
 164         description => "Select the agent type",
 165         type => 'string',
 166         default => 'virtio',
 167         optional => 1,
 168         enum => [qw(virtio isa)],
 169     },
 170 };
 171
 172 my $vga_fmt = {
 173     type => {
 174         description => "Select the VGA type.",
 175         type => 'string',
 176         default => 'std',
 177         optional => 1,
 178         default_key => 1,
 179         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 180     },
 181     memory => {
 182         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 183         type => 'integer',
 184         optional => 1,
 185         minimum => 4,
 186         maximum => 512,
 187     },
 188 };
 189
 190 my $ivshmem_fmt = {
 191     size => {
 192         type => 'integer',
 193         minimum => 1,
 194         description => "The size of the file in MB.",
 195     },
 196     name => {
 197         type => 'string',
 198         pattern => '[a-zA-Z0-9\-]+',
 199         optional => 1,
 200         format_description => 'string',
 201         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 202     },
 203 };
 204
 205 my $audio_fmt = {
 206     device => {
 207         type => 'string',
 208         enum => [qw(ich9-intel-hda intel-hda AC97)],
 209         description =>  "Configure an audio device."
 210     },
 211     driver =>  {
 212         type => 'string',
 213         enum => ['spice', 'none'],
 214         default => 'spice',
 215         optional => 1,
 216         description => "Driver backend for the audio device."
 217     },
 218 };
 219
 220 my $spice_enhancements_fmt = {
 221     foldersharing => {
 222         type => 'boolean',
 223         optional => 1,
 224         default => '0',
 225         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 226     },
 227     videostreaming =>  {
 228         type => 'string',
 229         enum => ['off', 'all', 'filter'],
 230         default => 'off',
 231         optional => 1,
 232         description => "Enable video streaming. Uses compression for detected video streams."
 233     },
 234 };
 235
 236 my $rng_fmt = {
 237     source => {
 238         type => 'string',
 239         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 240         default_key => 1,
 241         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 242             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 243             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 244             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 245             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 246             ." a hardware RNG from the host.",
 247     },
 248     max_bytes => {
 249         type => 'integer',
 250         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 251             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 252             ." `0` to disable limiting (potentially dangerous!).",
 253         optional => 1,
 254
 255         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 256         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 257         # reading from /dev/urandom
 258         default => 1024,
 259     },
 260     period => {
 261         type => 'integer',
 262         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 263             ." the guest to retrieve another 'max_bytes' of entropy.",
 264         optional => 1,
 265         default => 1000,
 266     },
 267 };
 268
 269 my $meta_info_fmt = {
 270     'ctime' => {
 271         type => 'integer',
 272         description => "The guest creation timestamp as UNIX epoch time",
 273         minimum => 0,
 274         optional => 1,
 275     },
 276     'creation-qemu' => {
 277         type => 'string',
 278         description => "The QEMU (machine) version from the time this VM was created.",
 279         pattern => '\d+(\.\d+)+',
 280         optional => 1,
 281     },
 282 };
 283
 284 my $confdesc = {
 285     onboot => {
 286         optional => 1,
 287         type => 'boolean',
 288         description => "Specifies whether a VM will be started during system bootup.",
 289         default => 0,
 290     },
 291     autostart => {
 292         optional => 1,
 293         type => 'boolean',
 294         description => "Automatic restart after crash (currently ignored).",
 295         default => 0,
 296     },
 297     hotplug => {
 298         optional => 1,
 299         type => 'string', format => 'pve-hotplug-features',
 300         description => "Selectively enable hotplug features. This is a comma separated list of"
 301             ." hotplug features: 'network', 'disk', 'cpu', 'memory', 'usb' and 'cloudinit'. Use '0' to disable"
 302             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`."
 303             ." USB hotplugging is possible for guests with machine version >= 7.1 and ostype l26 or"
 304             ." windows > 7.",
 305         default => 'network,disk,usb',
 306     },
 307     reboot => {
 308         optional => 1,
 309         type => 'boolean',
 310         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 311         default => 1,
 312     },
 313     lock => {
 314         optional => 1,
 315         type => 'string',
 316         description => "Lock/unlock the VM.",
 317         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 318     },
 319     cpulimit => {
 320         optional => 1,
 321         type => 'number',
 322         description => "Limit of CPU usage.",
 323         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 324             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 325         minimum => 0,
 326         maximum => 128,
 327         default => 0,
 328     },
 329     cpuunits => {
 330         optional => 1,
 331         type => 'integer',
 332         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 333         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 334             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 335             ." weights of all the other running VMs.",
 336         minimum => 1,
 337         maximum => 262144,
 338         default => 'cgroup v1: 1024, cgroup v2: 100',
 339     },
 340     memory => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 344             ." you use the balloon device.",
 345         minimum => 16,
 346         default => 512,
 347     },
 348     balloon => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 352         minimum => 0,
 353     },
 354     shares => {
 355         optional => 1,
 356         type => 'integer',
 357         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 358             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 359             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 360         minimum => 0,
 361         maximum => 50000,
 362         default => 1000,
 363     },
 364     keyboard => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Keyboard layout for VNC server. This option is generally not required and"
 368         ." is often better handled from within the guest OS.",
 369         enum => PVE::Tools::kvmkeymaplist(),
 370         default => undef,
 371     },
 372     name => {
 373         optional => 1,
 374         type => 'string', format => 'dns-name',
 375         description => "Set a name for the VM. Only used on the configuration web interface.",
 376     },
 377     scsihw => {
 378         optional => 1,
 379         type => 'string',
 380         description => "SCSI controller model",
 381         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 382         default => 'lsi',
 383     },
 384     description => {
 385         optional => 1,
 386         type => 'string',
 387         description => "Description for the VM. Shown in the web-interface VM's summary."
 388             ." This is saved as comment inside the configuration file.",
 389         maxLength => 1024 * 8,
 390     },
 391     ostype => {
 392         optional => 1,
 393         type => 'string',
 394         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 395         description => "Specify guest operating system.",
 396         verbose_description => <<EODESC,
 397 Specify guest operating system. This is used to enable special
 398 optimization/features for specific operating systems:
 399
 400 [horizontal]
 401 other;; unspecified OS
 402 wxp;; Microsoft Windows XP
 403 w2k;; Microsoft Windows 2000
 404 w2k3;; Microsoft Windows 2003
 405 w2k8;; Microsoft Windows 2008
 406 wvista;; Microsoft Windows Vista
 407 win7;; Microsoft Windows 7
 408 win8;; Microsoft Windows 8/2012/2012r2
 409 win10;; Microsoft Windows 10/2016/2019
 410 win11;; Microsoft Windows 11/2022
 411 l24;; Linux 2.4 Kernel
 412 l26;; Linux 2.6 - 5.X Kernel
 413 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 414 EODESC
 415     },
 416     boot => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-boot',
 419         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 420             ." key or 'legacy=' is deprecated.",
 421     },
 422     bootdisk => {
 423         optional => 1,
 424         type => 'string', format => 'pve-qm-bootdisk',
 425         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 426         pattern => '(ide|sata|scsi|virtio)\d+',
 427     },
 428     smp => {
 429         optional => 1,
 430         type => 'integer',
 431         description => "The number of CPUs. Please use option -sockets instead.",
 432         minimum => 1,
 433         default => 1,
 434     },
 435     sockets => {
 436         optional => 1,
 437         type => 'integer',
 438         description => "The number of CPU sockets.",
 439         minimum => 1,
 440         default => 1,
 441     },
 442     cores => {
 443         optional => 1,
 444         type => 'integer',
 445         description => "The number of cores per socket.",
 446         minimum => 1,
 447         default => 1,
 448     },
 449     numa => {
 450         optional => 1,
 451         type => 'boolean',
 452         description => "Enable/disable NUMA.",
 453         default => 0,
 454     },
 455     hugepages => {
 456         optional => 1,
 457         type => 'string',
 458         description => "Enable/disable hugepages memory.",
 459         enum => [qw(any 2 1024)],
 460     },
 461     keephugepages => {
 462         optional => 1,
 463         type => 'boolean',
 464         default => 0,
 465         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 466             ." after VM shutdown and can be used for subsequent starts.",
 467     },
 468     vcpus => {
 469         optional => 1,
 470         type => 'integer',
 471         description => "Number of hotplugged vcpus.",
 472         minimum => 1,
 473         default => 0,
 474     },
 475     acpi => {
 476         optional => 1,
 477         type => 'boolean',
 478         description => "Enable/disable ACPI.",
 479         default => 1,
 480     },
 481     agent => {
 482         optional => 1,
 483         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 484         type => 'string',
 485         format => $agent_fmt,
 486     },
 487     kvm => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Enable/disable KVM hardware virtualization.",
 491         default => 1,
 492     },
 493     tdf => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Enable/disable time drift fix.",
 497         default => 0,
 498     },
 499     localtime => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 503             ." the `ostype` indicates a Microsoft Windows OS.",
 504     },
 505     freeze => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 509     },
 510     vga => {
 511         optional => 1,
 512         type => 'string', format => $vga_fmt,
 513         description => "Configure the VGA hardware.",
 514         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 515             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 516             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 517             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 518             ." display server. For win* OS you can select how many independent displays you want,"
 519             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 520             ." using a serial device as terminal.",
 521     },
 522     watchdog => {
 523         optional => 1,
 524         type => 'string', format => 'pve-qm-watchdog',
 525         description => "Create a virtual hardware watchdog device.",
 526         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 527             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 528             ." else the watchdog will reset the guest (or execute the respective action specified)",
 529     },
 530     startdate => {
 531         optional => 1,
 532         type => 'string',
 533         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 534         description => "Set the initial date of the real time clock. Valid format for date are:"
 535             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 536         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 537         default => 'now',
 538     },
 539     startup =>  get_standard_option('pve-startup-order'),
 540     template => {
 541         optional => 1,
 542         type => 'boolean',
 543         description => "Enable/disable Template.",
 544         default => 0,
 545     },
 546     args => {
 547         optional => 1,
 548         type => 'string',
 549         description => "Arbitrary arguments passed to kvm.",
 550         verbose_description => <<EODESCR,
 551 Arbitrary arguments passed to kvm, for example:
 552
 553 args: -no-reboot -no-hpet
 554
 555 NOTE: this option is for experts only.
 556 EODESCR
 557     },
 558     tablet => {
 559         optional => 1,
 560         type => 'boolean',
 561         default => 1,
 562         description => "Enable/disable the USB tablet device.",
 563         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 564             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 565             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 566             ." may consider disabling this to save some context switches. This is turned off by"
 567             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 568     },
 569     migrate_speed => {
 570         optional => 1,
 571         type => 'integer',
 572         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 573         minimum => 0,
 574         default => 0,
 575     },
 576     migrate_downtime => {
 577         optional => 1,
 578         type => 'number',
 579         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 580         minimum => 0,
 581         default => 0.1,
 582     },
 583     cdrom => {
 584         optional => 1,
 585         type => 'string', format => 'pve-qm-ide',
 586         typetext => '<volume>',
 587         description => "This is an alias for option -ide2",
 588     },
 589     cpu => {
 590         optional => 1,
 591         description => "Emulated CPU type.",
 592         type => 'string',
 593         format => 'pve-vm-cpu-conf',
 594     },
 595     parent => get_standard_option('pve-snapshot-name', {
 596         optional => 1,
 597         description => "Parent snapshot name. This is used internally, and should not be modified.",
 598     }),
 599     snaptime => {
 600         optional => 1,
 601         description => "Timestamp for snapshots.",
 602         type => 'integer',
 603         minimum => 0,
 604     },
 605     vmstate => {
 606         optional => 1,
 607         type => 'string', format => 'pve-volume-id',
 608         description => "Reference to a volume which stores the VM state. This is used internally"
 609             ." for snapshots.",
 610     },
 611     vmstatestorage => get_standard_option('pve-storage-id', {
 612         description => "Default storage for VM state volumes/files.",
 613         optional => 1,
 614     }),
 615     runningmachine => get_standard_option('pve-qemu-machine', {
 616         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 617             ." for snapshots.",
 618     }),
 619     runningcpu => {
 620         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 621             ." internally for snapshots.",
 622         optional => 1,
 623         type => 'string',
 624         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 625         format_description => 'QEMU -cpu parameter'
 626     },
 627     machine => get_standard_option('pve-qemu-machine'),
 628     arch => {
 629         description => "Virtual processor architecture. Defaults to the host.",
 630         optional => 1,
 631         type => 'string',
 632         enum => [qw(x86_64 aarch64)],
 633     },
 634     smbios1 => {
 635         description => "Specify SMBIOS type 1 fields.",
 636         type => 'string', format => 'pve-qm-smbios1',
 637         maxLength => 512,
 638         optional => 1,
 639     },
 640     protection => {
 641         optional => 1,
 642         type => 'boolean',
 643         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 644             ." remove disk operations.",
 645         default => 0,
 646     },
 647     bios => {
 648         optional => 1,
 649         type => 'string',
 650         enum => [ qw(seabios ovmf) ],
 651         description => "Select BIOS implementation.",
 652         default => 'seabios',
 653     },
 654     vmgenid => {
 655         type => 'string',
 656         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 657         format_description => 'UUID',
 658         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 659             ." to disable explicitly.",
 660         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 661             ." value identifier to the guest OS. This allows to notify the guest operating system"
 662             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 663             ." execution or creation from a template). The guest operating system notices the"
 664             ." change, and is then able to react as appropriate by marking its copies of"
 665             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 666             ."Note that auto-creation only works when done through API/CLI create or update methods"
 667             .", but not when manually editing the config file.",
 668         default => "1 (autogenerated)",
 669         optional => 1,
 670     },
 671     hookscript => {
 672         type => 'string',
 673         format => 'pve-volume-id',
 674         optional => 1,
 675         description => "Script that will be executed during various steps in the vms lifetime.",
 676     },
 677     ivshmem => {
 678         type => 'string',
 679         format => $ivshmem_fmt,
 680         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 681             ." the host.",
 682         optional => 1,
 683     },
 684     audio0 => {
 685         type => 'string',
 686         format => $audio_fmt,
 687         description => "Configure a audio device, useful in combination with QXL/Spice.",
 688         optional => 1
 689     },
 690     spice_enhancements => {
 691         type => 'string',
 692         format => $spice_enhancements_fmt,
 693         description => "Configure additional enhancements for SPICE.",
 694         optional => 1
 695     },
 696     tags => {
 697         type => 'string', format => 'pve-tag-list',
 698         description => 'Tags of the VM. This is only meta information.',
 699         optional => 1,
 700     },
 701     rng0 => {
 702         type => 'string',
 703         format => $rng_fmt,
 704         description => "Configure a VirtIO-based Random Number Generator.",
 705         optional => 1,
 706     },
 707     meta => {
 708         type => 'string',
 709         format => $meta_info_fmt,
 710         description => "Some (read-only) meta-information about this guest.",
 711         optional => 1,
 712     },
 713     affinity => {
 714         type => 'string', format => 'pve-cpuset',
 715         description => "List of host cores used to execute guest processes.",
 716         optional => 1,
 717     },
 718 };
 719
 720 my $cicustom_fmt = {
 721     meta => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all meta data passed to the VM via"
 725             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 726         format => 'pve-volume-id',
 727         format_description => 'volume',
 728     },
 729     network => {
 730         type => 'string',
 731         optional => 1,
 732         description => 'Specify a custom file containing all network data passed to the VM via'
 733             .' cloud-init.',
 734         format => 'pve-volume-id',
 735         format_description => 'volume',
 736     },
 737     user => {
 738         type => 'string',
 739         optional => 1,
 740         description => 'Specify a custom file containing all user data passed to the VM via'
 741             .' cloud-init.',
 742         format => 'pve-volume-id',
 743         format_description => 'volume',
 744     },
 745     vendor => {
 746     type => 'string',
 747     optional => 1,
 748     description => 'Specify a custom file containing all vendor data passed to the VM via'
 749      .' cloud-init.',
 750     format => 'pve-volume-id',
 751     format_description => 'volume',
 752     },
 753 };
 754 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 755
 756 my $confdesc_cloudinit = {
 757     citype => {
 758         optional => 1,
 759         type => 'string',
 760         description => 'Specifies the cloud-init configuration format. The default depends on the'
 761             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 762             .' and `configdrive2` for windows.',
 763         enum => ['configdrive2', 'nocloud', 'opennebula'],
 764     },
 765     ciuser => {
 766         optional => 1,
 767         type => 'string',
 768         description => "cloud-init: User name to change ssh keys and password for instead of the"
 769             ." image's configured default user.",
 770     },
 771     cipassword => {
 772         optional => 1,
 773         type => 'string',
 774         description => 'cloud-init: Password to assign the user. Using this is generally not'
 775             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 776             .' support hashed passwords.',
 777     },
 778     cicustom => {
 779         optional => 1,
 780         type => 'string',
 781         description => 'cloud-init: Specify custom files to replace the automatically generated'
 782             .' ones at start.',
 783         format => 'pve-qm-cicustom',
 784     },
 785     searchdomain => {
 786         optional => 1,
 787         type => 'string',
 788         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 789             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 790             .' are set.',
 791     },
 792     nameserver => {
 793         optional => 1,
 794         type => 'string', format => 'address-list',
 795         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 796             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 797             .' are set.',
 798     },
 799     sshkeys => {
 800         optional => 1,
 801         type => 'string',
 802         format => 'urlencoded',
 803         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 804     },
 805 };
 806
 807 # what about other qemu settings ?
 808 #cpu => 'string',
 809 #machine => 'string',
 810 #fda => 'file',
 811 #fdb => 'file',
 812 #mtdblock => 'file',
 813 #sd => 'file',
 814 #pflash => 'file',
 815 #snapshot => 'bool',
 816 #bootp => 'file',
 817 ##tftp => 'dir',
 818 ##smb => 'dir',
 819 #kernel => 'file',
 820 #append => 'string',
 821 #initrd => 'file',
 822 ##soundhw => 'string',
 823
 824 while (my ($k, $v) = each %$confdesc) {
 825     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 826 }
 827
 828 my $MAX_USB_DEVICES = 14;
 829 my $MAX_NETS = 32;
 830 my $MAX_SERIAL_PORTS = 4;
 831 my $MAX_PARALLEL_PORTS = 3;
 832 my $MAX_NUMA = 8;
 833
 834 my $numa_fmt = {
 835     cpus => {
 836         type => "string",
 837         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 838         description => "CPUs accessing this NUMA node.",
 839         format_description => "id[-id];...",
 840     },
 841     memory => {
 842         type => "number",
 843         description => "Amount of memory this NUMA node provides.",
 844         optional => 1,
 845     },
 846     hostnodes => {
 847         type => "string",
 848         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 849         description => "Host NUMA nodes to use.",
 850         format_description => "id[-id];...",
 851         optional => 1,
 852     },
 853     policy => {
 854         type => 'string',
 855         enum => [qw(preferred bind interleave)],
 856         description => "NUMA allocation policy.",
 857         optional => 1,
 858     },
 859 };
 860 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 861 my $numadesc = {
 862     optional => 1,
 863     type => 'string', format => $numa_fmt,
 864     description => "NUMA topology.",
 865 };
 866 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 867
 868 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 869     $confdesc->{"numa$i"} = $numadesc;
 870 }
 871
 872 my $nic_model_list = [
 873     'e1000',
 874     'e1000-82540em',
 875     'e1000-82544gc',
 876     'e1000-82545em',
 877     'e1000e',
 878     'i82551',
 879     'i82557b',
 880     'i82559er',
 881     'ne2k_isa',
 882     'ne2k_pci',
 883     'pcnet',
 884     'rtl8139',
 885     'virtio',
 886     'vmxnet3',
 887 ];
 888 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 889
 890 my $net_fmt_bridge_descr = <<__EOD__;
 891 Bridge to attach the network device to. The Proxmox VE standard bridge
 892 is called 'vmbr0'.
 893
 894 If you do not specify a bridge, we create a kvm user (NATed) network
 895 device, which provides DHCP and DNS services. The following addresses
 896 are used:
 897
 898  10.0.2.2   Gateway
 899  10.0.2.3   DNS Server
 900  10.0.2.4   SMB Server
 901
 902 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 903 __EOD__
 904
 905 my $net_fmt = {
 906     macaddr  => get_standard_option('mac-addr', {
 907         description => "MAC address. That address must be unique withing your network. This is"
 908             ." automatically generated if not specified.",
 909     }),
 910     model => {
 911         type => 'string',
 912         description => "Network Card Model. The 'virtio' model provides the best performance with"
 913             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 914             ." best to use 'e1000'.",
 915         enum => $nic_model_list,
 916         default_key => 1,
 917     },
 918     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 919     bridge => get_standard_option('pve-bridge-id', {
 920         description => $net_fmt_bridge_descr,
 921         optional => 1,
 922     }),
 923     queues => {
 924         type => 'integer',
 925         minimum => 0, maximum => 16,
 926         description => 'Number of packet queues to be used on the device.',
 927         optional => 1,
 928     },
 929     rate => {
 930         type => 'number',
 931         minimum => 0,
 932         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 933         optional => 1,
 934     },
 935     tag => {
 936         type => 'integer',
 937         minimum => 1, maximum => 4094,
 938         description => 'VLAN tag to apply to packets on this interface.',
 939         optional => 1,
 940     },
 941     trunks => {
 942         type => 'string',
 943         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 944         description => 'VLAN trunks to pass through this interface.',
 945         format_description => 'vlanid[;vlanid...]',
 946         optional => 1,
 947     },
 948     firewall => {
 949         type => 'boolean',
 950         description => 'Whether this interface should be protected by the firewall.',
 951         optional => 1,
 952     },
 953     link_down => {
 954         type => 'boolean',
 955         description => 'Whether this interface should be disconnected (like pulling the plug).',
 956         optional => 1,
 957     },
 958     mtu => {
 959         type => 'integer',
 960         minimum => 1, maximum => 65520,
 961         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 962         optional => 1,
 963     },
 964 };
 965
 966 my $netdesc = {
 967     optional => 1,
 968     type => 'string', format => $net_fmt,
 969     description => "Specify network devices.",
 970 };
 971
 972 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 973
 974 my $ipconfig_fmt = {
 975     ip => {
 976         type => 'string',
 977         format => 'pve-ipv4-config',
 978         format_description => 'IPv4Format/CIDR',
 979         description => 'IPv4 address in CIDR format.',
 980         optional => 1,
 981         default => 'dhcp',
 982     },
 983     gw => {
 984         type => 'string',
 985         format => 'ipv4',
 986         format_description => 'GatewayIPv4',
 987         description => 'Default gateway for IPv4 traffic.',
 988         optional => 1,
 989         requires => 'ip',
 990     },
 991     ip6 => {
 992         type => 'string',
 993         format => 'pve-ipv6-config',
 994         format_description => 'IPv6Format/CIDR',
 995         description => 'IPv6 address in CIDR format.',
 996         optional => 1,
 997         default => 'dhcp',
 998     },
 999     gw6 => {
1000         type => 'string',
1001         format => 'ipv6',
1002         format_description => 'GatewayIPv6',
1003         description => 'Default gateway for IPv6 traffic.',
1004         optional => 1,
1005         requires => 'ip6',
1006     },
1007 };
1008 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1009 my $ipconfigdesc = {
1010     optional => 1,
1011     type => 'string', format => 'pve-qm-ipconfig',
1012     description => <<'EODESCR',
1013 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1014
1015 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1016
1017 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1018 gateway should be provided.
1019 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1020 cloud-init 19.4 or newer.
1021
1022 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1023 dhcp on IPv4.
1024 EODESCR
1025 };
1026 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1027
1028 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1029     $confdesc->{"net$i"} = $netdesc;
1030     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1031 }
1032
1033 foreach my $key (keys %$confdesc_cloudinit) {
1034     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1035 }
1036
1037 PVE::JSONSchema::register_format('pve-cpuset', \&pve_verify_cpuset);
1038 sub pve_verify_cpuset {
1039     my ($set_text, $noerr) = @_;
1040
1041     my ($count, $members) = eval { PVE::CpuSet::parse_cpuset($set_text) };
1042
1043     if ($@) {
1044         return if $noerr;
1045         die "unable to parse cpuset option\n";
1046     }
1047
1048     return PVE::CpuSet->new($members)->short_string();
1049 }
1050
1051 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1052 sub verify_volume_id_or_qm_path {
1053     my ($volid, $noerr) = @_;
1054
1055     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1056
1057     return verify_volume_id_or_absolute_path($volid, $noerr);
1058 }
1059
1060 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1061 sub verify_volume_id_or_absolute_path {
1062     my ($volid, $noerr) = @_;
1063
1064     return $volid if $volid =~ m|^/|;
1065
1066     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1067     if ($@) {
1068         return if $noerr;
1069         die $@;
1070     }
1071     return $volid;
1072 }
1073
1074 my $usb_fmt = {
1075     host => {
1076         default_key => 1,
1077         type => 'string', format => 'pve-qm-usb-device',
1078         format_description => 'HOSTUSBDEVICE|spice',
1079         description => <<EODESCR,
1080 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1081
1082  'bus-port(.port)*' (decimal numbers) or
1083  'vendor_id:product_id' (hexadeciaml numbers) or
1084  'spice'
1085
1086 You can use the 'lsusb -t' command to list existing usb devices.
1087
1088 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1089 machines - use with special care.
1090
1091 The value 'spice' can be used to add a usb redirection devices for spice.
1092 EODESCR
1093     },
1094     usb3 => {
1095         optional => 1,
1096         type => 'boolean',
1097         description => "Specifies whether if given host option is a USB3 device or port."
1098             ." For modern guests (machine version >= 7.1 and ostype l26 and windows > 7), this flag"
1099             ." is irrelevant (all devices are plugged into a xhci controller).",
1100         default => 0,
1101     },
1102 };
1103
1104 my $usbdesc = {
1105     optional => 1,
1106     type => 'string', format => $usb_fmt,
1107     description => "Configure an USB device (n is 0 to 4, for machine version >= 7.1 and ostype"
1108         ." l26 or windows > 7, n can be up to 14).",
1109 };
1110 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1111
1112 my $serialdesc = {
1113         optional => 1,
1114         type => 'string',
1115         pattern => '(/dev/.+|socket)',
1116         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1117         verbose_description =>  <<EODESCR,
1118 Create a serial device inside the VM (n is 0 to 3), and pass through a
1119 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1120 host side (use 'qm terminal' to open a terminal connection).
1121
1122 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1123 use with special care.
1124
1125 CAUTION: Experimental! User reported problems with this option.
1126 EODESCR
1127 };
1128
1129 my $paralleldesc= {
1130         optional => 1,
1131         type => 'string',
1132         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1133         description =>  "Map host parallel devices (n is 0 to 2).",
1134         verbose_description =>  <<EODESCR,
1135 Map host parallel devices (n is 0 to 2).
1136
1137 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1138 machines - use with special care.
1139
1140 CAUTION: Experimental! User reported problems with this option.
1141 EODESCR
1142 };
1143
1144 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1145     $confdesc->{"parallel$i"} = $paralleldesc;
1146 }
1147
1148 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1149     $confdesc->{"serial$i"} = $serialdesc;
1150 }
1151
1152 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1153     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1154 }
1155
1156 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1157     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1158 }
1159
1160 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1161     $confdesc->{"usb$i"} = $usbdesc;
1162 }
1163
1164 my $boot_fmt = {
1165     legacy => {
1166         optional => 1,
1167         default_key => 1,
1168         type => 'string',
1169         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1170                      . " Deprecated, use 'order=' instead.",
1171         pattern => '[acdn]{1,4}',
1172         format_description => "[acdn]{1,4}",
1173
1174         # note: this is also the fallback if boot: is not given at all
1175         default => 'cdn',
1176     },
1177     order => {
1178         optional => 1,
1179         type => 'string',
1180         format => 'pve-qm-bootdev-list',
1181         format_description => "device[;device...]",
1182         description => <<EODESC,
1183 The guest will attempt to boot from devices in the order they appear here.
1184
1185 Disks, optical drives and passed-through storage USB devices will be directly
1186 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1187 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1188
1189 Note that only devices in this list will be marked as bootable and thus loaded
1190 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1191 (e.g. software-raid), you need to specify all of them here.
1192
1193 Overrides the deprecated 'legacy=[acdn]*' value when given.
1194 EODESC
1195     },
1196 };
1197 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1198
1199 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1200 sub verify_bootdev {
1201     my ($dev, $noerr) = @_;
1202
1203     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1204     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1205
1206     my $check = sub {
1207         my ($base) = @_;
1208         return 0 if $dev !~ m/^$base\d+$/;
1209         return 0 if !$confdesc->{$dev};
1210         return 1;
1211     };
1212
1213     return $dev if $check->("net");
1214     return $dev if $check->("usb");
1215     return $dev if $check->("hostpci");
1216
1217     return if $noerr;
1218     die "invalid boot device '$dev'\n";
1219 }
1220
1221 sub print_bootorder {
1222     my ($devs) = @_;
1223     return "" if !@$devs;
1224     my $data = { order => join(';', @$devs) };
1225     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1226 }
1227
1228 my $kvm_api_version = 0;
1229
1230 sub kvm_version {
1231     return $kvm_api_version if $kvm_api_version;
1232
1233     open my $fh, '<', '/dev/kvm' or return;
1234
1235     # 0xae00 => KVM_GET_API_VERSION
1236     $kvm_api_version = ioctl($fh, 0xae00, 0);
1237     close($fh);
1238
1239     return $kvm_api_version;
1240 }
1241
1242 my $kvm_user_version = {};
1243 my $kvm_mtime = {};
1244
1245 sub kvm_user_version {
1246     my ($binary) = @_;
1247
1248     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1249     my $st = stat($binary);
1250
1251     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1252     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1253         $cachedmtime == $st->mtime;
1254
1255     $kvm_user_version->{$binary} = 'unknown';
1256     $kvm_mtime->{$binary} = $st->mtime;
1257
1258     my $code = sub {
1259         my $line = shift;
1260         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1261             $kvm_user_version->{$binary} = $2;
1262         }
1263     };
1264
1265     eval { run_command([$binary, '--version'], outfunc => $code); };
1266     warn $@ if $@;
1267
1268     return $kvm_user_version->{$binary};
1269
1270 }
1271 my sub extract_version {
1272     my ($machine_type, $version) = @_;
1273     $version = kvm_user_version() if !defined($version);
1274     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1275 }
1276
1277 sub kernel_has_vhost_net {
1278     return -c '/dev/vhost-net';
1279 }
1280
1281 sub option_exists {
1282     my $key = shift;
1283     return defined($confdesc->{$key});
1284 }
1285
1286 my $cdrom_path;
1287 sub get_cdrom_path {
1288
1289     return  $cdrom_path if $cdrom_path;
1290
1291     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1292     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1293     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1294 }
1295
1296 sub get_iso_path {
1297     my ($storecfg, $vmid, $cdrom) = @_;
1298
1299     if ($cdrom eq 'cdrom') {
1300         return get_cdrom_path();
1301     } elsif ($cdrom eq 'none') {
1302         return '';
1303     } elsif ($cdrom =~ m|^/|) {
1304         return $cdrom;
1305     } else {
1306         return PVE::Storage::path($storecfg, $cdrom);
1307     }
1308 }
1309
1310 # try to convert old style file names to volume IDs
1311 sub filename_to_volume_id {
1312     my ($vmid, $file, $media) = @_;
1313
1314      if (!($file eq 'none' || $file eq 'cdrom' ||
1315           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1316
1317         return if $file =~ m|/|;
1318
1319         if ($media && $media eq 'cdrom') {
1320             $file = "local:iso/$file";
1321         } else {
1322             $file = "local:$vmid/$file";
1323         }
1324     }
1325
1326     return $file;
1327 }
1328
1329 sub verify_media_type {
1330     my ($opt, $vtype, $media) = @_;
1331
1332     return if !$media;
1333
1334     my $etype;
1335     if ($media eq 'disk') {
1336         $etype = 'images';
1337     } elsif ($media eq 'cdrom') {
1338         $etype = 'iso';
1339     } else {
1340         die "internal error";
1341     }
1342
1343     return if ($vtype eq $etype);
1344
1345     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1346 }
1347
1348 sub cleanup_drive_path {
1349     my ($opt, $storecfg, $drive) = @_;
1350
1351     # try to convert filesystem paths to volume IDs
1352
1353     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1354         ($drive->{file} !~ m|^/dev/.+|) &&
1355         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1356         ($drive->{file} !~ m/^\d+$/)) {
1357         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1358         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1359             if !$vtype;
1360         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1361         verify_media_type($opt, $vtype, $drive->{media});
1362         $drive->{file} = $volid;
1363     }
1364
1365     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1366 }
1367
1368 sub parse_hotplug_features {
1369     my ($data) = @_;
1370
1371     my $res = {};
1372
1373     return $res if $data eq '0';
1374
1375     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1376
1377     foreach my $feature (PVE::Tools::split_list($data)) {
1378         if ($feature =~ m/^(network|disk|cpu|memory|usb|cloudinit)$/) {
1379             $res->{$1} = 1;
1380         } else {
1381             die "invalid hotplug feature '$feature'\n";
1382         }
1383     }
1384     return $res;
1385 }
1386
1387 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1388 sub pve_verify_hotplug_features {
1389     my ($value, $noerr) = @_;
1390
1391     return $value if parse_hotplug_features($value);
1392
1393     return if $noerr;
1394
1395     die "unable to parse hotplug option\n";
1396 }
1397
1398 sub scsi_inquiry {
1399     my($fh, $noerr) = @_;
1400
1401     my $SG_IO = 0x2285;
1402     my $SG_GET_VERSION_NUM = 0x2282;
1403
1404     my $versionbuf = "\x00" x 8;
1405     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1406     if (!$ret) {
1407         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1408         return;
1409     }
1410     my $version = unpack("I", $versionbuf);
1411     if ($version < 30000) {
1412         die "scsi generic interface too old\n"  if !$noerr;
1413         return;
1414     }
1415
1416     my $buf = "\x00" x 36;
1417     my $sensebuf = "\x00" x 8;
1418     my $cmd = pack("C x3 C x1", 0x12, 36);
1419
1420     # see /usr/include/scsi/sg.h
1421     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1422
1423     my $packet = pack(
1424         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1425     );
1426
1427     $ret = ioctl($fh, $SG_IO, $packet);
1428     if (!$ret) {
1429         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1430         return;
1431     }
1432
1433     my @res = unpack($sg_io_hdr_t, $packet);
1434     if ($res[17] || $res[18]) {
1435         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1436         return;
1437     }
1438
1439     my $res = {};
1440     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1441
1442     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1443     $res->{type} &= 0x1F;
1444
1445     return $res;
1446 }
1447
1448 sub path_is_scsi {
1449     my ($path) = @_;
1450
1451     my $fh = IO::File->new("+<$path") || return;
1452     my $res = scsi_inquiry($fh, 1);
1453     close($fh);
1454
1455     return $res;
1456 }
1457
1458 sub print_tabletdevice_full {
1459     my ($conf, $arch) = @_;
1460
1461     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1462
1463     # we use uhci for old VMs because tablet driver was buggy in older qemu
1464     my $usbbus;
1465     if ($q35 || $arch eq 'aarch64') {
1466         $usbbus = 'ehci';
1467     } else {
1468         $usbbus = 'uhci';
1469     }
1470
1471     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1472 }
1473
1474 sub print_keyboarddevice_full {
1475     my ($conf, $arch) = @_;
1476
1477     return if $arch ne 'aarch64';
1478
1479     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1480 }
1481
1482 my sub get_drive_id {
1483     my ($drive) = @_;
1484     return "$drive->{interface}$drive->{index}";
1485 }
1486
1487 sub print_drivedevice_full {
1488     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1489
1490     my $device = '';
1491     my $maxdev = 0;
1492
1493     my $drive_id = get_drive_id($drive);
1494     if ($drive->{interface} eq 'virtio') {
1495         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1496         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1497         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1498     } elsif ($drive->{interface} eq 'scsi') {
1499
1500         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1501         my $unit = $drive->{index} % $maxdev;
1502         my $devicetype = 'hd';
1503         my $path = '';
1504         if (drive_is_cdrom($drive)) {
1505             $devicetype = 'cd';
1506         } else {
1507             if ($drive->{file} =~ m|^/|) {
1508                 $path = $drive->{file};
1509                 if (my $info = path_is_scsi($path)) {
1510                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1511                         $devicetype = 'block';
1512                     } elsif ($info->{type} == 1) { # tape
1513                         $devicetype = 'generic';
1514                     }
1515                 }
1516             } else {
1517                  $path = PVE::Storage::path($storecfg, $drive->{file});
1518             }
1519
1520             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1521             my $version = extract_version($machine_type, kvm_user_version());
1522             if ($path =~ m/^iscsi\:\/\// &&
1523                !min_version($version, 4, 1)) {
1524                 $devicetype = 'generic';
1525             }
1526         }
1527
1528         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1529             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1530         } else {
1531             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1532                 .",lun=$drive->{index}";
1533         }
1534         $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1537             $device .= ",rotation_rate=1";
1538         }
1539         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1540
1541     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1542         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1543         my $controller = int($drive->{index} / $maxdev);
1544         my $unit = $drive->{index} % $maxdev;
1545         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1546
1547         $device = "ide-$devicetype";
1548         if ($drive->{interface} eq 'ide') {
1549             $device .= ",bus=ide.$controller,unit=$unit";
1550         } else {
1551             $device .= ",bus=ahci$controller.$unit";
1552         }
1553         $device .= ",drive=drive-$drive_id,id=$drive_id";
1554
1555         if ($devicetype eq 'hd') {
1556             if (my $model = $drive->{model}) {
1557                 $model = URI::Escape::uri_unescape($model);
1558                 $device .= ",model=$model";
1559             }
1560             if ($drive->{ssd}) {
1561                 $device .= ",rotation_rate=1";
1562             }
1563         }
1564         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1565     } elsif ($drive->{interface} eq 'usb') {
1566         die "implement me";
1567         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1568     } else {
1569         die "unsupported interface type";
1570     }
1571
1572     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1573
1574     if (my $serial = $drive->{serial}) {
1575         $serial = URI::Escape::uri_unescape($serial);
1576         $device .= ",serial=$serial";
1577     }
1578
1579
1580     return $device;
1581 }
1582
1583 sub get_initiator_name {
1584     my $initiator;
1585
1586     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1587     while (defined(my $line = <$fh>)) {
1588         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1589         $initiator = $1;
1590         last;
1591     }
1592     $fh->close();
1593
1594     return $initiator;
1595 }
1596
1597 sub print_drive_commandline_full {
1598     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1599
1600     my $path;
1601     my $volid = $drive->{file};
1602     my $format = $drive->{format};
1603     my $drive_id = get_drive_id($drive);
1604
1605     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1606     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1607
1608     if (drive_is_cdrom($drive)) {
1609         $path = get_iso_path($storecfg, $vmid, $volid);
1610         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1611     } else {
1612         if ($storeid) {
1613             $path = PVE::Storage::path($storecfg, $volid);
1614             $format //= qemu_img_format($scfg, $volname);
1615         } else {
1616             $path = $volid;
1617             $format //= "raw";
1618         }
1619    }
1620
1621    my $is_rbd = $path =~ m/^rbd:/;
1622
1623     my $opts = '';
1624     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1625     foreach my $o (@qemu_drive_options) {
1626         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1627     }
1628
1629     # snapshot only accepts on|off
1630     if (defined($drive->{snapshot})) {
1631         my $v = $drive->{snapshot} ? 'on' : 'off';
1632         $opts .= ",snapshot=$v";
1633     }
1634
1635     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1636         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1637     }
1638
1639     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1640         my ($dir, $qmpname) = @$type;
1641         if (my $v = $drive->{"mbps$dir"}) {
1642             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1643         }
1644         if (my $v = $drive->{"mbps${dir}_max"}) {
1645             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1646         }
1647         if (my $v = $drive->{"bps${dir}_max_length"}) {
1648             $opts .= ",throttling.bps$qmpname-max-length=$v";
1649         }
1650         if (my $v = $drive->{"iops${dir}"}) {
1651             $opts .= ",throttling.iops$qmpname=$v";
1652         }
1653         if (my $v = $drive->{"iops${dir}_max"}) {
1654             $opts .= ",throttling.iops$qmpname-max=$v";
1655         }
1656         if (my $v = $drive->{"iops${dir}_max_length"}) {
1657             $opts .= ",throttling.iops$qmpname-max-length=$v";
1658         }
1659     }
1660
1661     if ($pbs_name) {
1662         $format = "rbd" if $is_rbd;
1663         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1664             if !$format;
1665         $opts .= ",format=alloc-track,file.driver=$format";
1666     } elsif ($format) {
1667         $opts .= ",format=$format";
1668     }
1669
1670     my $cache_direct = 0;
1671
1672     if (my $cache = $drive->{cache}) {
1673         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1674     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1675         $opts .= ",cache=none";
1676         $cache_direct = 1;
1677     }
1678
1679     # io_uring with cache mode writeback or writethrough on krbd will hang...
1680     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1681
1682     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1683     # sometimes, just plain disable...
1684     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1685
1686     # io_uring causes problems when used with CIFS since kernel 5.15
1687     # Some discussion: https://www.spinics.net/lists/linux-cifs/msg26734.html
1688     my $cifs_no_io_uring = $scfg && $scfg->{type} eq 'cifs';
1689
1690     if (!$drive->{aio}) {
1691         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring && !$cifs_no_io_uring) {
1692             # io_uring supports all cache modes
1693             $opts .= ",aio=io_uring";
1694         } else {
1695             # aio native works only with O_DIRECT
1696             if($cache_direct) {
1697                 $opts .= ",aio=native";
1698             } else {
1699                 $opts .= ",aio=threads";
1700             }
1701         }
1702     }
1703
1704     if (!drive_is_cdrom($drive)) {
1705         my $detectzeroes;
1706         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1707             $detectzeroes = 'off';
1708         } elsif ($drive->{discard}) {
1709             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1710         } else {
1711             # This used to be our default with discard not being specified:
1712             $detectzeroes = 'on';
1713         }
1714
1715         # note: 'detect-zeroes' works per blockdev and we want it to persist
1716         # after the alloc-track is removed, so put it on 'file' directly
1717         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1718         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1719     }
1720
1721     if ($pbs_name) {
1722         $opts .= ",backing=$pbs_name";
1723         $opts .= ",auto-remove=on";
1724     }
1725
1726     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1727     my $file_param = "file";
1728     if ($pbs_name) {
1729         # non-rbd drivers require the underlying file to be a seperate block
1730         # node, so add a second .file indirection
1731         $file_param .= ".file" if !$is_rbd;
1732         $file_param .= ".filename";
1733     }
1734     my $pathinfo = $path ? "$file_param=$path," : '';
1735
1736     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1737 }
1738
1739 sub print_pbs_blockdev {
1740     my ($pbs_conf, $pbs_name) = @_;
1741     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1742     $blockdev .= ",repository=$pbs_conf->{repository}";
1743     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1744     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1745     $blockdev .= ",archive=$pbs_conf->{archive}";
1746     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1747     return $blockdev;
1748 }
1749
1750 sub print_netdevice_full {
1751     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version) = @_;
1752
1753     my $device = $net->{model};
1754     if ($net->{model} eq 'virtio') {
1755          $device = 'virtio-net-pci';
1756      };
1757
1758     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1759     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1760     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1761         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1762         # and out of each queue plus one config interrupt and control vector queue
1763         my $vectors = $net->{queues} * 2 + 2;
1764         $tmpstr .= ",vectors=$vectors,mq=on";
1765         if (min_version($machine_version, 7, 1)) {
1766             $tmpstr .= ",packed=on";
1767         }
1768     }
1769
1770     if (min_version($machine_version, 7, 1) && $net->{model} eq 'virtio'){
1771         $tmpstr .= ",rx_queue_size=1024,tx_queue_size=1024";
1772     }
1773
1774     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1775
1776     if (my $mtu = $net->{mtu}) {
1777         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1778             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1779             if ($mtu == 1) {
1780                  $mtu = $bridge_mtu;
1781             } elsif ($mtu < 576) {
1782                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1783             } elsif ($mtu > $bridge_mtu) {
1784                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1785             }
1786             $tmpstr .= ",host_mtu=$mtu";
1787         } else {
1788             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1789         }
1790     }
1791
1792     if ($use_old_bios_files) {
1793         my $romfile;
1794         if ($device eq 'virtio-net-pci') {
1795             $romfile = 'pxe-virtio.rom';
1796         } elsif ($device eq 'e1000') {
1797             $romfile = 'pxe-e1000.rom';
1798         } elsif ($device eq 'e1000e') {
1799             $romfile = 'pxe-e1000e.rom';
1800         } elsif ($device eq 'ne2k') {
1801             $romfile = 'pxe-ne2k_pci.rom';
1802         } elsif ($device eq 'pcnet') {
1803             $romfile = 'pxe-pcnet.rom';
1804         } elsif ($device eq 'rtl8139') {
1805             $romfile = 'pxe-rtl8139.rom';
1806         }
1807         $tmpstr .= ",romfile=$romfile" if $romfile;
1808     }
1809
1810     return $tmpstr;
1811 }
1812
1813 sub print_netdev_full {
1814     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1815
1816     my $i = '';
1817     if ($netid =~ m/^net(\d+)$/) {
1818         $i = int($1);
1819     }
1820
1821     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1822
1823     my $ifname = "tap${vmid}i$i";
1824
1825     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1826     die "interface name '$ifname' is too long (max 15 character)\n"
1827         if length($ifname) >= 16;
1828
1829     my $vhostparam = '';
1830     if (is_native($arch)) {
1831         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1832     }
1833
1834     my $vmname = $conf->{name} || "vm$vmid";
1835
1836     my $netdev = "";
1837     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1838
1839     if ($net->{bridge}) {
1840         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1841             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1842     } else {
1843         $netdev = "type=user,id=$netid,hostname=$vmname";
1844     }
1845
1846     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1847
1848     return $netdev;
1849 }
1850
1851 my $vga_map = {
1852     'cirrus' => 'cirrus-vga',
1853     'std' => 'VGA',
1854     'vmware' => 'vmware-svga',
1855     'virtio' => 'virtio-vga',
1856     'virtio-gl' => 'virtio-vga-gl',
1857 };
1858
1859 sub print_vga_device {
1860     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1861
1862     my $type = $vga_map->{$vga->{type}};
1863     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1864         $type = 'virtio-gpu';
1865     }
1866     my $vgamem_mb = $vga->{memory};
1867
1868     my $max_outputs = '';
1869     if ($qxlnum) {
1870         $type = $id ? 'qxl' : 'qxl-vga';
1871
1872         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1873             # set max outputs so linux can have up to 4 qxl displays with one device
1874             if (min_version($machine_version, 4, 1)) {
1875                 $max_outputs = ",max_outputs=4";
1876             }
1877         }
1878     }
1879
1880     die "no devicetype for $vga->{type}\n" if !$type;
1881
1882     my $memory = "";
1883     if ($vgamem_mb) {
1884         if ($vga->{type} =~ /^virtio/) {
1885             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1886             $memory = ",max_hostmem=$bytes";
1887         } elsif ($qxlnum) {
1888             # from https://www.spice-space.org/multiple-monitors.html
1889             $memory = ",vgamem_mb=$vga->{memory}";
1890             my $ram = $vgamem_mb * 4;
1891             my $vram = $vgamem_mb * 2;
1892             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1893         } else {
1894             $memory = ",vgamem_mb=$vga->{memory}";
1895         }
1896     } elsif ($qxlnum && $id) {
1897         $memory = ",ram_size=67108864,vram_size=33554432";
1898     }
1899
1900     my $edidoff = "";
1901     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1902         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1903     }
1904
1905     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1906     my $vgaid = "vga" . ($id // '');
1907     my $pciaddr;
1908     if ($q35 && $vgaid eq 'vga') {
1909         # the first display uses pcie.0 bus on q35 machines
1910         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1911     } else {
1912         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1913     }
1914
1915     if ($vga->{type} eq 'virtio-gl') {
1916         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1917         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1918             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1919
1920         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1921             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1922     }
1923
1924     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1925 }
1926
1927 sub parse_number_sets {
1928     my ($set) = @_;
1929     my $res = [];
1930     foreach my $part (split(/;/, $set)) {
1931         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1932             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1933             push @$res, [ $1, $2 ];
1934         } else {
1935             die "invalid range: $part\n";
1936         }
1937     }
1938     return $res;
1939 }
1940
1941 sub parse_numa {
1942     my ($data) = @_;
1943
1944     my $res = parse_property_string($numa_fmt, $data);
1945     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1946     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1947     return $res;
1948 }
1949
1950 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1951 sub parse_net {
1952     my ($data, $disable_mac_autogen) = @_;
1953
1954     my $res = eval { parse_property_string($net_fmt, $data) };
1955     if ($@) {
1956         warn $@;
1957         return;
1958     }
1959     if (!defined($res->{macaddr}) && !$disable_mac_autogen) {
1960         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1961         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1962     }
1963     return $res;
1964 }
1965
1966 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1967 sub parse_ipconfig {
1968     my ($data) = @_;
1969
1970     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1971     if ($@) {
1972         warn $@;
1973         return;
1974     }
1975
1976     if ($res->{gw} && !$res->{ip}) {
1977         warn 'gateway specified without specifying an IP address';
1978         return;
1979     }
1980     if ($res->{gw6} && !$res->{ip6}) {
1981         warn 'IPv6 gateway specified without specifying an IPv6 address';
1982         return;
1983     }
1984     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1985         warn 'gateway specified together with DHCP';
1986         return;
1987     }
1988     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1989         # gw6 + auto/dhcp
1990         warn "IPv6 gateway specified together with $res->{ip6} address";
1991         return;
1992     }
1993
1994     if (!$res->{ip} && !$res->{ip6}) {
1995         return { ip => 'dhcp', ip6 => 'dhcp' };
1996     }
1997
1998     return $res;
1999 }
2000
2001 sub print_net {
2002     my $net = shift;
2003
2004     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2005 }
2006
2007 sub add_random_macs {
2008     my ($settings) = @_;
2009
2010     foreach my $opt (keys %$settings) {
2011         next if $opt !~ m/^net(\d+)$/;
2012         my $net = parse_net($settings->{$opt});
2013         next if !$net;
2014         $settings->{$opt} = print_net($net);
2015     }
2016 }
2017
2018 sub vm_is_volid_owner {
2019     my ($storecfg, $vmid, $volid) = @_;
2020
2021     if ($volid !~  m|^/|) {
2022         my ($path, $owner);
2023         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2024         if ($owner && ($owner == $vmid)) {
2025             return 1;
2026         }
2027     }
2028
2029     return;
2030 }
2031
2032 sub vmconfig_register_unused_drive {
2033     my ($storecfg, $vmid, $conf, $drive) = @_;
2034
2035     if (drive_is_cloudinit($drive)) {
2036         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2037         warn $@ if $@;
2038         delete $conf->{cloudinit};
2039     } elsif (!drive_is_cdrom($drive)) {
2040         my $volid = $drive->{file};
2041         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2042             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2043         }
2044     }
2045 }
2046
2047 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2048 my $smbios1_fmt = {
2049     uuid => {
2050         type => 'string',
2051         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2052         format_description => 'UUID',
2053         description => "Set SMBIOS1 UUID.",
2054         optional => 1,
2055     },
2056     version => {
2057         type => 'string',
2058         pattern => '[A-Za-z0-9+\/]+={0,2}',
2059         format_description => 'Base64 encoded string',
2060         description => "Set SMBIOS1 version.",
2061         optional => 1,
2062     },
2063     serial => {
2064         type => 'string',
2065         pattern => '[A-Za-z0-9+\/]+={0,2}',
2066         format_description => 'Base64 encoded string',
2067         description => "Set SMBIOS1 serial number.",
2068         optional => 1,
2069     },
2070     manufacturer => {
2071         type => 'string',
2072         pattern => '[A-Za-z0-9+\/]+={0,2}',
2073         format_description => 'Base64 encoded string',
2074         description => "Set SMBIOS1 manufacturer.",
2075         optional => 1,
2076     },
2077     product => {
2078         type => 'string',
2079         pattern => '[A-Za-z0-9+\/]+={0,2}',
2080         format_description => 'Base64 encoded string',
2081         description => "Set SMBIOS1 product ID.",
2082         optional => 1,
2083     },
2084     sku => {
2085         type => 'string',
2086         pattern => '[A-Za-z0-9+\/]+={0,2}',
2087         format_description => 'Base64 encoded string',
2088         description => "Set SMBIOS1 SKU string.",
2089         optional => 1,
2090     },
2091     family => {
2092         type => 'string',
2093         pattern => '[A-Za-z0-9+\/]+={0,2}',
2094         format_description => 'Base64 encoded string',
2095         description => "Set SMBIOS1 family string.",
2096         optional => 1,
2097     },
2098     base64 => {
2099         type => 'boolean',
2100         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2101         optional => 1,
2102     },
2103 };
2104
2105 sub parse_smbios1 {
2106     my ($data) = @_;
2107
2108     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2109     warn $@ if $@;
2110     return $res;
2111 }
2112
2113 sub print_smbios1 {
2114     my ($smbios1) = @_;
2115     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2116 }
2117
2118 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2119
2120 sub parse_watchdog {
2121     my ($value) = @_;
2122
2123     return if !$value;
2124
2125     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2126     warn $@ if $@;
2127     return $res;
2128 }
2129
2130 sub parse_guest_agent {
2131     my ($conf) = @_;
2132
2133     return {} if !defined($conf->{agent});
2134
2135     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2136     warn $@ if $@;
2137
2138     # if the agent is disabled ignore the other potentially set properties
2139     return {} if !$res->{enabled};
2140     return $res;
2141 }
2142
2143 sub get_qga_key {
2144     my ($conf, $key) = @_;
2145     return undef if !defined($conf->{agent});
2146
2147     my $agent = parse_guest_agent($conf);
2148     return $agent->{$key};
2149 }
2150
2151 sub parse_vga {
2152     my ($value) = @_;
2153
2154     return {} if !$value;
2155     my $res = eval { parse_property_string($vga_fmt, $value) };
2156     warn $@ if $@;
2157     return $res;
2158 }
2159
2160 sub parse_rng {
2161     my ($value) = @_;
2162
2163     return if !$value;
2164
2165     my $res = eval { parse_property_string($rng_fmt, $value) };
2166     warn $@ if $@;
2167     return $res;
2168 }
2169
2170 sub parse_meta_info {
2171     my ($value) = @_;
2172
2173     return if !$value;
2174
2175     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2176     warn $@ if $@;
2177     return $res;
2178 }
2179
2180 sub new_meta_info_string {
2181     my () = @_; # for now do not allow to override any value
2182
2183     return PVE::JSONSchema::print_property_string(
2184         {
2185             'creation-qemu' => kvm_user_version(),
2186             ctime => "". int(time()),
2187         },
2188         $meta_info_fmt
2189     );
2190 }
2191
2192 sub qemu_created_version_fixups {
2193     my ($conf, $forcemachine, $kvmver) = @_;
2194
2195     my $meta = parse_meta_info($conf->{meta}) // {};
2196     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2197
2198     # check if we need to apply some handling for VMs that always use the latest machine version but
2199     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2200     # would be required (we do not want to pin machine version for non-windows OS type)
2201     if (
2202         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2203         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2204         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2205         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2206     ) {
2207         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2208         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2209             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2210             # and thus with the predictable interface naming of systemd
2211             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2212         }
2213     }
2214     return;
2215 }
2216
2217 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2218 sub verify_usb_device {
2219     my ($value, $noerr) = @_;
2220
2221     return $value if parse_usb_device($value);
2222
2223     return if $noerr;
2224
2225     die "unable to parse usb device\n";
2226 }
2227
2228 # add JSON properties for create and set function
2229 sub json_config_properties {
2230     my ($prop, $with_disk_alloc) = @_;
2231
2232     my $skip_json_config_opts = {
2233         parent => 1,
2234         snaptime => 1,
2235         vmstate => 1,
2236         runningmachine => 1,
2237         runningcpu => 1,
2238         meta => 1,
2239     };
2240
2241     foreach my $opt (keys %$confdesc) {
2242         next if $skip_json_config_opts->{$opt};
2243
2244         if ($with_disk_alloc && is_valid_drivename($opt)) {
2245             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2246         } else {
2247             $prop->{$opt} = $confdesc->{$opt};
2248         }
2249     }
2250
2251     return $prop;
2252 }
2253
2254 # Properties that we can read from an OVF file
2255 sub json_ovf_properties {
2256     my $prop = {};
2257
2258     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2259         $prop->{$device} = {
2260             type => 'string',
2261             format => 'pve-volume-id-or-absolute-path',
2262             description => "Disk image that gets imported to $device",
2263             optional => 1,
2264         };
2265     }
2266
2267     $prop->{cores} = {
2268         type => 'integer',
2269         description => "The number of CPU cores.",
2270         optional => 1,
2271     };
2272     $prop->{memory} = {
2273         type => 'integer',
2274         description => "Amount of RAM for the VM in MB.",
2275         optional => 1,
2276     };
2277     $prop->{name} = {
2278         type => 'string',
2279         description => "Name of the VM.",
2280         optional => 1,
2281     };
2282
2283     return $prop;
2284 }
2285
2286 # return copy of $confdesc_cloudinit to generate documentation
2287 sub cloudinit_config_properties {
2288
2289     return dclone($confdesc_cloudinit);
2290 }
2291
2292 sub check_type {
2293     my ($key, $value) = @_;
2294
2295     die "unknown setting '$key'\n" if !$confdesc->{$key};
2296
2297     my $type = $confdesc->{$key}->{type};
2298
2299     if (!defined($value)) {
2300         die "got undefined value\n";
2301     }
2302
2303     if ($value =~ m/[\n\r]/) {
2304         die "property contains a line feed\n";
2305     }
2306
2307     if ($type eq 'boolean') {
2308         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2309         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2310         die "type check ('boolean') failed - got '$value'\n";
2311     } elsif ($type eq 'integer') {
2312         return int($1) if $value =~ m/^(\d+)$/;
2313         die "type check ('integer') failed - got '$value'\n";
2314     } elsif ($type eq 'number') {
2315         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2316         die "type check ('number') failed - got '$value'\n";
2317     } elsif ($type eq 'string') {
2318         if (my $fmt = $confdesc->{$key}->{format}) {
2319             PVE::JSONSchema::check_format($fmt, $value);
2320             return $value;
2321         }
2322         $value =~ s/^\"(.*)\"$/$1/;
2323         return $value;
2324     } else {
2325         die "internal error"
2326     }
2327 }
2328
2329 sub destroy_vm {
2330     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2331
2332     my $conf = PVE::QemuConfig->load_config($vmid);
2333
2334     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2335
2336     if ($conf->{template}) {
2337         # check if any base image is still used by a linked clone
2338         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2339                 my ($ds, $drive) = @_;
2340                 return if drive_is_cdrom($drive);
2341
2342                 my $volid = $drive->{file};
2343                 return if !$volid || $volid =~ m|^/|;
2344
2345                 die "base volume '$volid' is still in use by linked cloned\n"
2346                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2347
2348         });
2349     }
2350
2351     my $volids = {};
2352     my $remove_owned_drive = sub {
2353         my ($ds, $drive) = @_;
2354         return if drive_is_cdrom($drive, 1);
2355
2356         my $volid = $drive->{file};
2357         return if !$volid || $volid =~ m|^/|;
2358         return if $volids->{$volid};
2359
2360         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2361         return if !$path || !$owner || ($owner != $vmid);
2362
2363         $volids->{$volid} = 1;
2364         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2365         warn "Could not remove disk '$volid', check manually: $@" if $@;
2366     };
2367
2368     # only remove disks owned by this VM (referenced in the config)
2369     my $include_opts = {
2370         include_unused => 1,
2371         extra_keys => ['vmstate'],
2372     };
2373     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2374
2375     for my $snap (values %{$conf->{snapshots}}) {
2376         next if !defined($snap->{vmstate});
2377         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2378         next if !defined($drive);
2379         $remove_owned_drive->('vmstate', $drive);
2380     }
2381
2382     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2383
2384     if ($purge_unreferenced) { # also remove unreferenced disk
2385         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2386         PVE::Storage::foreach_volid($vmdisks, sub {
2387             my ($volid, $sid, $volname, $d) = @_;
2388             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2389             warn $@ if $@;
2390         });
2391     }
2392
2393     if (defined $replacement_conf) {
2394         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2395     } else {
2396         PVE::QemuConfig->destroy_config($vmid);
2397     }
2398 }
2399
2400 sub parse_vm_config {
2401     my ($filename, $raw, $strict) = @_;
2402
2403     return if !defined($raw);
2404
2405     my $res = {
2406         digest => Digest::SHA::sha1_hex($raw),
2407         snapshots => {},
2408         pending => {},
2409         cloudinit => {},
2410     };
2411
2412     my $handle_error = sub {
2413         my ($msg) = @_;
2414
2415         if ($strict) {
2416             die $msg;
2417         } else {
2418             warn $msg;
2419         }
2420     };
2421
2422     $filename =~ m|/qemu-server/(\d+)\.conf$|
2423         || die "got strange filename '$filename'";
2424
2425     my $vmid = $1;
2426
2427     my $conf = $res;
2428     my $descr;
2429     my $section = '';
2430
2431     my @lines = split(/\n/, $raw);
2432     foreach my $line (@lines) {
2433         next if $line =~ m/^\s*$/;
2434
2435         if ($line =~ m/^\[PENDING\]\s*$/i) {
2436             $section = 'pending';
2437             if (defined($descr)) {
2438                 $descr =~ s/\s+$//;
2439                 $conf->{description} = $descr;
2440             }
2441             $descr = undef;
2442             $conf = $res->{$section} = {};
2443             next;
2444         } elsif ($line =~ m/^\[special:cloudinit\]\s*$/i) {
2445             $section = 'cloudinit';
2446             $descr = undef;
2447             $conf = $res->{$section} = {};
2448             next;
2449
2450         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2451             $section = $1;
2452             if (defined($descr)) {
2453                 $descr =~ s/\s+$//;
2454                 $conf->{description} = $descr;
2455             }
2456             $descr = undef;
2457             $conf = $res->{snapshots}->{$section} = {};
2458             next;
2459         }
2460
2461         if ($line =~ m/^\#(.*)$/) {
2462             $descr = '' if !defined($descr);
2463             $descr .= PVE::Tools::decode_text($1) . "\n";
2464             next;
2465         }
2466
2467         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2468             $descr = '' if !defined($descr);
2469             $descr .= PVE::Tools::decode_text($2);
2470         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2471             $conf->{snapstate} = $1;
2472         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2473             my $key = $1;
2474             my $value = $2;
2475             $conf->{$key} = $value;
2476         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2477             my $value = $1;
2478             if ($section eq 'pending') {
2479                 $conf->{delete} = $value; # we parse this later
2480             } else {
2481                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2482             }
2483         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2484             my $key = $1;
2485             my $value = $2;
2486             eval { $value = check_type($key, $value); };
2487             if ($@) {
2488                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2489             } else {
2490                 $key = 'ide2' if $key eq 'cdrom';
2491                 my $fmt = $confdesc->{$key}->{format};
2492                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2493                     my $v = parse_drive($key, $value);
2494                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2495                         $v->{file} = $volid;
2496                         $value = print_drive($v);
2497                     } else {
2498                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2499                         next;
2500                     }
2501                 }
2502
2503                 $conf->{$key} = $value;
2504             }
2505         } else {
2506             $handle_error->("vm $vmid - unable to parse config: $line\n");
2507         }
2508     }
2509
2510     if (defined($descr)) {
2511         $descr =~ s/\s+$//;
2512         $conf->{description} = $descr;
2513     }
2514     delete $res->{snapstate}; # just to be sure
2515
2516     return $res;
2517 }
2518
2519 sub write_vm_config {
2520     my ($filename, $conf) = @_;
2521
2522     delete $conf->{snapstate}; # just to be sure
2523
2524     if ($conf->{cdrom}) {
2525         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2526         $conf->{ide2} = $conf->{cdrom};
2527         delete $conf->{cdrom};
2528     }
2529
2530     # we do not use 'smp' any longer
2531     if ($conf->{sockets}) {
2532         delete $conf->{smp};
2533     } elsif ($conf->{smp}) {
2534         $conf->{sockets} = $conf->{smp};
2535         delete $conf->{cores};
2536         delete $conf->{smp};
2537     }
2538
2539     my $used_volids = {};
2540
2541     my $cleanup_config = sub {
2542         my ($cref, $pending, $snapname) = @_;
2543
2544         foreach my $key (keys %$cref) {
2545             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2546                 $key eq 'snapstate' || $key eq 'pending' || $key eq 'cloudinit';
2547             my $value = $cref->{$key};
2548             if ($key eq 'delete') {
2549                 die "propertry 'delete' is only allowed in [PENDING]\n"
2550                     if !$pending;
2551                 # fixme: check syntax?
2552                 next;
2553             }
2554             eval { $value = check_type($key, $value); };
2555             die "unable to parse value of '$key' - $@" if $@;
2556
2557             $cref->{$key} = $value;
2558
2559             if (!$snapname && is_valid_drivename($key)) {
2560                 my $drive = parse_drive($key, $value);
2561                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2562             }
2563         }
2564     };
2565
2566     &$cleanup_config($conf);
2567
2568     &$cleanup_config($conf->{pending}, 1);
2569
2570     &$cleanup_config($conf->{cloudinit});
2571
2572     foreach my $snapname (keys %{$conf->{snapshots}}) {
2573         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2574         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2575     }
2576
2577     # remove 'unusedX' settings if we re-add a volume
2578     foreach my $key (keys %$conf) {
2579         my $value = $conf->{$key};
2580         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2581             delete $conf->{$key};
2582         }
2583     }
2584
2585     my $generate_raw_config = sub {
2586         my ($conf, $pending) = @_;
2587
2588         my $raw = '';
2589
2590         # add description as comment to top of file
2591         if (defined(my $descr = $conf->{description})) {
2592             if ($descr) {
2593                 foreach my $cl (split(/\n/, $descr)) {
2594                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2595                 }
2596             } else {
2597                 $raw .= "#\n" if $pending;
2598             }
2599         }
2600
2601         foreach my $key (sort keys %$conf) {
2602             next if $key =~ /^(digest|description|pending|cloudinit|snapshots)$/;
2603             $raw .= "$key: $conf->{$key}\n";
2604         }
2605         return $raw;
2606     };
2607
2608     my $raw = &$generate_raw_config($conf);
2609
2610     if (scalar(keys %{$conf->{pending}})){
2611         $raw .= "\n[PENDING]\n";
2612         $raw .= &$generate_raw_config($conf->{pending}, 1);
2613     }
2614
2615     if (scalar(keys %{$conf->{cloudinit}})){
2616         $raw .= "\n[special:cloudinit]\n";
2617         $raw .= &$generate_raw_config($conf->{cloudinit});
2618     }
2619
2620     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2621         $raw .= "\n[$snapname]\n";
2622         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2623     }
2624
2625     return $raw;
2626 }
2627
2628 sub load_defaults {
2629
2630     my $res = {};
2631
2632     # we use static defaults from our JSON schema configuration
2633     foreach my $key (keys %$confdesc) {
2634         if (defined(my $default = $confdesc->{$key}->{default})) {
2635             $res->{$key} = $default;
2636         }
2637     }
2638
2639     return $res;
2640 }
2641
2642 sub config_list {
2643     my $vmlist = PVE::Cluster::get_vmlist();
2644     my $res = {};
2645     return $res if !$vmlist || !$vmlist->{ids};
2646     my $ids = $vmlist->{ids};
2647     my $nodename = nodename();
2648
2649     foreach my $vmid (keys %$ids) {
2650         my $d = $ids->{$vmid};
2651         next if !$d->{node} || $d->{node} ne $nodename;
2652         next if !$d->{type} || $d->{type} ne 'qemu';
2653         $res->{$vmid}->{exists} = 1;
2654     }
2655     return $res;
2656 }
2657
2658 # test if VM uses local resources (to prevent migration)
2659 sub check_local_resources {
2660     my ($conf, $noerr) = @_;
2661
2662     my @loc_res = ();
2663
2664     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2665     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2666
2667     push @loc_res, "ivshmem" if $conf->{ivshmem};
2668
2669     foreach my $k (keys %$conf) {
2670         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2671         # sockets are safe: they will recreated be on the target side post-migrate
2672         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2673         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2674     }
2675
2676     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2677
2678     return \@loc_res;
2679 }
2680
2681 # check if used storages are available on all nodes (use by migrate)
2682 sub check_storage_availability {
2683     my ($storecfg, $conf, $node) = @_;
2684
2685     PVE::QemuConfig->foreach_volume($conf, sub {
2686         my ($ds, $drive) = @_;
2687
2688         my $volid = $drive->{file};
2689         return if !$volid;
2690
2691         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2692         return if !$sid;
2693
2694         # check if storage is available on both nodes
2695         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2696         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2697
2698         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2699
2700         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2701             if !$scfg->{content}->{$vtype};
2702    });
2703 }
2704
2705 # list nodes where all VM images are available (used by has_feature API)
2706 sub shared_nodes {
2707     my ($conf, $storecfg) = @_;
2708
2709     my $nodelist = PVE::Cluster::get_nodelist();
2710     my $nodehash = { map { $_ => 1 } @$nodelist };
2711     my $nodename = nodename();
2712
2713     PVE::QemuConfig->foreach_volume($conf, sub {
2714         my ($ds, $drive) = @_;
2715
2716         my $volid = $drive->{file};
2717         return if !$volid;
2718
2719         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2720         if ($storeid) {
2721             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2722             if ($scfg->{disable}) {
2723                 $nodehash = {};
2724             } elsif (my $avail = $scfg->{nodes}) {
2725                 foreach my $node (keys %$nodehash) {
2726                     delete $nodehash->{$node} if !$avail->{$node};
2727                 }
2728             } elsif (!$scfg->{shared}) {
2729                 foreach my $node (keys %$nodehash) {
2730                     delete $nodehash->{$node} if $node ne $nodename
2731                 }
2732             }
2733         }
2734     });
2735
2736     return $nodehash
2737 }
2738
2739 sub check_local_storage_availability {
2740     my ($conf, $storecfg) = @_;
2741
2742     my $nodelist = PVE::Cluster::get_nodelist();
2743     my $nodehash = { map { $_ => {} } @$nodelist };
2744
2745     PVE::QemuConfig->foreach_volume($conf, sub {
2746         my ($ds, $drive) = @_;
2747
2748         my $volid = $drive->{file};
2749         return if !$volid;
2750
2751         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2752         if ($storeid) {
2753             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2754
2755             if ($scfg->{disable}) {
2756                 foreach my $node (keys %$nodehash) {
2757                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2758                 }
2759             } elsif (my $avail = $scfg->{nodes}) {
2760                 foreach my $node (keys %$nodehash) {
2761                     if (!$avail->{$node}) {
2762                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2763                     }
2764                 }
2765             }
2766         }
2767     });
2768
2769     foreach my $node (values %$nodehash) {
2770         if (my $unavail = $node->{unavailable_storages}) {
2771             $node->{unavailable_storages} = [ sort keys %$unavail ];
2772         }
2773     }
2774
2775     return $nodehash
2776 }
2777
2778 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2779 sub check_running {
2780     my ($vmid, $nocheck, $node) = @_;
2781
2782     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2783     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2784 }
2785
2786 sub vzlist {
2787
2788     my $vzlist = config_list();
2789
2790     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2791
2792     while (defined(my $de = $fd->read)) {
2793         next if $de !~ m/^(\d+)\.pid$/;
2794         my $vmid = $1;
2795         next if !defined($vzlist->{$vmid});
2796         if (my $pid = check_running($vmid)) {
2797             $vzlist->{$vmid}->{pid} = $pid;
2798         }
2799     }
2800
2801     return $vzlist;
2802 }
2803
2804 our $vmstatus_return_properties = {
2805     vmid => get_standard_option('pve-vmid'),
2806     status => {
2807         description => "Qemu process status.",
2808         type => 'string',
2809         enum => ['stopped', 'running'],
2810     },
2811     maxmem => {
2812         description => "Maximum memory in bytes.",
2813         type => 'integer',
2814         optional => 1,
2815         renderer => 'bytes',
2816     },
2817     maxdisk => {
2818         description => "Root disk size in bytes.",
2819         type => 'integer',
2820         optional => 1,
2821         renderer => 'bytes',
2822     },
2823     name => {
2824         description => "VM name.",
2825         type => 'string',
2826         optional => 1,
2827     },
2828     qmpstatus => {
2829         description => "Qemu QMP agent status.",
2830         type => 'string',
2831         optional => 1,
2832     },
2833     pid => {
2834         description => "PID of running qemu process.",
2835         type => 'integer',
2836         optional => 1,
2837     },
2838     uptime => {
2839         description => "Uptime.",
2840         type => 'integer',
2841         optional => 1,
2842         renderer => 'duration',
2843     },
2844     cpus => {
2845         description => "Maximum usable CPUs.",
2846         type => 'number',
2847         optional => 1,
2848     },
2849     lock => {
2850         description => "The current config lock, if any.",
2851         type => 'string',
2852         optional => 1,
2853     },
2854     tags => {
2855         description  => "The current configured tags, if any",
2856         type => 'string',
2857         optional => 1,
2858     },
2859     'running-machine' => {
2860         description => "The currently running machine type (if running).",
2861         type => 'string',
2862         optional => 1,
2863     },
2864     'running-qemu' => {
2865         description => "The currently running QEMU version (if running).",
2866         type => 'string',
2867         optional => 1,
2868     },
2869 };
2870
2871 my $last_proc_pid_stat;
2872
2873 # get VM status information
2874 # This must be fast and should not block ($full == false)
2875 # We only query KVM using QMP if $full == true (this can be slow)
2876 sub vmstatus {
2877     my ($opt_vmid, $full) = @_;
2878
2879     my $res = {};
2880
2881     my $storecfg = PVE::Storage::config();
2882
2883     my $list = vzlist();
2884     my $defaults = load_defaults();
2885
2886     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2887
2888     my $cpucount = $cpuinfo->{cpus} || 1;
2889
2890     foreach my $vmid (keys %$list) {
2891         next if $opt_vmid && ($vmid ne $opt_vmid);
2892
2893         my $conf = PVE::QemuConfig->load_config($vmid);
2894
2895         my $d = { vmid => int($vmid) };
2896         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2897
2898         # fixme: better status?
2899         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2900
2901         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2902         if (defined($size)) {
2903             $d->{disk} = 0; # no info available
2904             $d->{maxdisk} = $size;
2905         } else {
2906             $d->{disk} = 0;
2907             $d->{maxdisk} = 0;
2908         }
2909
2910         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2911             * ($conf->{cores} || $defaults->{cores});
2912         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2913         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2914
2915         $d->{name} = $conf->{name} || "VM $vmid";
2916         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2917             : $defaults->{memory}*(1024*1024);
2918
2919         if ($conf->{balloon}) {
2920             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2921             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2922                 : $defaults->{shares};
2923         }
2924
2925         $d->{uptime} = 0;
2926         $d->{cpu} = 0;
2927         $d->{mem} = 0;
2928
2929         $d->{netout} = 0;
2930         $d->{netin} = 0;
2931
2932         $d->{diskread} = 0;
2933         $d->{diskwrite} = 0;
2934
2935         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2936
2937         $d->{serial} = 1 if conf_has_serial($conf);
2938         $d->{lock} = $conf->{lock} if $conf->{lock};
2939         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2940
2941         $res->{$vmid} = $d;
2942     }
2943
2944     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2945     foreach my $dev (keys %$netdev) {
2946         next if $dev !~ m/^tap([1-9]\d*)i/;
2947         my $vmid = $1;
2948         my $d = $res->{$vmid};
2949         next if !$d;
2950
2951         $d->{netout} += $netdev->{$dev}->{receive};
2952         $d->{netin} += $netdev->{$dev}->{transmit};
2953
2954         if ($full) {
2955             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2956             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2957         }
2958
2959     }
2960
2961     my $ctime = gettimeofday;
2962
2963     foreach my $vmid (keys %$list) {
2964
2965         my $d = $res->{$vmid};
2966         my $pid = $d->{pid};
2967         next if !$pid;
2968
2969         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2970         next if !$pstat; # not running
2971
2972         my $used = $pstat->{utime} + $pstat->{stime};
2973
2974         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2975
2976         if ($pstat->{vsize}) {
2977             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2978         }
2979
2980         my $old = $last_proc_pid_stat->{$pid};
2981         if (!$old) {
2982             $last_proc_pid_stat->{$pid} = {
2983                 time => $ctime,
2984                 used => $used,
2985                 cpu => 0,
2986             };
2987             next;
2988         }
2989
2990         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2991
2992         if ($dtime > 1000) {
2993             my $dutime = $used -  $old->{used};
2994
2995             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2996             $last_proc_pid_stat->{$pid} = {
2997                 time => $ctime,
2998                 used => $used,
2999                 cpu => $d->{cpu},
3000             };
3001         } else {
3002             $d->{cpu} = $old->{cpu};
3003         }
3004     }
3005
3006     return $res if !$full;
3007
3008     my $qmpclient = PVE::QMPClient->new();
3009
3010     my $ballooncb = sub {
3011         my ($vmid, $resp) = @_;
3012
3013         my $info = $resp->{'return'};
3014         return if !$info->{max_mem};
3015
3016         my $d = $res->{$vmid};
3017
3018         # use memory assigned to VM
3019         $d->{maxmem} = $info->{max_mem};
3020         $d->{balloon} = $info->{actual};
3021
3022         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3023             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3024             $d->{freemem} = $info->{free_mem};
3025         }
3026
3027         $d->{ballooninfo} = $info;
3028     };
3029
3030     my $blockstatscb = sub {
3031         my ($vmid, $resp) = @_;
3032         my $data = $resp->{'return'} || [];
3033         my $totalrdbytes = 0;
3034         my $totalwrbytes = 0;
3035
3036         for my $blockstat (@$data) {
3037             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3038             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3039
3040             $blockstat->{device} =~ s/drive-//;
3041             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3042         }
3043         $res->{$vmid}->{diskread} = $totalrdbytes;
3044         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3045     };
3046
3047     my $machinecb = sub {
3048         my ($vmid, $resp) = @_;
3049         my $data = $resp->{'return'} || [];
3050
3051         $res->{$vmid}->{'running-machine'} =
3052             PVE::QemuServer::Machine::current_from_query_machines($data);
3053     };
3054
3055     my $versioncb = sub {
3056         my ($vmid, $resp) = @_;
3057         my $data = $resp->{'return'} // {};
3058         my $version = 'unknown';
3059
3060         if (my $v = $data->{qemu}) {
3061             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3062         }
3063
3064         $res->{$vmid}->{'running-qemu'} = $version;
3065     };
3066
3067     my $statuscb = sub {
3068         my ($vmid, $resp) = @_;
3069
3070         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3071         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3072         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3073         # this fails if ballon driver is not loaded, so this must be
3074         # the last commnand (following command are aborted if this fails).
3075         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3076
3077         my $status = 'unknown';
3078         if (!defined($status = $resp->{'return'}->{status})) {
3079             warn "unable to get VM status\n";
3080             return;
3081         }
3082
3083         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3084     };
3085
3086     foreach my $vmid (keys %$list) {
3087         next if $opt_vmid && ($vmid ne $opt_vmid);
3088         next if !$res->{$vmid}->{pid}; # not running
3089         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3090     }
3091
3092     $qmpclient->queue_execute(undef, 2);
3093
3094     foreach my $vmid (keys %$list) {
3095         next if $opt_vmid && ($vmid ne $opt_vmid);
3096         next if !$res->{$vmid}->{pid}; #not running
3097
3098         # we can't use the $qmpclient since it might have already aborted on
3099         # 'query-balloon', but this might also fail for older versions...
3100         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3101         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3102     }
3103
3104     foreach my $vmid (keys %$list) {
3105         next if $opt_vmid && ($vmid ne $opt_vmid);
3106         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3107     }
3108
3109     return $res;
3110 }
3111
3112 sub conf_has_serial {
3113     my ($conf) = @_;
3114
3115     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3116         if ($conf->{"serial$i"}) {
3117             return 1;
3118         }
3119     }
3120
3121     return 0;
3122 }
3123
3124 sub conf_has_audio {
3125     my ($conf, $id) = @_;
3126
3127     $id //= 0;
3128     my $audio = $conf->{"audio$id"};
3129     return if !defined($audio);
3130
3131     my $audioproperties = parse_property_string($audio_fmt, $audio);
3132     my $audiodriver = $audioproperties->{driver} // 'spice';
3133
3134     return {
3135         dev => $audioproperties->{device},
3136         dev_id => "audiodev$id",
3137         backend => $audiodriver,
3138         backend_id => "$audiodriver-backend${id}",
3139     };
3140 }
3141
3142 sub audio_devs {
3143     my ($audio, $audiopciaddr, $machine_version) = @_;
3144
3145     my $devs = [];
3146
3147     my $id = $audio->{dev_id};
3148     my $audiodev = "";
3149     if (min_version($machine_version, 4, 2)) {
3150         $audiodev = ",audiodev=$audio->{backend_id}";
3151     }
3152
3153     if ($audio->{dev} eq 'AC97') {
3154         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3155     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3156         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3157         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3158         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3159     } else {
3160         die "unkown audio device '$audio->{dev}', implement me!";
3161     }
3162
3163     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3164
3165     return $devs;
3166 }
3167
3168 sub get_tpm_paths {
3169     my ($vmid) = @_;
3170     return {
3171         socket => "/var/run/qemu-server/$vmid.swtpm",
3172         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3173     };
3174 }
3175
3176 sub add_tpm_device {
3177     my ($vmid, $devices, $conf) = @_;
3178
3179     return if !$conf->{tpmstate0};
3180
3181     my $paths = get_tpm_paths($vmid);
3182
3183     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3184     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3185     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3186 }
3187
3188 sub start_swtpm {
3189     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3190
3191     return if !$tpmdrive;
3192
3193     my $state;
3194     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3195     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3196     if ($storeid) {
3197         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3198     } else {
3199         $state = $tpm->{file};
3200     }
3201
3202     my $paths = get_tpm_paths($vmid);
3203
3204     # during migration, we will get state from remote
3205     #
3206     if (!$migration) {
3207         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3208         my $setup_cmd = [
3209             "swtpm_setup",
3210             "--tpmstate",
3211             "file://$state",
3212             "--createek",
3213             "--create-ek-cert",
3214             "--create-platform-cert",
3215             "--lock-nvram",
3216             "--config",
3217             "/etc/swtpm_setup.conf", # do not use XDG configs
3218             "--runas",
3219             "0", # force creation as root, error if not possible
3220             "--not-overwrite", # ignore existing state, do not modify
3221         ];
3222
3223         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3224         # TPM 2.0 supports ECC crypto, use if possible
3225         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3226
3227         run_command($setup_cmd, outfunc => sub {
3228             print "swtpm_setup: $1\n";
3229         });
3230     }
3231
3232     my $emulator_cmd = [
3233         "swtpm",
3234         "socket",
3235         "--tpmstate",
3236         "backend-uri=file://$state,mode=0600",
3237         "--ctrl",
3238         "type=unixio,path=$paths->{socket},mode=0600",
3239         "--pid",
3240         "file=$paths->{pid}",
3241         "--terminate", # terminate on QEMU disconnect
3242         "--daemon",
3243     ];
3244     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3245     run_command($emulator_cmd, outfunc => sub { print $1; });
3246
3247     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3248     while (! -e $paths->{pid}) {
3249         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3250         usleep(50_000);
3251     }
3252
3253     # return untainted PID of swtpm daemon so it can be killed on error
3254     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3255     return $1;
3256 }
3257
3258 sub vga_conf_has_spice {
3259     my ($vga) = @_;
3260
3261     my $vgaconf = parse_vga($vga);
3262     my $vgatype = $vgaconf->{type};
3263     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3264
3265     return $1 || 1;
3266 }
3267
3268 sub is_native($) {
3269     my ($arch) = @_;
3270     return get_host_arch() eq $arch;
3271 }
3272
3273 sub get_vm_arch {
3274     my ($conf) = @_;
3275     return $conf->{arch} // get_host_arch();
3276 }
3277
3278 my $default_machines = {
3279     x86_64 => 'pc',
3280     aarch64 => 'virt',
3281 };
3282
3283 sub get_installed_machine_version {
3284     my ($kvmversion) = @_;
3285     $kvmversion = kvm_user_version() if !defined($kvmversion);
3286     $kvmversion =~ m/^(\d+\.\d+)/;
3287     return $1;
3288 }
3289
3290 sub windows_get_pinned_machine_version {
3291     my ($machine, $base_version, $kvmversion) = @_;
3292
3293     my $pin_version = $base_version;
3294     if (!defined($base_version) ||
3295         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3296     ) {
3297         $pin_version = get_installed_machine_version($kvmversion);
3298     }
3299     if (!$machine || $machine eq 'pc') {
3300         $machine = "pc-i440fx-$pin_version";
3301     } elsif ($machine eq 'q35') {
3302         $machine = "pc-q35-$pin_version";
3303     } elsif ($machine eq 'virt') {
3304         $machine = "virt-$pin_version";
3305     } else {
3306         warn "unknown machine type '$machine', not touching that!\n";
3307     }
3308
3309     return $machine;
3310 }
3311
3312 sub get_vm_machine {
3313     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3314
3315     my $machine = $forcemachine || $conf->{machine};
3316
3317     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3318         $kvmversion //= kvm_user_version();
3319         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3320         # layout which confuses windows quite a bit and may result in various regressions..
3321         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3322         if (windows_version($conf->{ostype})) {
3323             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3324         }
3325         $arch //= 'x86_64';
3326         $machine ||= $default_machines->{$arch};
3327         if ($add_pve_version) {
3328             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3329             $machine .= "+pve$pvever";
3330         }
3331     }
3332
3333     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3334         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3335         $machine = $1 if $is_pxe;
3336
3337         # for version-pinned machines that do not include a pve-version (e.g.
3338         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3339         $machine .= '+pve0';
3340
3341         $machine .= '.pxe' if $is_pxe;
3342     }
3343
3344     return $machine;
3345 }
3346
3347 sub get_ovmf_files($$$) {
3348     my ($arch, $efidisk, $smm) = @_;
3349
3350     my $types = $OVMF->{$arch}
3351         or die "no OVMF images known for architecture '$arch'\n";
3352
3353     my $type = 'default';
3354     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3355         $type = $smm ? "4m" : "4m-no-smm";
3356         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3357     }
3358
3359     return $types->{$type}->@*;
3360 }
3361
3362 my $Arch2Qemu = {
3363     aarch64 => '/usr/bin/qemu-system-aarch64',
3364     x86_64 => '/usr/bin/qemu-system-x86_64',
3365 };
3366 sub get_command_for_arch($) {
3367     my ($arch) = @_;
3368     return '/usr/bin/kvm' if is_native($arch);
3369
3370     my $cmd = $Arch2Qemu->{$arch}
3371         or die "don't know how to emulate architecture '$arch'\n";
3372     return $cmd;
3373 }
3374
3375 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3376 # to use in a QEMU command line (-cpu element), first array_intersect the result
3377 # of query_supported_ with query_understood_. This is necessary because:
3378 #
3379 # a) query_understood_ returns flags the host cannot use and
3380 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3381 #    flags, but CPU settings - with most of them being flags. Those settings
3382 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3383 #
3384 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3385 # expensive. If you need the value returned from this, you can get it much
3386 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3387 # $accel being 'kvm' or 'tcg'.
3388 #
3389 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3390 # changes, automatically populating pmxcfs.
3391 #
3392 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3393 # since kvm and tcg machines support different flags
3394 #
3395 sub query_supported_cpu_flags {
3396     my ($arch) = @_;
3397
3398     $arch //= get_host_arch();
3399     my $default_machine = $default_machines->{$arch};
3400
3401     my $flags = {};
3402
3403     # FIXME: Once this is merged, the code below should work for ARM as well:
3404     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3405     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3406         $arch eq "aarch64";
3407
3408     my $kvm_supported = defined(kvm_version());
3409     my $qemu_cmd = get_command_for_arch($arch);
3410     my $fakevmid = -1;
3411     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3412
3413     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3414     my $query_supported_run_qemu = sub {
3415         my ($kvm) = @_;
3416
3417         my $flags = {};
3418         my $cmd = [
3419             $qemu_cmd,
3420             '-machine', $default_machine,
3421             '-display', 'none',
3422             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3423             '-mon', 'chardev=qmp,mode=control',
3424             '-pidfile', $pidfile,
3425             '-S', '-daemonize'
3426         ];
3427
3428         if (!$kvm) {
3429             push @$cmd, '-accel', 'tcg';
3430         }
3431
3432         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3433         die "QEMU flag querying VM exited with code " . $rc if $rc;
3434
3435         eval {
3436             my $cmd_result = mon_cmd(
3437                 $fakevmid,
3438                 'query-cpu-model-expansion',
3439                 type => 'full',
3440                 model => { name => 'host' }
3441             );
3442
3443             my $props = $cmd_result->{model}->{props};
3444             foreach my $prop (keys %$props) {
3445                 next if $props->{$prop} ne '1';
3446                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3447                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3448                 # We only keep those with underscores, to match /proc/cpuinfo
3449                 $prop =~ s/\.|-/_/g;
3450                 $flags->{$prop} = 1;
3451             }
3452         };
3453         my $err = $@;
3454
3455         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3456         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3457
3458         die $err if $err;
3459
3460         return [ sort keys %$flags ];
3461     };
3462
3463     # We need to query QEMU twice, since KVM and TCG have different supported flags
3464     PVE::QemuConfig->lock_config($fakevmid, sub {
3465         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3466         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3467
3468         if ($kvm_supported) {
3469             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3470             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3471         }
3472     });
3473
3474     return $flags;
3475 }
3476
3477 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3478 my $understood_cpu_flag_dir = "/usr/share/kvm";
3479 sub query_understood_cpu_flags {
3480     my $arch = get_host_arch();
3481     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3482
3483     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3484         if ! -e $filepath;
3485
3486     my $raw = file_get_contents($filepath);
3487     $raw =~ s/^\s+|\s+$//g;
3488     my @flags = split(/\s+/, $raw);
3489
3490     return \@flags;
3491 }
3492
3493 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3494 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3495 my sub should_disable_smm {
3496     my ($conf, $vga) = @_;
3497
3498     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3499         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3500 }
3501
3502 sub config_to_command {
3503     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3504         $pbs_backing) = @_;
3505
3506     my $cmd = [];
3507     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3508     my $devices = [];
3509     my $bridges = {};
3510     my $ostype = $conf->{ostype};
3511     my $winversion = windows_version($ostype);
3512     my $kvm = $conf->{kvm};
3513     my $nodename = nodename();
3514
3515     my $arch = get_vm_arch($conf);
3516     my $kvm_binary = get_command_for_arch($arch);
3517     my $kvmver = kvm_user_version($kvm_binary);
3518
3519     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3520         $kvmver //= "undefined";
3521         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3522     }
3523
3524     my $add_pve_version = min_version($kvmver, 4, 1);
3525
3526     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3527     my $machine_version = extract_version($machine_type, $kvmver);
3528     $kvm //= 1 if is_native($arch);
3529
3530     $machine_version =~ m/(\d+)\.(\d+)/;
3531     my ($machine_major, $machine_minor) = ($1, $2);
3532
3533     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3534         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3535     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3536         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3537             ." please upgrade node '$nodename'\n"
3538     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3539         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3540         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3541             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3542             ." node '$nodename'\n";
3543     }
3544
3545     # if a specific +pve version is required for a feature, use $version_guard
3546     # instead of min_version to allow machines to be run with the minimum
3547     # required version
3548     my $required_pve_version = 0;
3549     my $version_guard = sub {
3550         my ($major, $minor, $pve) = @_;
3551         return 0 if !min_version($machine_version, $major, $minor, $pve);
3552         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3553         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3554         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3555         return 1;
3556     };
3557
3558     if ($kvm && !defined kvm_version()) {
3559         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3560             ." or enable in BIOS.\n";
3561     }
3562
3563     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3564     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3565     my $use_old_bios_files = undef;
3566     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3567
3568     if ($conf->{affinity}) {
3569         push @$cmd, "/usr/bin/taskset";
3570         push @$cmd, "--cpu-list";
3571         push @$cmd, "--all-tasks";
3572         push @$cmd, $conf->{affinity};
3573     }
3574
3575     push @$cmd, $kvm_binary;
3576
3577     push @$cmd, '-id', $vmid;
3578
3579     my $vmname = $conf->{name} || "vm$vmid";
3580
3581     push @$cmd, '-name', "$vmname,debug-threads=on";
3582
3583     push @$cmd, '-no-shutdown';
3584
3585     my $use_virtio = 0;
3586
3587     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3588     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3589     push @$cmd, '-mon', "chardev=qmp,mode=control";
3590
3591     if (min_version($machine_version, 2, 12)) {
3592         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3593         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3594     }
3595
3596     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3597
3598     push @$cmd, '-daemonize';
3599
3600     if ($conf->{smbios1}) {
3601         my $smbios_conf = parse_smbios1($conf->{smbios1});
3602         if ($smbios_conf->{base64}) {
3603             # Do not pass base64 flag to qemu
3604             delete $smbios_conf->{base64};
3605             my $smbios_string = "";
3606             foreach my $key (keys %$smbios_conf) {
3607                 my $value;
3608                 if ($key eq "uuid") {
3609                     $value = $smbios_conf->{uuid}
3610                 } else {
3611                     $value = decode_base64($smbios_conf->{$key});
3612                 }
3613                 # qemu accepts any binary data, only commas need escaping by double comma
3614                 $value =~ s/,/,,/g;
3615                 $smbios_string .= "," . $key . "=" . $value if $value;
3616             }
3617             push @$cmd, '-smbios', "type=1" . $smbios_string;
3618         } else {
3619             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3620         }
3621     }
3622
3623     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3624         my $d;
3625         if (my $efidisk = $conf->{efidisk0}) {
3626             $d = parse_drive('efidisk0', $efidisk);
3627         }
3628
3629         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3630         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3631
3632         my ($path, $format);
3633         my $read_only_str = '';
3634         if ($d) {
3635             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3636             $format = $d->{format};
3637             if ($storeid) {
3638                 $path = PVE::Storage::path($storecfg, $d->{file});
3639                 if (!defined($format)) {
3640                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3641                     $format = qemu_img_format($scfg, $volname);
3642                 }
3643             } else {
3644                 $path = $d->{file};
3645                 die "efidisk format must be specified\n"
3646                     if !defined($format);
3647             }
3648
3649             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3650         } else {
3651             log_warn("no efidisk configured! Using temporary efivars disk.");
3652             $path = "/tmp/$vmid-ovmf.fd";
3653             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3654             $format = 'raw';
3655         }
3656
3657         my $size_str = "";
3658
3659         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3660             $size_str = ",size=" . (-s $ovmf_vars);
3661         }
3662
3663         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3664         my $cache = "";
3665         if ($path =~ m/^rbd:/) {
3666                 $cache = ',cache=writeback';
3667                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3668         }
3669
3670         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3671         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3672     }
3673
3674     if ($q35) { # tell QEMU to load q35 config early
3675         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3676         if (min_version($machine_version, 4, 0)) {
3677             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3678         } else {
3679             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3680         }
3681     }
3682
3683     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3684         push @$cmd, $fixups->@*;
3685     }
3686
3687     if ($conf->{vmgenid}) {
3688         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3689     }
3690
3691     # add usb controllers
3692     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3693         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES, $machine_version);
3694     push @$devices, @usbcontrollers if @usbcontrollers;
3695     my $vga = parse_vga($conf->{vga});
3696
3697     my $qxlnum = vga_conf_has_spice($conf->{vga});
3698     $vga->{type} = 'qxl' if $qxlnum;
3699
3700     if (!$vga->{type}) {
3701         if ($arch eq 'aarch64') {
3702             $vga->{type} = 'virtio';
3703         } elsif (min_version($machine_version, 2, 9)) {
3704             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3705         } else {
3706             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3707         }
3708     }
3709
3710     # enable absolute mouse coordinates (needed by vnc)
3711     my $tablet = $conf->{tablet};
3712     if (!defined($tablet)) {
3713         $tablet = $defaults->{tablet};
3714         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3715         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3716     }
3717
3718     if ($tablet) {
3719         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3720         my $kbd = print_keyboarddevice_full($conf, $arch);
3721         push @$devices, '-device', $kbd if defined($kbd);
3722     }
3723
3724     my $bootorder = device_bootorder($conf);
3725
3726     # host pci device passthrough
3727     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3728         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3729
3730     # usb devices
3731     my $usb_dev_features = {};
3732     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3733
3734     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3735         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder, $machine_version);
3736     push @$devices, @usbdevices if @usbdevices;
3737
3738     # serial devices
3739     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3740         my $path = $conf->{"serial$i"} or next;
3741         if ($path eq 'socket') {
3742             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3743             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3744             # On aarch64, serial0 is the UART device. Qemu only allows
3745             # connecting UART devices via the '-serial' command line, as
3746             # the device has a fixed slot on the hardware...
3747             if ($arch eq 'aarch64' && $i == 0) {
3748                 push @$devices, '-serial', "chardev:serial$i";
3749             } else {
3750                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3751             }
3752         } else {
3753             die "no such serial device\n" if ! -c $path;
3754             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3755             push @$devices, '-device', "isa-serial,chardev=serial$i";
3756         }
3757     }
3758
3759     # parallel devices
3760     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3761         if (my $path = $conf->{"parallel$i"}) {
3762             die "no such parallel device\n" if ! -c $path;
3763             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3764             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3765             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3766         }
3767     }
3768
3769     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3770         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3771         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3772         push @$devices, @$audio_devs;
3773     }
3774
3775     add_tpm_device($vmid, $devices, $conf);
3776
3777     my $sockets = 1;
3778     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3779     $sockets = $conf->{sockets} if  $conf->{sockets};
3780
3781     my $cores = $conf->{cores} || 1;
3782
3783     my $maxcpus = $sockets * $cores;
3784
3785     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3786
3787     my $allowed_vcpus = $cpuinfo->{cpus};
3788
3789     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3790
3791     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3792         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3793         for (my $i = 2; $i <= $vcpus; $i++)  {
3794             my $cpustr = print_cpu_device($conf,$i);
3795             push @$cmd, '-device', $cpustr;
3796         }
3797
3798     } else {
3799
3800         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3801     }
3802     push @$cmd, '-nodefaults';
3803
3804     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3805
3806     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3807
3808     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3809
3810     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3811         push @$devices, '-device', print_vga_device(
3812             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3813
3814         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3815
3816         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3817         push @$cmd,  '-vnc', "unix:$socket,password=on";
3818     } else {
3819         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3820         push @$cmd, '-nographic';
3821     }
3822
3823     # time drift fix
3824     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3825     my $useLocaltime = $conf->{localtime};
3826
3827     if ($winversion >= 5) { # windows
3828         $useLocaltime = 1 if !defined($conf->{localtime});
3829
3830         # use time drift fix when acpi is enabled
3831         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3832             $tdf = 1 if !defined($conf->{tdf});
3833         }
3834     }
3835
3836     if ($winversion >= 6) {
3837         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3838         push @$cmd, '-no-hpet';
3839     }
3840
3841     push @$rtcFlags, 'driftfix=slew' if $tdf;
3842
3843     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3844         push @$rtcFlags, "base=$conf->{startdate}";
3845     } elsif ($useLocaltime) {
3846         push @$rtcFlags, 'base=localtime';
3847     }
3848
3849     if ($forcecpu) {
3850         push @$cmd, '-cpu', $forcecpu;
3851     } else {
3852         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3853     }
3854
3855     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3856
3857     push @$cmd, '-S' if $conf->{freeze};
3858
3859     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3860
3861     my $guest_agent = parse_guest_agent($conf);
3862
3863     if ($guest_agent->{enabled}) {
3864         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3865         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3866
3867         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3868             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3869             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3870             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3871         } elsif ($guest_agent->{type} eq 'isa') {
3872             push @$devices, '-device', "isa-serial,chardev=qga0";
3873         }
3874     }
3875
3876     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3877     if ($rng && $version_guard->(4, 1, 2)) {
3878         check_rng_source($rng->{source});
3879
3880         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3881         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3882         my $limiter_str = "";
3883         if ($max_bytes) {
3884             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3885         }
3886
3887         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3888         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3889         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3890     }
3891
3892     my $spice_port;
3893
3894     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3895         if ($qxlnum > 1) {
3896             if ($winversion){
3897                 for (my $i = 1; $i < $qxlnum; $i++){
3898                     push @$devices, '-device', print_vga_device(
3899                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3900                 }
3901             } else {
3902                 # assume other OS works like Linux
3903                 my ($ram, $vram) = ("134217728", "67108864");
3904                 if ($vga->{memory}) {
3905                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3906                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3907                 }
3908                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3909                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3910             }
3911         }
3912
3913         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3914
3915         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3916         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3917         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3918
3919         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3920         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3921         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3922
3923         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3924         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3925
3926         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3927         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3928         if ($spice_enhancement->{foldersharing}) {
3929             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3930             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3931         }
3932
3933         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3934         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3935             if $spice_enhancement->{videostreaming};
3936
3937         push @$devices, '-spice', "$spice_opts";
3938     }
3939
3940     # enable balloon by default, unless explicitly disabled
3941     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3942         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3943         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3944         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3945         push @$devices, '-device', $ballooncmd;
3946     }
3947
3948     if ($conf->{watchdog}) {
3949         my $wdopts = parse_watchdog($conf->{watchdog});
3950         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3951         my $watchdog = $wdopts->{model} || 'i6300esb';
3952         push @$devices, '-device', "$watchdog$pciaddr";
3953         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3954     }
3955
3956     my $vollist = [];
3957     my $scsicontroller = {};
3958     my $ahcicontroller = {};
3959     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3960
3961     # Add iscsi initiator name if available
3962     if (my $initiator = get_initiator_name()) {
3963         push @$devices, '-iscsi', "initiator-name=$initiator";
3964     }
3965
3966     PVE::QemuConfig->foreach_volume($conf, sub {
3967         my ($ds, $drive) = @_;
3968
3969         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3970             check_volume_storage_type($storecfg, $drive->{file});
3971             push @$vollist, $drive->{file};
3972         }
3973
3974         # ignore efidisk here, already added in bios/fw handling code above
3975         return if $drive->{interface} eq 'efidisk';
3976         # similar for TPM
3977         return if $drive->{interface} eq 'tpmstate';
3978
3979         $use_virtio = 1 if $ds =~ m/^virtio/;
3980
3981         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3982
3983         if ($drive->{interface} eq 'virtio'){
3984            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3985         }
3986
3987         if ($drive->{interface} eq 'scsi') {
3988
3989             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3990
3991             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3992                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3993
3994             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3995             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3996
3997             my $iothread = '';
3998             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3999                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4000                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4001             } elsif ($drive->{iothread}) {
4002                 log_warn(
4003                     "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n"
4004                 );
4005             }
4006
4007             my $queues = '';
4008             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4009                 $queues = ",num_queues=$drive->{queues}";
4010             }
4011
4012             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
4013                 if !$scsicontroller->{$controller};
4014             $scsicontroller->{$controller}=1;
4015         }
4016
4017         if ($drive->{interface} eq 'sata') {
4018             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
4019             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4020             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
4021                 if !$ahcicontroller->{$controller};
4022             $ahcicontroller->{$controller}=1;
4023         }
4024
4025         my $pbs_conf = $pbs_backing->{$ds};
4026         my $pbs_name = undef;
4027         if ($pbs_conf) {
4028             $pbs_name = "drive-$ds-pbs";
4029             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
4030         }
4031
4032         my $drive_cmd = print_drive_commandline_full(
4033             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
4034
4035         # extra protection for templates, but SATA and IDE don't support it..
4036         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
4037
4038         push @$devices, '-drive',$drive_cmd;
4039         push @$devices, '-device', print_drivedevice_full(
4040             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4041     });
4042
4043     for (my $i = 0; $i < $MAX_NETS; $i++) {
4044         my $netname = "net$i";
4045
4046         next if !$conf->{$netname};
4047         my $d = parse_net($conf->{$netname});
4048         next if !$d;
4049         # save the MAC addr here (could be auto-gen. in some odd setups) for FDB registering later?
4050
4051         $use_virtio = 1 if $d->{model} eq 'virtio';
4052
4053         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4054
4055         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4056         push @$devices, '-netdev', $netdevfull;
4057
4058         my $netdevicefull = print_netdevice_full(
4059             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type, $machine_version);
4060
4061         push @$devices, '-device', $netdevicefull;
4062     }
4063
4064     if ($conf->{ivshmem}) {
4065         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4066
4067         my $bus;
4068         if ($q35) {
4069             $bus = print_pcie_addr("ivshmem");
4070         } else {
4071             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4072         }
4073
4074         my $ivshmem_name = $ivshmem->{name} // $vmid;
4075         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4076
4077         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4078         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4079             .",size=$ivshmem->{size}M";
4080     }
4081
4082     # pci.4 is nested in pci.1
4083     $bridges->{1} = 1 if $bridges->{4};
4084
4085     if (!$q35) { # add pci bridges
4086         if (min_version($machine_version, 2, 3)) {
4087            $bridges->{1} = 1;
4088            $bridges->{2} = 1;
4089         }
4090         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4091     }
4092
4093     for my $k (sort {$b cmp $a} keys %$bridges) {
4094         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4095
4096         my $k_name = $k;
4097         if ($k == 2 && $legacy_igd) {
4098             $k_name = "$k-igd";
4099         }
4100         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4101         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4102
4103         if ($q35) { # add after -readconfig pve-q35.cfg
4104             splice @$devices, 2, 0, '-device', $devstr;
4105         } else {
4106             unshift @$devices, '-device', $devstr if $k > 0;
4107         }
4108     }
4109
4110     if (!$kvm) {
4111         push @$machineFlags, 'accel=tcg';
4112     }
4113
4114     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4115
4116     my $machine_type_min = $machine_type;
4117     if ($add_pve_version) {
4118         $machine_type_min =~ s/\+pve\d+$//;
4119         $machine_type_min .= "+pve$required_pve_version";
4120     }
4121     push @$machineFlags, "type=${machine_type_min}";
4122
4123     push @$cmd, @$devices;
4124     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4125     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4126     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4127
4128     if (my $vmstate = $conf->{vmstate}) {
4129         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4130         push @$vollist, $vmstate;
4131         push @$cmd, '-loadstate', $statepath;
4132         print "activating and using '$vmstate' as vmstate\n";
4133     }
4134
4135     if (PVE::QemuConfig->is_template($conf)) {
4136         # needed to workaround base volumes being read-only
4137         push @$cmd, '-snapshot';
4138     }
4139
4140     # add custom args
4141     if ($conf->{args}) {
4142         my $aa = PVE::Tools::split_args($conf->{args});
4143         push @$cmd, @$aa;
4144     }
4145
4146     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4147 }
4148
4149 sub check_rng_source {
4150     my ($source) = @_;
4151
4152     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4153     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4154         if ! -e $source;
4155
4156     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4157     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4158         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4159         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4160         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4161             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4162             ." to the host.\n";
4163     }
4164 }
4165
4166 sub spice_port {
4167     my ($vmid) = @_;
4168
4169     my $res = mon_cmd($vmid, 'query-spice');
4170
4171     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4172 }
4173
4174 sub vm_devices_list {
4175     my ($vmid) = @_;
4176
4177     my $res = mon_cmd($vmid, 'query-pci');
4178     my $devices_to_check = [];
4179     my $devices = {};
4180     foreach my $pcibus (@$res) {
4181         push @$devices_to_check, @{$pcibus->{devices}},
4182     }
4183
4184     while (@$devices_to_check) {
4185         my $to_check = [];
4186         for my $d (@$devices_to_check) {
4187             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4188             next if !$d->{'pci_bridge'};
4189
4190             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4191             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4192         }
4193         $devices_to_check = $to_check;
4194     }
4195
4196     my $resblock = mon_cmd($vmid, 'query-block');
4197     foreach my $block (@$resblock) {
4198         if($block->{device} =~ m/^drive-(\S+)/){
4199                 $devices->{$1} = 1;
4200         }
4201     }
4202
4203     my $resmice = mon_cmd($vmid, 'query-mice');
4204     foreach my $mice (@$resmice) {
4205         if ($mice->{name} eq 'QEMU HID Tablet') {
4206             $devices->{tablet} = 1;
4207             last;
4208         }
4209     }
4210
4211     # for usb devices there is no query-usb
4212     # but we can iterate over the entries in
4213     # qom-list path=/machine/peripheral
4214     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4215     foreach my $per (@$resperipheral) {
4216         if ($per->{name} =~ m/^usb(?:redirdev)?\d+$/) {
4217             $devices->{$per->{name}} = 1;
4218         }
4219     }
4220
4221     return $devices;
4222 }
4223
4224 sub vm_deviceplug {
4225     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4226
4227     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4228
4229     my $devices_list = vm_devices_list($vmid);
4230     return 1 if defined($devices_list->{$deviceid});
4231
4232     # add PCI bridge if we need it for the device
4233     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4234
4235     if ($deviceid eq 'tablet') {
4236         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4237     } elsif ($deviceid eq 'keyboard') {
4238         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4239     } elsif ($deviceid =~ m/^usbredirdev(\d+)$/) {
4240         my $id = $1;
4241         qemu_spice_usbredir_chardev_add($vmid, "usbredirchardev$id");
4242         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_spice_usbdevice($id, "xhci", $id + 1));
4243     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4244         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device, {}, $1 + 1));
4245     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4246         qemu_iothread_add($vmid, $deviceid, $device);
4247
4248         qemu_driveadd($storecfg, $vmid, $device);
4249         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4250
4251         qemu_deviceadd($vmid, $devicefull);
4252         eval { qemu_deviceaddverify($vmid, $deviceid); };
4253         if (my $err = $@) {
4254             eval { qemu_drivedel($vmid, $deviceid); };
4255             warn $@ if $@;
4256             die $err;
4257         }
4258     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4259         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4260         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4261         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4262
4263         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4264
4265         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4266             qemu_iothread_add($vmid, $deviceid, $device);
4267             $devicefull .= ",iothread=iothread-$deviceid";
4268         }
4269
4270         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4271             $devicefull .= ",num_queues=$device->{queues}";
4272         }
4273
4274         qemu_deviceadd($vmid, $devicefull);
4275         qemu_deviceaddverify($vmid, $deviceid);
4276     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4277         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4278         qemu_driveadd($storecfg, $vmid, $device);
4279
4280         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4281         eval { qemu_deviceadd($vmid, $devicefull); };
4282         if (my $err = $@) {
4283             eval { qemu_drivedel($vmid, $deviceid); };
4284             warn $@ if $@;
4285             die $err;
4286         }
4287     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4288         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4289
4290         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4291         my $machine_version = PVE::QemuServer::Machine::extract_version($machine_type);
4292         my $use_old_bios_files = undef;
4293         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4294
4295         my $netdevicefull = print_netdevice_full(
4296             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type, $machine_version);
4297         qemu_deviceadd($vmid, $netdevicefull);
4298         eval {
4299             qemu_deviceaddverify($vmid, $deviceid);
4300             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4301         };
4302         if (my $err = $@) {
4303             eval { qemu_netdevdel($vmid, $deviceid); };
4304             warn $@ if $@;
4305             die $err;
4306         }
4307     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4308         my $bridgeid = $2;
4309         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4310         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4311
4312         qemu_deviceadd($vmid, $devicefull);
4313         qemu_deviceaddverify($vmid, $deviceid);
4314     } else {
4315         die "can't hotplug device '$deviceid'\n";
4316     }
4317
4318     return 1;
4319 }
4320
4321 # fixme: this should raise exceptions on error!
4322 sub vm_deviceunplug {
4323     my ($vmid, $conf, $deviceid) = @_;
4324
4325     my $devices_list = vm_devices_list($vmid);
4326     return 1 if !defined($devices_list->{$deviceid});
4327
4328     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4329     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4330
4331     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard' || $deviceid eq 'xhci') {
4332         qemu_devicedel($vmid, $deviceid);
4333     } elsif ($deviceid =~ m/^usbredirdev\d+$/) {
4334         qemu_devicedel($vmid, $deviceid);
4335         qemu_devicedelverify($vmid, $deviceid);
4336     } elsif ($deviceid =~ m/^usb\d+$/) {
4337         qemu_devicedel($vmid, $deviceid);
4338         qemu_devicedelverify($vmid, $deviceid);
4339     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4340         my $device = parse_drive($deviceid, $conf->{$deviceid});
4341
4342         qemu_devicedel($vmid, $deviceid);
4343         qemu_devicedelverify($vmid, $deviceid);
4344         qemu_drivedel($vmid, $deviceid);
4345         qemu_iothread_del($vmid, $deviceid, $device);
4346     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4347         qemu_devicedel($vmid, $deviceid);
4348         qemu_devicedelverify($vmid, $deviceid);
4349     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4350         my $device = parse_drive($deviceid, $conf->{$deviceid});
4351
4352         qemu_devicedel($vmid, $deviceid);
4353         qemu_devicedelverify($vmid, $deviceid);
4354         qemu_drivedel($vmid, $deviceid);
4355         qemu_deletescsihw($conf, $vmid, $deviceid);
4356
4357         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4358             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4359     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4360         qemu_devicedel($vmid, $deviceid);
4361         qemu_devicedelverify($vmid, $deviceid);
4362         qemu_netdevdel($vmid, $deviceid);
4363     } else {
4364         die "can't unplug device '$deviceid'\n";
4365     }
4366
4367     return 1;
4368 }
4369
4370 sub qemu_spice_usbredir_chardev_add {
4371     my ($vmid, $id) = @_;
4372
4373     mon_cmd($vmid, "chardev-add" , (
4374         id => $id,
4375         backend => {
4376             type => 'spicevmc',
4377             data => {
4378                 type => "usbredir",
4379             },
4380         },
4381     ));
4382 }
4383
4384 sub qemu_deviceadd {
4385     my ($vmid, $devicefull) = @_;
4386
4387     $devicefull = "driver=".$devicefull;
4388     my %options =  split(/[=,]/, $devicefull);
4389
4390     mon_cmd($vmid, "device_add" , %options);
4391 }
4392
4393 sub qemu_devicedel {
4394     my ($vmid, $deviceid) = @_;
4395
4396     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4397 }
4398
4399 sub qemu_iothread_add {
4400     my ($vmid, $deviceid, $device) = @_;
4401
4402     if ($device->{iothread}) {
4403         my $iothreads = vm_iothreads_list($vmid);
4404         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4405     }
4406 }
4407
4408 sub qemu_iothread_del {
4409     my ($vmid, $deviceid, $device) = @_;
4410
4411     if ($device->{iothread}) {
4412         my $iothreads = vm_iothreads_list($vmid);
4413         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4414     }
4415 }
4416
4417 sub qemu_objectadd {
4418     my ($vmid, $objectid, $qomtype) = @_;
4419
4420     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4421
4422     return 1;
4423 }
4424
4425 sub qemu_objectdel {
4426     my ($vmid, $objectid) = @_;
4427
4428     mon_cmd($vmid, "object-del", id => $objectid);
4429
4430     return 1;
4431 }
4432
4433 sub qemu_driveadd {
4434     my ($storecfg, $vmid, $device) = @_;
4435
4436     my $kvmver = get_running_qemu_version($vmid);
4437     my $io_uring = min_version($kvmver, 6, 0);
4438     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4439     $drive =~ s/\\/\\\\/g;
4440     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4441
4442     # If the command succeeds qemu prints: "OK"
4443     return 1 if $ret =~ m/OK/s;
4444
4445     die "adding drive failed: $ret\n";
4446 }
4447
4448 sub qemu_drivedel {
4449     my ($vmid, $deviceid) = @_;
4450
4451     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4452     $ret =~ s/^\s+//;
4453
4454     return 1 if $ret eq "";
4455
4456     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4457     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4458
4459     die "deleting drive $deviceid failed : $ret\n";
4460 }
4461
4462 sub qemu_deviceaddverify {
4463     my ($vmid, $deviceid) = @_;
4464
4465     for (my $i = 0; $i <= 5; $i++) {
4466          my $devices_list = vm_devices_list($vmid);
4467          return 1 if defined($devices_list->{$deviceid});
4468          sleep 1;
4469     }
4470
4471     die "error on hotplug device '$deviceid'\n";
4472 }
4473
4474
4475 sub qemu_devicedelverify {
4476     my ($vmid, $deviceid) = @_;
4477
4478     # need to verify that the device is correctly removed as device_del
4479     # is async and empty return is not reliable
4480
4481     for (my $i = 0; $i <= 5; $i++) {
4482          my $devices_list = vm_devices_list($vmid);
4483          return 1 if !defined($devices_list->{$deviceid});
4484          sleep 1;
4485     }
4486
4487     die "error on hot-unplugging device '$deviceid'\n";
4488 }
4489
4490 sub qemu_findorcreatescsihw {
4491     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4492
4493     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4494
4495     my $scsihwid="$controller_prefix$controller";
4496     my $devices_list = vm_devices_list($vmid);
4497
4498     if (!defined($devices_list->{$scsihwid})) {
4499         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4500     }
4501
4502     return 1;
4503 }
4504
4505 sub qemu_deletescsihw {
4506     my ($conf, $vmid, $opt) = @_;
4507
4508     my $device = parse_drive($opt, $conf->{$opt});
4509
4510     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4511         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4512         return 1;
4513     }
4514
4515     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4516
4517     my $devices_list = vm_devices_list($vmid);
4518     foreach my $opt (keys %{$devices_list}) {
4519         if (is_valid_drivename($opt)) {
4520             my $drive = parse_drive($opt, $conf->{$opt});
4521             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4522                 return 1;
4523             }
4524         }
4525     }
4526
4527     my $scsihwid="scsihw$controller";
4528
4529     vm_deviceunplug($vmid, $conf, $scsihwid);
4530
4531     return 1;
4532 }
4533
4534 sub qemu_add_pci_bridge {
4535     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4536
4537     my $bridges = {};
4538
4539     my $bridgeid;
4540
4541     print_pci_addr($device, $bridges, $arch, $machine_type);
4542
4543     while (my ($k, $v) = each %$bridges) {
4544         $bridgeid = $k;
4545     }
4546     return 1 if !defined($bridgeid) || $bridgeid < 1;
4547
4548     my $bridge = "pci.$bridgeid";
4549     my $devices_list = vm_devices_list($vmid);
4550
4551     if (!defined($devices_list->{$bridge})) {
4552         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4553     }
4554
4555     return 1;
4556 }
4557
4558 sub qemu_set_link_status {
4559     my ($vmid, $device, $up) = @_;
4560
4561     mon_cmd($vmid, "set_link", name => $device,
4562                up => $up ? JSON::true : JSON::false);
4563 }
4564
4565 sub qemu_netdevadd {
4566     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4567
4568     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4569     my %options =  split(/[=,]/, $netdev);
4570
4571     if (defined(my $vhost = $options{vhost})) {
4572         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4573     }
4574
4575     if (defined(my $queues = $options{queues})) {
4576         $options{queues} = $queues + 0;
4577     }
4578
4579     mon_cmd($vmid, "netdev_add",  %options);
4580     return 1;
4581 }
4582
4583 sub qemu_netdevdel {
4584     my ($vmid, $deviceid) = @_;
4585
4586     mon_cmd($vmid, "netdev_del", id => $deviceid);
4587 }
4588
4589 sub qemu_usb_hotplug {
4590     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4591
4592     return if !$device;
4593
4594     # remove the old one first
4595     vm_deviceunplug($vmid, $conf, $deviceid);
4596
4597     # check if xhci controller is necessary and available
4598     my $devicelist = vm_devices_list($vmid);
4599
4600     if (!$devicelist->{xhci}) {
4601         my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4602         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_qemu_xhci_controller($pciaddr));
4603     }
4604
4605     # print_usbdevice_full expects the parsed device
4606     my $d = parse_usb_device($device->{host});
4607     $d->{usb3} = $device->{usb3};
4608
4609     # add the new one
4610     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4611 }
4612
4613 sub qemu_cpu_hotplug {
4614     my ($vmid, $conf, $vcpus) = @_;
4615
4616     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4617
4618     my $sockets = 1;
4619     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4620     $sockets = $conf->{sockets} if  $conf->{sockets};
4621     my $cores = $conf->{cores} || 1;
4622     my $maxcpus = $sockets * $cores;
4623
4624     $vcpus = $maxcpus if !$vcpus;
4625
4626     die "you can't add more vcpus than maxcpus\n"
4627         if $vcpus > $maxcpus;
4628
4629     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4630
4631     if ($vcpus < $currentvcpus) {
4632
4633         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4634
4635             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4636                 qemu_devicedel($vmid, "cpu$i");
4637                 my $retry = 0;
4638                 my $currentrunningvcpus = undef;
4639                 while (1) {
4640                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4641                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4642                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4643                     $retry++;
4644                     sleep 1;
4645                 }
4646                 #update conf after each succesfull cpu unplug
4647                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4648                 PVE::QemuConfig->write_config($vmid, $conf);
4649             }
4650         } else {
4651             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4652         }
4653
4654         return;
4655     }
4656
4657     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4658     die "vcpus in running vm does not match its configuration\n"
4659         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4660
4661     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4662
4663         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4664             my $cpustr = print_cpu_device($conf, $i);
4665             qemu_deviceadd($vmid, $cpustr);
4666
4667             my $retry = 0;
4668             my $currentrunningvcpus = undef;
4669             while (1) {
4670                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4671                 last if scalar(@{$currentrunningvcpus}) == $i;
4672                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4673                 sleep 1;
4674                 $retry++;
4675             }
4676             #update conf after each succesfull cpu hotplug
4677             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4678             PVE::QemuConfig->write_config($vmid, $conf);
4679         }
4680     } else {
4681
4682         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4683             mon_cmd($vmid, "cpu-add", id => int($i));
4684         }
4685     }
4686 }
4687
4688 sub qemu_block_set_io_throttle {
4689     my ($vmid, $deviceid,
4690         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4691         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4692         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4693         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4694
4695     return if !check_running($vmid) ;
4696
4697     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4698         bps => int($bps),
4699         bps_rd => int($bps_rd),
4700         bps_wr => int($bps_wr),
4701         iops => int($iops),
4702         iops_rd => int($iops_rd),
4703         iops_wr => int($iops_wr),
4704         bps_max => int($bps_max),
4705         bps_rd_max => int($bps_rd_max),
4706         bps_wr_max => int($bps_wr_max),
4707         iops_max => int($iops_max),
4708         iops_rd_max => int($iops_rd_max),
4709         iops_wr_max => int($iops_wr_max),
4710         bps_max_length => int($bps_max_length),
4711         bps_rd_max_length => int($bps_rd_max_length),
4712         bps_wr_max_length => int($bps_wr_max_length),
4713         iops_max_length => int($iops_max_length),
4714         iops_rd_max_length => int($iops_rd_max_length),
4715         iops_wr_max_length => int($iops_wr_max_length),
4716     );
4717
4718 }
4719
4720 sub qemu_block_resize {
4721     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4722
4723     my $running = check_running($vmid);
4724
4725     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4726
4727     return if !$running;
4728
4729     my $padding = (1024 - $size % 1024) % 1024;
4730     $size = $size + $padding;
4731
4732     mon_cmd(
4733         $vmid,
4734         "block_resize",
4735         device => $deviceid,
4736         size => int($size),
4737         timeout => 60,
4738     );
4739 }
4740
4741 sub qemu_volume_snapshot {
4742     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4743
4744     my $running = check_running($vmid);
4745
4746     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4747         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4748     } else {
4749         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4750     }
4751 }
4752
4753 sub qemu_volume_snapshot_delete {
4754     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4755
4756     my $running = check_running($vmid);
4757
4758     if($running) {
4759
4760         $running = undef;
4761         my $conf = PVE::QemuConfig->load_config($vmid);
4762         PVE::QemuConfig->foreach_volume($conf, sub {
4763             my ($ds, $drive) = @_;
4764             $running = 1 if $drive->{file} eq $volid;
4765         });
4766     }
4767
4768     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4769         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4770     } else {
4771         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4772     }
4773 }
4774
4775 sub set_migration_caps {
4776     my ($vmid, $savevm) = @_;
4777
4778     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4779
4780     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4781     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4782
4783     my $cap_ref = [];
4784
4785     my $enabled_cap = {
4786         "auto-converge" => 1,
4787         "xbzrle" => 1,
4788         "x-rdma-pin-all" => 0,
4789         "zero-blocks" => 0,
4790         "compress" => 0,
4791         "dirty-bitmaps" => $dirty_bitmaps,
4792     };
4793
4794     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4795
4796     for my $supported_capability (@$supported_capabilities) {
4797         push @$cap_ref, {
4798             capability => $supported_capability->{capability},
4799             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4800         };
4801     }
4802
4803     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4804 }
4805
4806 sub foreach_volid {
4807     my ($conf, $func, @param) = @_;
4808
4809     my $volhash = {};
4810
4811     my $test_volid = sub {
4812         my ($key, $drive, $snapname) = @_;
4813
4814         my $volid = $drive->{file};
4815         return if !$volid;
4816
4817         $volhash->{$volid}->{cdrom} //= 1;
4818         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4819
4820         my $replicate = $drive->{replicate} // 1;
4821         $volhash->{$volid}->{replicate} //= 0;
4822         $volhash->{$volid}->{replicate} = 1 if $replicate;
4823
4824         $volhash->{$volid}->{shared} //= 0;
4825         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4826
4827         $volhash->{$volid}->{referenced_in_config} //= 0;
4828         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4829
4830         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4831             if defined($snapname);
4832
4833         my $size = $drive->{size};
4834         $volhash->{$volid}->{size} //= $size if $size;
4835
4836         $volhash->{$volid}->{is_vmstate} //= 0;
4837         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4838
4839         $volhash->{$volid}->{is_tpmstate} //= 0;
4840         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4841
4842         $volhash->{$volid}->{is_unused} //= 0;
4843         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4844
4845         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4846     };
4847
4848     my $include_opts = {
4849         extra_keys => ['vmstate'],
4850         include_unused => 1,
4851     };
4852
4853     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4854     foreach my $snapname (keys %{$conf->{snapshots}}) {
4855         my $snap = $conf->{snapshots}->{$snapname};
4856         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4857     }
4858
4859     foreach my $volid (keys %$volhash) {
4860         &$func($volid, $volhash->{$volid}, @param);
4861     }
4862 }
4863
4864 my $fast_plug_option = {
4865     'lock' => 1,
4866     'name' => 1,
4867     'onboot' => 1,
4868     'shares' => 1,
4869     'startup' => 1,
4870     'description' => 1,
4871     'protection' => 1,
4872     'vmstatestorage' => 1,
4873     'hookscript' => 1,
4874     'tags' => 1,
4875 };
4876
4877 for my $opt (keys %$confdesc_cloudinit) {
4878     $fast_plug_option->{$opt} = 1;
4879 };
4880
4881 # hotplug changes in [PENDING]
4882 # $selection hash can be used to only apply specified options, for
4883 # example: { cores => 1 } (only apply changed 'cores')
4884 # $errors ref is used to return error messages
4885 sub vmconfig_hotplug_pending {
4886     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4887
4888     my $defaults = load_defaults();
4889     my $arch = get_vm_arch($conf);
4890     my $machine_type = get_vm_machine($conf, undef, $arch);
4891
4892     # commit values which do not have any impact on running VM first
4893     # Note: those option cannot raise errors, we we do not care about
4894     # $selection and always apply them.
4895
4896     my $add_error = sub {
4897         my ($opt, $msg) = @_;
4898         $errors->{$opt} = "hotplug problem - $msg";
4899     };
4900
4901     my $changes = 0;
4902     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4903         if ($fast_plug_option->{$opt}) {
4904             $conf->{$opt} = $conf->{pending}->{$opt};
4905             delete $conf->{pending}->{$opt};
4906             $changes = 1;
4907         }
4908     }
4909
4910     if ($changes) {
4911         PVE::QemuConfig->write_config($vmid, $conf);
4912     }
4913
4914     my $ostype = $conf->{ostype};
4915     my $version = extract_version($machine_type, get_running_qemu_version($vmid));
4916     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4917     my $usb_hotplug = $hotplug_features->{usb}
4918         && min_version($version, 7, 1)
4919         && defined($ostype) && ($ostype eq 'l26' || windows_version($ostype) > 7);
4920
4921     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4922     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4923     foreach my $opt (sort keys %$pending_delete_hash) {
4924         next if $selection && !$selection->{$opt};
4925         my $force = $pending_delete_hash->{$opt}->{force};
4926         eval {
4927             if ($opt eq 'hotplug') {
4928                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4929             } elsif ($opt eq 'tablet') {
4930                 die "skip\n" if !$hotplug_features->{usb};
4931                 if ($defaults->{tablet}) {
4932                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4933                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4934                         if $arch eq 'aarch64';
4935                 } else {
4936                     vm_deviceunplug($vmid, $conf, 'tablet');
4937                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4938                 }
4939             } elsif ($opt =~ m/^usb(\d+)$/) {
4940                 my $index = $1;
4941                 die "skip\n" if !$usb_hotplug;
4942                 vm_deviceunplug($vmid, $conf, "usbredirdev$index"); # if it's a spice port
4943                 vm_deviceunplug($vmid, $conf, $opt);
4944             } elsif ($opt eq 'vcpus') {
4945                 die "skip\n" if !$hotplug_features->{cpu};
4946                 qemu_cpu_hotplug($vmid, $conf, undef);
4947             } elsif ($opt eq 'balloon') {
4948                 # enable balloon device is not hotpluggable
4949                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4950                 # here we reset the ballooning value to memory
4951                 my $balloon = $conf->{memory} || $defaults->{memory};
4952                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4953             } elsif ($fast_plug_option->{$opt}) {
4954                 # do nothing
4955             } elsif ($opt =~ m/^net(\d+)$/) {
4956                 die "skip\n" if !$hotplug_features->{network};
4957                 vm_deviceunplug($vmid, $conf, $opt);
4958             } elsif (is_valid_drivename($opt)) {
4959                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4960                 vm_deviceunplug($vmid, $conf, $opt);
4961                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4962             } elsif ($opt =~ m/^memory$/) {
4963                 die "skip\n" if !$hotplug_features->{memory};
4964                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4965             } elsif ($opt eq 'cpuunits') {
4966                 $cgroup->change_cpu_shares(undef);
4967             } elsif ($opt eq 'cpulimit') {
4968                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4969             } else {
4970                 die "skip\n";
4971             }
4972         };
4973         if (my $err = $@) {
4974             &$add_error($opt, $err) if $err ne "skip\n";
4975         } else {
4976             delete $conf->{$opt};
4977             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4978         }
4979     }
4980
4981     foreach my $opt (keys %{$conf->{pending}}) {
4982         next if $selection && !$selection->{$opt};
4983         my $value = $conf->{pending}->{$opt};
4984         eval {
4985             if ($opt eq 'hotplug') {
4986                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4987             } elsif ($opt eq 'tablet') {
4988                 die "skip\n" if !$hotplug_features->{usb};
4989                 if ($value == 1) {
4990                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4991                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4992                         if $arch eq 'aarch64';
4993                 } elsif ($value == 0) {
4994                     vm_deviceunplug($vmid, $conf, 'tablet');
4995                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4996                 }
4997             } elsif ($opt =~ m/^usb(\d+)$/) {
4998                 my $index = $1;
4999                 die "skip\n" if !$usb_hotplug;
5000                 my $d = eval { parse_property_string($usbdesc->{format}, $value) };
5001                 my $id = $opt;
5002                 if ($d->{host} eq 'spice')  {
5003                     $id = "usbredirdev$index";
5004                 }
5005                 qemu_usb_hotplug($storecfg, $conf, $vmid, $id, $d, $arch, $machine_type);
5006             } elsif ($opt eq 'vcpus') {
5007                 die "skip\n" if !$hotplug_features->{cpu};
5008                 qemu_cpu_hotplug($vmid, $conf, $value);
5009             } elsif ($opt eq 'balloon') {
5010                 # enable/disable balloning device is not hotpluggable
5011                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5012                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5013                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5014
5015                 # allow manual ballooning if shares is set to zero
5016                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5017                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5018                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5019                 }
5020             } elsif ($opt =~ m/^net(\d+)$/) {
5021                 # some changes can be done without hotplug
5022                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5023                                     $vmid, $opt, $value, $arch, $machine_type);
5024             } elsif (is_valid_drivename($opt)) {
5025                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
5026                 # some changes can be done without hotplug
5027                 my $drive = parse_drive($opt, $value);
5028                 if (drive_is_cloudinit($drive)) {
5029                     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5030                 }
5031                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5032                                      $vmid, $opt, $value, $arch, $machine_type);
5033             } elsif ($opt =~ m/^memory$/) { #dimms
5034                 die "skip\n" if !$hotplug_features->{memory};
5035                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5036             } elsif ($opt eq 'cpuunits') {
5037                 my $new_cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{pending}->{$opt}); #clamp
5038                 $cgroup->change_cpu_shares($new_cpuunits);
5039             } elsif ($opt eq 'cpulimit') {
5040                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5041                 $cgroup->change_cpu_quota($cpulimit, 100000);
5042             } elsif ($opt eq 'agent') {
5043                 vmconfig_update_agent($conf, $opt, $value);
5044             } else {
5045                 die "skip\n";  # skip non-hot-pluggable options
5046             }
5047         };
5048         if (my $err = $@) {
5049             &$add_error($opt, $err) if $err ne "skip\n";
5050         } else {
5051             $conf->{$opt} = $value;
5052             delete $conf->{pending}->{$opt};
5053         }
5054     }
5055
5056     # unplug xhci controller if no usb device is left
5057     if ($usb_hotplug) {
5058         my $has_usb = 0;
5059         for (my $i = 0; $i < $MAX_USB_DEVICES; $i++) {
5060             next if !defined($conf->{"usb$i"});
5061             $has_usb = 1;
5062             last;
5063         }
5064         if (!$has_usb) {
5065             vm_deviceunplug($vmid, $conf, 'xhci');
5066         }
5067     }
5068
5069     PVE::QemuConfig->write_config($vmid, $conf);
5070
5071     if($hotplug_features->{cloudinit}) {
5072         my $pending = PVE::QemuServer::Cloudinit::get_pending_config($conf, $vmid);
5073         my $regenerate = undef;
5074         for my $item (@$pending) {
5075             $regenerate = 1 if defined($item->{delete}) or defined($item->{pending});
5076         }
5077         PVE::QemuServer::vmconfig_update_cloudinit_drive($storecfg, $conf, $vmid) if $regenerate;
5078     }
5079 }
5080
5081 sub try_deallocate_drive {
5082     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5083
5084     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5085         my $volid = $drive->{file};
5086         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5087             my $sid = PVE::Storage::parse_volume_id($volid);
5088             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5089
5090             # check if the disk is really unused
5091             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5092                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5093             PVE::Storage::vdisk_free($storecfg, $volid);
5094             return 1;
5095         } else {
5096             # If vm is not owner of this disk remove from config
5097             return 1;
5098         }
5099     }
5100
5101     return;
5102 }
5103
5104 sub vmconfig_delete_or_detach_drive {
5105     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5106
5107     my $drive = parse_drive($opt, $conf->{$opt});
5108
5109     my $rpcenv = PVE::RPCEnvironment::get();
5110     my $authuser = $rpcenv->get_user();
5111
5112     if ($force) {
5113         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5114         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5115     } else {
5116         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5117     }
5118 }
5119
5120
5121
5122 sub vmconfig_apply_pending {
5123     my ($vmid, $conf, $storecfg, $errors) = @_;
5124
5125     return if !scalar(keys %{$conf->{pending}});
5126
5127     my $add_apply_error = sub {
5128         my ($opt, $msg) = @_;
5129         my $err_msg = "unable to apply pending change $opt : $msg";
5130         $errors->{$opt} = $err_msg;
5131         warn $err_msg;
5132     };
5133
5134     # cold plug
5135
5136     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5137     foreach my $opt (sort keys %$pending_delete_hash) {
5138         my $force = $pending_delete_hash->{$opt}->{force};
5139         eval {
5140             if ($opt =~ m/^unused/) {
5141                 die "internal error";
5142             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5143                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5144             }
5145         };
5146         if (my $err = $@) {
5147             $add_apply_error->($opt, $err);
5148         } else {
5149             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5150             delete $conf->{$opt};
5151         }
5152     }
5153
5154     PVE::QemuConfig->cleanup_pending($conf);
5155
5156     my $generate_cloudnit = undef;
5157
5158     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5159         next if $opt eq 'delete'; # just to be sure
5160         eval {
5161             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5162                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5163             }
5164         };
5165         if (my $err = $@) {
5166             $add_apply_error->($opt, $err);
5167         } else {
5168
5169             if (is_valid_drivename($opt)) {
5170                 my $drive = parse_drive($opt, $conf->{pending}->{$opt});
5171                 $generate_cloudnit = 1 if drive_is_cloudinit($drive);
5172             }
5173
5174             $conf->{$opt} = delete $conf->{pending}->{$opt};
5175         }
5176     }
5177
5178     # write all changes at once to avoid unnecessary i/o
5179     PVE::QemuConfig->write_config($vmid, $conf);
5180     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if $generate_cloudnit;
5181 }
5182
5183 sub vmconfig_update_net {
5184     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5185
5186     my $newnet = parse_net($value);
5187
5188     if ($conf->{$opt}) {
5189         my $oldnet = parse_net($conf->{$opt});
5190
5191         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5192             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5193             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5194             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5195
5196             # for non online change, we try to hot-unplug
5197             die "skip\n" if !$hotplug;
5198             vm_deviceunplug($vmid, $conf, $opt);
5199         } else {
5200
5201             die "internal error" if $opt !~ m/net(\d+)/;
5202             my $iface = "tap${vmid}i$1";
5203
5204             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5205                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5206                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5207                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5208                 PVE::Network::tap_unplug($iface);
5209
5210                 if ($have_sdn) {
5211                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5212                 } else {
5213                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5214                 }
5215             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5216                 # Rate can be applied on its own but any change above needs to
5217                 # include the rate in tap_plug since OVS resets everything.
5218                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5219             }
5220
5221             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5222                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5223             }
5224
5225             return 1;
5226         }
5227     }
5228
5229     if ($hotplug) {
5230         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5231     } else {
5232         die "skip\n";
5233     }
5234 }
5235
5236 sub vmconfig_update_agent {
5237     my ($conf, $opt, $value) = @_;
5238
5239     die "skip\n" if !$conf->{$opt};
5240
5241     my $hotplug_options = { fstrim_cloned_disks => 1 };
5242
5243     my $old_agent = parse_guest_agent($conf);
5244     my $agent = parse_guest_agent({$opt => $value});
5245
5246     for my $option (keys %$agent) { # added/changed options
5247         next if defined($hotplug_options->{$option});
5248         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5249     }
5250
5251     for my $option (keys %$old_agent) { # removed options
5252         next if defined($hotplug_options->{$option});
5253         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5254     }
5255
5256     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5257 }
5258
5259 sub vmconfig_update_disk {
5260     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5261
5262     my $drive = parse_drive($opt, $value);
5263
5264     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5265         my $media = $drive->{media} || 'disk';
5266         my $oldmedia = $old_drive->{media} || 'disk';
5267         die "unable to change media type\n" if $media ne $oldmedia;
5268
5269         if (!drive_is_cdrom($old_drive)) {
5270
5271             if ($drive->{file} ne $old_drive->{file}) {
5272
5273                 die "skip\n" if !$hotplug;
5274
5275                 # unplug and register as unused
5276                 vm_deviceunplug($vmid, $conf, $opt);
5277                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5278
5279             } else {
5280                 # update existing disk
5281
5282                 # skip non hotpluggable value
5283                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5284                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5285                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5286                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5287                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5288                     die "skip\n";
5289                 }
5290
5291                 # apply throttle
5292                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5293                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5294                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5295                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5296                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5297                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5298                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5299                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5300                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5301                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5302                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5303                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5304                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5305                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5306                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5307                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5308                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5309                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5310
5311                     qemu_block_set_io_throttle(
5312                         $vmid,"drive-$opt",
5313                         ($drive->{mbps} || 0)*1024*1024,
5314                         ($drive->{mbps_rd} || 0)*1024*1024,
5315                         ($drive->{mbps_wr} || 0)*1024*1024,
5316                          $drive->{iops} || 0,
5317                          $drive->{iops_rd} || 0,
5318                          $drive->{iops_wr} || 0,
5319                         ($drive->{mbps_max} || 0)*1024*1024,
5320                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5321                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5322                          $drive->{iops_max} || 0,
5323                          $drive->{iops_rd_max} || 0,
5324                          $drive->{iops_wr_max} || 0,
5325                          $drive->{bps_max_length} || 1,
5326                          $drive->{bps_rd_max_length} || 1,
5327                          $drive->{bps_wr_max_length} || 1,
5328                          $drive->{iops_max_length} || 1,
5329                          $drive->{iops_rd_max_length} || 1,
5330                          $drive->{iops_wr_max_length} || 1,
5331                     );
5332
5333                 }
5334
5335                 return 1;
5336             }
5337
5338         } else { # cdrom
5339
5340             if ($drive->{file} eq 'none') {
5341                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5342                 if (drive_is_cloudinit($old_drive)) {
5343                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5344                 }
5345             } else {
5346                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5347
5348                 # force eject if locked
5349                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5350
5351                 if ($path) {
5352                     mon_cmd($vmid, "blockdev-change-medium",
5353                         id => "$opt", filename => "$path");
5354                 }
5355             }
5356
5357             return 1;
5358         }
5359     }
5360
5361     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5362     # hotplug new disks
5363     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5364     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5365 }
5366
5367 sub vmconfig_update_cloudinit_drive {
5368     my ($storecfg, $conf, $vmid) = @_;
5369
5370     my $cloudinit_ds = undef;
5371     my $cloudinit_drive = undef;
5372
5373     PVE::QemuConfig->foreach_volume($conf, sub {
5374         my ($ds, $drive) = @_;
5375         if (PVE::QemuServer::drive_is_cloudinit($drive)) {
5376             $cloudinit_ds = $ds;
5377             $cloudinit_drive = $drive;
5378         }
5379     });
5380
5381     return if !$cloudinit_drive;
5382
5383     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5384     my $running = PVE::QemuServer::check_running($vmid);
5385
5386     if ($running) {
5387         my $path = PVE::Storage::path($storecfg, $cloudinit_drive->{file});
5388         if ($path) {
5389             mon_cmd($vmid, "eject", force => JSON::true, id => "$cloudinit_ds");
5390             mon_cmd($vmid, "blockdev-change-medium", id => "$cloudinit_ds", filename => "$path");
5391         }
5392     }
5393 }
5394
5395 # called in locked context by incoming migration
5396 sub vm_migrate_get_nbd_disks {
5397     my ($storecfg, $conf, $replicated_volumes) = @_;
5398
5399     my $local_volumes = {};
5400     PVE::QemuConfig->foreach_volume($conf, sub {
5401         my ($ds, $drive) = @_;
5402
5403         return if drive_is_cdrom($drive);
5404         return if $ds eq 'tpmstate0';
5405
5406         my $volid = $drive->{file};
5407
5408         return if !$volid;
5409
5410         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5411
5412         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5413         return if $scfg->{shared};
5414
5415         # replicated disks re-use existing state via bitmap
5416         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5417         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5418     });
5419     return $local_volumes;
5420 }
5421
5422 # called in locked context by incoming migration
5423 sub vm_migrate_alloc_nbd_disks {
5424     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5425
5426     my $nbd = {};
5427     foreach my $opt (sort keys %$source_volumes) {
5428         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5429
5430         if ($use_existing) {
5431             $nbd->{$opt}->{drivestr} = print_drive($drive);
5432             $nbd->{$opt}->{volid} = $volid;
5433             $nbd->{$opt}->{replicated} = 1;
5434             next;
5435         }
5436
5437         # storage mapping + volname = regular migration
5438         # storage mapping + format = remote migration
5439         # order of precedence, filtered by whether storage supports it:
5440         # 1. explicit requested format
5441         # 2. format of current volume
5442         # 3. default format of storage
5443         if (!$storagemap->{identity}) {
5444             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5445             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5446             if (!$format || !grep { $format eq $_ } @$validFormats) {
5447                 if ($volname) {
5448                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5449                     my $fileFormat = qemu_img_format($scfg, $volname);
5450                     $format = $fileFormat
5451                         if grep { $fileFormat eq $_ } @$validFormats;
5452                 }
5453                 $format //= $defFormat;
5454             }
5455         } else {
5456             # can't happen for remote migration, so $volname is always defined
5457             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5458             $format = qemu_img_format($scfg, $volname);
5459         }
5460
5461         my $size = $drive->{size} / 1024;
5462         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5463         my $newdrive = $drive;
5464         $newdrive->{format} = $format;
5465         $newdrive->{file} = $newvolid;
5466         my $drivestr = print_drive($newdrive);
5467         $nbd->{$opt}->{drivestr} = $drivestr;
5468         $nbd->{$opt}->{volid} = $newvolid;
5469     }
5470
5471     return $nbd;
5472 }
5473
5474 # see vm_start_nolock for parameters, additionally:
5475 # migrate_opts:
5476 #   storagemap = parsed storage map for allocating NBD disks
5477 sub vm_start {
5478     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5479
5480     return PVE::QemuConfig->lock_config($vmid, sub {
5481         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5482
5483         die "you can't start a vm if it's a template\n"
5484             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5485
5486         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5487         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5488
5489         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5490
5491         if ($has_backup_lock && $running) {
5492             # a backup is currently running, attempt to start the guest in the
5493             # existing QEMU instance
5494             return vm_resume($vmid);
5495         }
5496
5497         PVE::QemuConfig->check_lock($conf)
5498             if !($params->{skiplock} || $has_suspended_lock);
5499
5500         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5501
5502         die "VM $vmid already running\n" if $running;
5503
5504         if (my $storagemap = $migrate_opts->{storagemap}) {
5505             my $replicated = $migrate_opts->{replicated_volumes};
5506             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5507             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5508
5509             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5510                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5511             }
5512         }
5513
5514         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5515     });
5516 }
5517
5518
5519 # params:
5520 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5521 #   skiplock => 0/1, skip checking for config lock
5522 #   skiptemplate => 0/1, skip checking whether VM is template
5523 #   forcemachine => to force Qemu machine (rollback/migration)
5524 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5525 #   timeout => in seconds
5526 #   paused => start VM in paused state (backup)
5527 #   resume => resume from hibernation
5528 #   pbs-backing => {
5529 #      sata0 => {
5530 #         repository
5531 #         snapshot
5532 #         keyfile
5533 #         archive
5534 #      },
5535 #      virtio2 => ...
5536 #   }
5537 # migrate_opts:
5538 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5539 #   migratedfrom => source node
5540 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5541 #   network => CIDR of migration network
5542 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5543 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5544 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5545 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5546 #       contained in config
5547 sub vm_start_nolock {
5548     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5549
5550     my $statefile = $params->{statefile};
5551     my $resume = $params->{resume};
5552
5553     my $migratedfrom = $migrate_opts->{migratedfrom};
5554     my $migration_type = $migrate_opts->{type};
5555
5556     my $res = {};
5557
5558     # clean up leftover reboot request files
5559     eval { clear_reboot_request($vmid); };
5560     warn $@ if $@;
5561
5562     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5563         vmconfig_apply_pending($vmid, $conf, $storecfg);
5564         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5565     }
5566
5567     # don't regenerate the ISO if the VM is started as part of a live migration
5568     # this way we can reuse the old ISO with the correct config
5569     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5570
5571     # override offline migrated volumes, conf is out of date still
5572     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5573         for my $key (sort keys $offline_volumes->%*) {
5574             my $parsed = parse_drive($key, $conf->{$key});
5575             $parsed->{file} = $offline_volumes->{$key};
5576             $conf->{$key} = print_drive($parsed);
5577         }
5578     }
5579
5580     my $defaults = load_defaults();
5581
5582     # set environment variable useful inside network script
5583     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5584
5585     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5586
5587     my $forcemachine = $params->{forcemachine};
5588     my $forcecpu = $params->{forcecpu};
5589     if ($resume) {
5590         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5591         $forcemachine = $conf->{runningmachine};
5592         $forcecpu = $conf->{runningcpu};
5593         print "Resuming suspended VM\n";
5594     }
5595
5596     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5597         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5598
5599     my $migration_ip;
5600     my $get_migration_ip = sub {
5601         my ($nodename) = @_;
5602
5603         return $migration_ip if defined($migration_ip);
5604
5605         my $cidr = $migrate_opts->{network};
5606
5607         if (!defined($cidr)) {
5608             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5609             $cidr = $dc_conf->{migration}->{network};
5610         }
5611
5612         if (defined($cidr)) {
5613             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5614
5615             die "could not get IP: no address configured on local " .
5616                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5617
5618             die "could not get IP: multiple addresses configured on local " .
5619                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5620
5621             $migration_ip = @$ips[0];
5622         }
5623
5624         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5625             if !defined($migration_ip);
5626
5627         return $migration_ip;
5628     };
5629
5630     my $migrate_uri;
5631     if ($statefile) {
5632         if ($statefile eq 'tcp') {
5633             my $localip = "localhost";
5634             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5635             my $nodename = nodename();
5636
5637             if (!defined($migration_type)) {
5638                 if (defined($datacenterconf->{migration}->{type})) {
5639                     $migration_type = $datacenterconf->{migration}->{type};
5640                 } else {
5641                     $migration_type = 'secure';
5642                 }
5643             }
5644
5645             if ($migration_type eq 'insecure') {
5646                 $localip = $get_migration_ip->($nodename);
5647                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5648             }
5649
5650             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5651             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5652             $migrate_uri = "tcp:${localip}:${migrate_port}";
5653             push @$cmd, '-incoming', $migrate_uri;
5654             push @$cmd, '-S';
5655
5656         } elsif ($statefile eq 'unix') {
5657             # should be default for secure migrations as a ssh TCP forward
5658             # tunnel is not deterministic reliable ready and fails regurarly
5659             # to set up in time, so use UNIX socket forwards
5660             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5661             unlink $socket_addr;
5662
5663             $migrate_uri = "unix:$socket_addr";
5664
5665             push @$cmd, '-incoming', $migrate_uri;
5666             push @$cmd, '-S';
5667
5668         } elsif (-e $statefile) {
5669             push @$cmd, '-loadstate', $statefile;
5670         } else {
5671             my $statepath = PVE::Storage::path($storecfg, $statefile);
5672             push @$vollist, $statefile;
5673             push @$cmd, '-loadstate', $statepath;
5674         }
5675     } elsif ($params->{paused}) {
5676         push @$cmd, '-S';
5677     }
5678
5679     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5680
5681     my $pci_devices = {}; # host pci devices
5682     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5683         my $dev = $conf->{"hostpci$i"} or next;
5684         $pci_devices->{$i} = parse_hostpci($dev);
5685     }
5686
5687     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5688     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5689
5690     # map to a flat list of pci ids
5691     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5692
5693     # reserve all PCI IDs before actually doing anything with them
5694     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5695
5696     eval {
5697         my $uuid;
5698         for my $id (sort keys %$pci_devices) {
5699             my $d = $pci_devices->{$id};
5700             for my $dev ($d->{pciid}->@*) {
5701                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5702
5703                 # nvidia grid needs the uuid of the mdev as qemu parameter
5704                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5705                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5706                 }
5707             }
5708         }
5709         push @$cmd, '-uuid', $uuid if defined($uuid);
5710     };
5711     if (my $err = $@) {
5712         eval { cleanup_pci_devices($vmid, $conf) };
5713         warn $@ if $@;
5714         die $err;
5715     }
5716
5717     PVE::Storage::activate_volumes($storecfg, $vollist);
5718
5719     eval {
5720         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5721     };
5722     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5723     # timeout should be more than enough here...
5724     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5725
5726     my $cpuunits = PVE::CGroup::clamp_cpu_shares($conf->{cpuunits});
5727
5728     my %run_params = (
5729         timeout => $statefile ? undef : $start_timeout,
5730         umask => 0077,
5731         noerr => 1,
5732     );
5733
5734     # when migrating, prefix QEMU output so other side can pick up any
5735     # errors that might occur and show the user
5736     if ($migratedfrom) {
5737         $run_params{quiet} = 1;
5738         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5739     }
5740
5741     my %systemd_properties = (
5742         Slice => 'qemu.slice',
5743         KillMode => 'process',
5744         SendSIGKILL => 0,
5745         TimeoutStopUSec => ULONG_MAX, # infinity
5746     );
5747
5748     if (PVE::CGroup::cgroup_mode() == 2) {
5749         $systemd_properties{CPUWeight} = $cpuunits;
5750     } else {
5751         $systemd_properties{CPUShares} = $cpuunits;
5752     }
5753
5754     if (my $cpulimit = $conf->{cpulimit}) {
5755         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5756     }
5757     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5758
5759     my $run_qemu = sub {
5760         PVE::Tools::run_fork sub {
5761             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5762
5763             my $tpmpid;
5764             if (my $tpm = $conf->{tpmstate0}) {
5765                 # start the TPM emulator so QEMU can connect on start
5766                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5767             }
5768
5769             my $exitcode = run_command($cmd, %run_params);
5770             if ($exitcode) {
5771                 if ($tpmpid) {
5772                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5773                     kill 'TERM', $tpmpid;
5774                 }
5775                 die "QEMU exited with code $exitcode\n";
5776             }
5777         };
5778     };
5779
5780     if ($conf->{hugepages}) {
5781
5782         my $code = sub {
5783             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5784             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5785
5786             PVE::QemuServer::Memory::hugepages_mount();
5787             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5788
5789             eval { $run_qemu->() };
5790             if (my $err = $@) {
5791                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5792                     if !$conf->{keephugepages};
5793                 die $err;
5794             }
5795
5796             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5797                 if !$conf->{keephugepages};
5798         };
5799         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5800
5801     } else {
5802         eval { $run_qemu->() };
5803     }
5804
5805     if (my $err = $@) {
5806         # deactivate volumes if start fails
5807         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5808         warn $@ if $@;
5809         eval { cleanup_pci_devices($vmid, $conf) };
5810         warn $@ if $@;
5811
5812         die "start failed: $err";
5813     }
5814
5815     # re-reserve all PCI IDs now that we can know the actual VM PID
5816     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5817     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5818     warn $@ if $@;
5819
5820     print "migration listens on $migrate_uri\n" if $migrate_uri;
5821     $res->{migrate_uri} = $migrate_uri;
5822
5823     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5824         eval { mon_cmd($vmid, "cont"); };
5825         warn $@ if $@;
5826     }
5827
5828     #start nbd server for storage migration
5829     if (my $nbd = $migrate_opts->{nbd}) {
5830         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5831
5832         my $migrate_storage_uri;
5833         # nbd_protocol_version > 0 for unix socket support
5834         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5835             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5836             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5837             $migrate_storage_uri = "nbd:unix:$socket_path";
5838         } else {
5839             my $nodename = nodename();
5840             my $localip = $get_migration_ip->($nodename);
5841             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5842             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5843
5844             mon_cmd($vmid, "nbd-server-start", addr => {
5845                 type => 'inet',
5846                 data => {
5847                     host => "${localip}",
5848                     port => "${storage_migrate_port}",
5849                 },
5850             });
5851             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5852             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5853         }
5854
5855         $res->{migrate_storage_uri} = $migrate_storage_uri;
5856
5857         foreach my $opt (sort keys %$nbd) {
5858             my $drivestr = $nbd->{$opt}->{drivestr};
5859             my $volid = $nbd->{$opt}->{volid};
5860             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5861             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5862             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5863             print "re-using replicated volume: $opt - $volid\n"
5864                 if $nbd->{$opt}->{replicated};
5865
5866             $res->{drives}->{$opt} = $nbd->{$opt};
5867             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5868         }
5869     }
5870
5871     if ($migratedfrom) {
5872         eval {
5873             set_migration_caps($vmid);
5874         };
5875         warn $@ if $@;
5876
5877         if ($spice_port) {
5878             print "spice listens on port $spice_port\n";
5879             $res->{spice_port} = $spice_port;
5880             if ($migrate_opts->{spice_ticket}) {
5881                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5882                     $migrate_opts->{spice_ticket});
5883                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5884             }
5885         }
5886
5887     } else {
5888         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5889             if !$statefile && $conf->{balloon};
5890
5891         foreach my $opt (keys %$conf) {
5892             next if $opt !~  m/^net\d+$/;
5893             my $nicconf = parse_net($conf->{$opt});
5894             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5895         }
5896         add_nets_bridge_fdb($conf, $vmid);
5897     }
5898
5899     mon_cmd($vmid, 'qom-set',
5900                 path => "machine/peripheral/balloon0",
5901                 property => "guest-stats-polling-interval",
5902                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5903
5904     if ($resume) {
5905         print "Resumed VM, removing state\n";
5906         if (my $vmstate = $conf->{vmstate}) {
5907             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5908             PVE::Storage::vdisk_free($storecfg, $vmstate);
5909         }
5910         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5911         PVE::QemuConfig->write_config($vmid, $conf);
5912     }
5913
5914     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5915
5916     return $res;
5917 }
5918
5919 sub vm_commandline {
5920     my ($storecfg, $vmid, $snapname) = @_;
5921
5922     my $conf = PVE::QemuConfig->load_config($vmid);
5923
5924     my ($forcemachine, $forcecpu);
5925     if ($snapname) {
5926         my $snapshot = $conf->{snapshots}->{$snapname};
5927         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5928
5929         # check for machine or CPU overrides in snapshot
5930         $forcemachine = $snapshot->{runningmachine};
5931         $forcecpu = $snapshot->{runningcpu};
5932
5933         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5934
5935         $conf = $snapshot;
5936     }
5937
5938     my $defaults = load_defaults();
5939
5940     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5941
5942     return PVE::Tools::cmd2string($cmd);
5943 }
5944
5945 sub vm_reset {
5946     my ($vmid, $skiplock) = @_;
5947
5948     PVE::QemuConfig->lock_config($vmid, sub {
5949
5950         my $conf = PVE::QemuConfig->load_config($vmid);
5951
5952         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5953
5954         mon_cmd($vmid, "system_reset");
5955     });
5956 }
5957
5958 sub get_vm_volumes {
5959     my ($conf) = @_;
5960
5961     my $vollist = [];
5962     foreach_volid($conf, sub {
5963         my ($volid, $attr) = @_;
5964
5965         return if $volid =~ m|^/|;
5966
5967         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5968         return if !$sid;
5969
5970         push @$vollist, $volid;
5971     });
5972
5973     return $vollist;
5974 }
5975
5976 sub cleanup_pci_devices {
5977     my ($vmid, $conf) = @_;
5978
5979     foreach my $key (keys %$conf) {
5980         next if $key !~ m/^hostpci(\d+)$/;
5981         my $hostpciindex = $1;
5982         my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5983         my $d = parse_hostpci($conf->{$key});
5984         if ($d->{mdev}) {
5985             # NOTE: avoid PVE::SysFSTools::pci_cleanup_mdev_device as it requires PCI ID and we
5986             # don't want to break ABI just for this two liner
5987             my $dev_sysfs_dir = "/sys/bus/mdev/devices/$uuid";
5988             PVE::SysFSTools::file_write("$dev_sysfs_dir/remove", "1") if -e $dev_sysfs_dir;
5989         }
5990     }
5991     PVE::QemuServer::PCI::remove_pci_reservation($vmid);
5992 }
5993
5994 sub vm_stop_cleanup {
5995     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5996
5997     eval {
5998
5999         if (!$keepActive) {
6000             my $vollist = get_vm_volumes($conf);
6001             PVE::Storage::deactivate_volumes($storecfg, $vollist);
6002
6003             if (my $tpmdrive = $conf->{tpmstate0}) {
6004                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
6005                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
6006                 if ($storeid) {
6007                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
6008                 }
6009             }
6010         }
6011
6012         foreach my $ext (qw(mon qmp pid vnc qga)) {
6013             unlink "/var/run/qemu-server/${vmid}.$ext";
6014         }
6015
6016         if ($conf->{ivshmem}) {
6017             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
6018             # just delete it for now, VMs which have this already open do not
6019             # are affected, but new VMs will get a separated one. If this
6020             # becomes an issue we either add some sort of ref-counting or just
6021             # add a "don't delete on stop" flag to the ivshmem format.
6022             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
6023         }
6024
6025         cleanup_pci_devices($vmid, $conf);
6026
6027         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
6028     };
6029     warn $@ if $@; # avoid errors - just warn
6030 }
6031
6032 # call only in locked context
6033 sub _do_vm_stop {
6034     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
6035
6036     my $pid = check_running($vmid, $nocheck);
6037     return if !$pid;
6038
6039     my $conf;
6040     if (!$nocheck) {
6041         $conf = PVE::QemuConfig->load_config($vmid);
6042         PVE::QemuConfig->check_lock($conf) if !$skiplock;
6043         if (!defined($timeout) && $shutdown && $conf->{startup}) {
6044             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
6045             $timeout = $opts->{down} if $opts->{down};
6046         }
6047         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
6048     }
6049
6050     eval {
6051         if ($shutdown) {
6052             if (defined($conf) && get_qga_key($conf, 'enabled')) {
6053                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
6054             } else {
6055                 mon_cmd($vmid, "system_powerdown");
6056             }
6057         } else {
6058             mon_cmd($vmid, "quit");
6059         }
6060     };
6061     my $err = $@;
6062
6063     if (!$err) {
6064         $timeout = 60 if !defined($timeout);
6065
6066         my $count = 0;
6067         while (($count < $timeout) && check_running($vmid, $nocheck)) {
6068             $count++;
6069             sleep 1;
6070         }
6071
6072         if ($count >= $timeout) {
6073             if ($force) {
6074                 warn "VM still running - terminating now with SIGTERM\n";
6075                 kill 15, $pid;
6076             } else {
6077                 die "VM quit/powerdown failed - got timeout\n";
6078             }
6079         } else {
6080             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6081             return;
6082         }
6083     } else {
6084        if (!check_running($vmid, $nocheck)) {
6085             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
6086             return;
6087        }
6088        if ($force) {
6089             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
6090             kill 15, $pid;
6091         } else {
6092             die "VM quit/powerdown failed\n";
6093         }
6094     }
6095
6096     # wait again
6097     $timeout = 10;
6098
6099     my $count = 0;
6100     while (($count < $timeout) && check_running($vmid, $nocheck)) {
6101         $count++;
6102         sleep 1;
6103     }
6104
6105     if ($count >= $timeout) {
6106         warn "VM still running - terminating now with SIGKILL\n";
6107         kill 9, $pid;
6108         sleep 1;
6109     }
6110
6111     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6112 }
6113
6114 # Note: use $nocheck to skip tests if VM configuration file exists.
6115 # We need that when migration VMs to other nodes (files already moved)
6116 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6117 sub vm_stop {
6118     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6119
6120     $force = 1 if !defined($force) && !$shutdown;
6121
6122     if ($migratedfrom){
6123         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6124         kill 15, $pid if $pid;
6125         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6126         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6127         return;
6128     }
6129
6130     PVE::QemuConfig->lock_config($vmid, sub {
6131         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6132    });
6133 }
6134
6135 sub vm_reboot {
6136     my ($vmid, $timeout) = @_;
6137
6138     PVE::QemuConfig->lock_config($vmid, sub {
6139         eval {
6140
6141             # only reboot if running, as qmeventd starts it again on a stop event
6142             return if !check_running($vmid);
6143
6144             create_reboot_request($vmid);
6145
6146             my $storecfg = PVE::Storage::config();
6147             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6148
6149         };
6150         if (my $err = $@) {
6151             # avoid that the next normal shutdown will be confused for a reboot
6152             clear_reboot_request($vmid);
6153             die $err;
6154         }
6155    });
6156 }
6157
6158 # note: if using the statestorage parameter, the caller has to check privileges
6159 sub vm_suspend {
6160     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6161
6162     my $conf;
6163     my $path;
6164     my $storecfg;
6165     my $vmstate;
6166
6167     PVE::QemuConfig->lock_config($vmid, sub {
6168
6169         $conf = PVE::QemuConfig->load_config($vmid);
6170
6171         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6172         PVE::QemuConfig->check_lock($conf)
6173             if !($skiplock || $is_backing_up);
6174
6175         die "cannot suspend to disk during backup\n"
6176             if $is_backing_up && $includestate;
6177
6178         if ($includestate) {
6179             $conf->{lock} = 'suspending';
6180             my $date = strftime("%Y-%m-%d", localtime(time()));
6181             $storecfg = PVE::Storage::config();
6182             if (!$statestorage) {
6183                 $statestorage = find_vmstate_storage($conf, $storecfg);
6184                 # check permissions for the storage
6185                 my $rpcenv = PVE::RPCEnvironment::get();
6186                 if ($rpcenv->{type} ne 'cli') {
6187                     my $authuser = $rpcenv->get_user();
6188                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6189                 }
6190             }
6191
6192
6193             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6194                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6195             $path = PVE::Storage::path($storecfg, $vmstate);
6196             PVE::QemuConfig->write_config($vmid, $conf);
6197         } else {
6198             mon_cmd($vmid, "stop");
6199         }
6200     });
6201
6202     if ($includestate) {
6203         # save vm state
6204         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6205
6206         eval {
6207             set_migration_caps($vmid, 1);
6208             mon_cmd($vmid, "savevm-start", statefile => $path);
6209             for(;;) {
6210                 my $state = mon_cmd($vmid, "query-savevm");
6211                 if (!$state->{status}) {
6212                     die "savevm not active\n";
6213                 } elsif ($state->{status} eq 'active') {
6214                     sleep(1);
6215                     next;
6216                 } elsif ($state->{status} eq 'completed') {
6217                     print "State saved, quitting\n";
6218                     last;
6219                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6220                     die "query-savevm failed with error '$state->{error}'\n"
6221                 } else {
6222                     die "query-savevm returned status '$state->{status}'\n";
6223                 }
6224             }
6225         };
6226         my $err = $@;
6227
6228         PVE::QemuConfig->lock_config($vmid, sub {
6229             $conf = PVE::QemuConfig->load_config($vmid);
6230             if ($err) {
6231                 # cleanup, but leave suspending lock, to indicate something went wrong
6232                 eval {
6233                     mon_cmd($vmid, "savevm-end");
6234                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6235                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6236                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6237                     PVE::QemuConfig->write_config($vmid, $conf);
6238                 };
6239                 warn $@ if $@;
6240                 die $err;
6241             }
6242
6243             die "lock changed unexpectedly\n"
6244                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6245
6246             mon_cmd($vmid, "quit");
6247             $conf->{lock} = 'suspended';
6248             PVE::QemuConfig->write_config($vmid, $conf);
6249         });
6250     }
6251 }
6252
6253 sub vm_resume {
6254     my ($vmid, $skiplock, $nocheck) = @_;
6255
6256     PVE::QemuConfig->lock_config($vmid, sub {
6257         my $res = mon_cmd($vmid, 'query-status');
6258         my $resume_cmd = 'cont';
6259         my $reset = 0;
6260         my $conf = PVE::QemuConfig->load_config($vmid);
6261
6262         if ($res->{status}) {
6263             return if $res->{status} eq 'running'; # job done, go home
6264             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6265             $reset = 1 if $res->{status} eq 'shutdown';
6266         }
6267
6268         if (!$nocheck) {
6269
6270             PVE::QemuConfig->check_lock($conf)
6271                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6272         }
6273
6274         if ($reset) {
6275             # required if a VM shuts down during a backup and we get a resume
6276             # request before the backup finishes for example
6277             mon_cmd($vmid, "system_reset");
6278         }
6279
6280         add_nets_bridge_fdb($conf, $vmid) if $resume_cmd eq 'cont';
6281
6282         mon_cmd($vmid, $resume_cmd);
6283     });
6284 }
6285
6286 sub vm_sendkey {
6287     my ($vmid, $skiplock, $key) = @_;
6288
6289     PVE::QemuConfig->lock_config($vmid, sub {
6290
6291         my $conf = PVE::QemuConfig->load_config($vmid);
6292
6293         # there is no qmp command, so we use the human monitor command
6294         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6295         die $res if $res ne '';
6296     });
6297 }
6298
6299 # vzdump restore implementaion
6300
6301 sub tar_archive_read_firstfile {
6302     my $archive = shift;
6303
6304     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6305
6306     # try to detect archive type first
6307     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6308         die "unable to open file '$archive'\n";
6309     my $firstfile = <$fh>;
6310     kill 15, $pid;
6311     close $fh;
6312
6313     die "ERROR: archive contaions no data\n" if !$firstfile;
6314     chomp $firstfile;
6315
6316     return $firstfile;
6317 }
6318
6319 sub tar_restore_cleanup {
6320     my ($storecfg, $statfile) = @_;
6321
6322     print STDERR "starting cleanup\n";
6323
6324     if (my $fd = IO::File->new($statfile, "r")) {
6325         while (defined(my $line = <$fd>)) {
6326             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6327                 my $volid = $2;
6328                 eval {
6329                     if ($volid =~ m|^/|) {
6330                         unlink $volid || die 'unlink failed\n';
6331                     } else {
6332                         PVE::Storage::vdisk_free($storecfg, $volid);
6333                     }
6334                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6335                 };
6336                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6337             } else {
6338                 print STDERR "unable to parse line in statfile - $line";
6339             }
6340         }
6341         $fd->close();
6342     }
6343 }
6344
6345 sub restore_file_archive {
6346     my ($archive, $vmid, $user, $opts) = @_;
6347
6348     return restore_vma_archive($archive, $vmid, $user, $opts)
6349         if $archive eq '-';
6350
6351     my $info = PVE::Storage::archive_info($archive);
6352     my $format = $opts->{format} // $info->{format};
6353     my $comp = $info->{compression};
6354
6355     # try to detect archive format
6356     if ($format eq 'tar') {
6357         return restore_tar_archive($archive, $vmid, $user, $opts);
6358     } else {
6359         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6360     }
6361 }
6362
6363 # hepler to remove disks that will not be used after restore
6364 my $restore_cleanup_oldconf = sub {
6365     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6366
6367     my $kept_disks = {};
6368
6369     PVE::QemuConfig->foreach_volume($oldconf, sub {
6370         my ($ds, $drive) = @_;
6371
6372         return if drive_is_cdrom($drive, 1);
6373
6374         my $volid = $drive->{file};
6375         return if !$volid || $volid =~ m|^/|;
6376
6377         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6378         return if !$path || !$owner || ($owner != $vmid);
6379
6380         # Note: only delete disk we want to restore
6381         # other volumes will become unused
6382         if ($virtdev_hash->{$ds}) {
6383             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6384             if (my $err = $@) {
6385                 warn $err;
6386             }
6387         } else {
6388             $kept_disks->{$volid} = 1;
6389         }
6390     });
6391
6392     # after the restore we have no snapshots anymore
6393     for my $snapname (keys $oldconf->{snapshots}->%*) {
6394         my $snap = $oldconf->{snapshots}->{$snapname};
6395         if ($snap->{vmstate}) {
6396             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6397             if (my $err = $@) {
6398                 warn $err;
6399             }
6400         }
6401
6402         for my $volid (keys $kept_disks->%*) {
6403             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6404             warn $@ if $@;
6405         }
6406     }
6407 };
6408
6409 # Helper to parse vzdump backup device hints
6410 #
6411 # $rpcenv: Environment, used to ckeck storage permissions
6412 # $user: User ID, to check storage permissions
6413 # $storecfg: Storage configuration
6414 # $fh: the file handle for reading the configuration
6415 # $devinfo: should contain device sizes for all backu-up'ed devices
6416 # $options: backup options (pool, default storage)
6417 #
6418 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6419 my $parse_backup_hints = sub {
6420     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6421
6422     my $check_storage = sub { # assert if an image can be allocate
6423         my ($storeid, $scfg) = @_;
6424         die "Content type 'images' is not available on storage '$storeid'\n"
6425             if !$scfg->{content}->{images};
6426         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6427             if $user ne 'root@pam';
6428     };
6429
6430     my $virtdev_hash = {};
6431     while (defined(my $line = <$fh>)) {
6432         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6433             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6434             die "archive does not contain data for drive '$virtdev'\n"
6435                 if !$devinfo->{$devname};
6436
6437             if (defined($options->{storage})) {
6438                 $storeid = $options->{storage} || 'local';
6439             } elsif (!$storeid) {
6440                 $storeid = 'local';
6441             }
6442             $format = 'raw' if !$format;
6443             $devinfo->{$devname}->{devname} = $devname;
6444             $devinfo->{$devname}->{virtdev} = $virtdev;
6445             $devinfo->{$devname}->{format} = $format;
6446             $devinfo->{$devname}->{storeid} = $storeid;
6447
6448             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6449             $check_storage->($storeid, $scfg); # permission and content type check
6450
6451             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6452         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6453             my $virtdev = $1;
6454             my $drive = parse_drive($virtdev, $2);
6455
6456             if (drive_is_cloudinit($drive)) {
6457                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6458                 $storeid = $options->{storage} if defined ($options->{storage});
6459                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6460                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6461
6462                 $check_storage->($storeid, $scfg); # permission and content type check
6463
6464                 $virtdev_hash->{$virtdev} = {
6465                     format => $format,
6466                     storeid => $storeid,
6467                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6468                     is_cloudinit => 1,
6469                 };
6470             }
6471         }
6472     }
6473
6474     return $virtdev_hash;
6475 };
6476
6477 # Helper to allocate and activate all volumes required for a restore
6478 #
6479 # $storecfg: Storage configuration
6480 # $virtdev_hash: as returned by parse_backup_hints()
6481 #
6482 # Returns: { $virtdev => $volid }
6483 my $restore_allocate_devices = sub {
6484     my ($storecfg, $virtdev_hash, $vmid) = @_;
6485
6486     my $map = {};
6487     foreach my $virtdev (sort keys %$virtdev_hash) {
6488         my $d = $virtdev_hash->{$virtdev};
6489         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6490         my $storeid = $d->{storeid};
6491         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6492
6493         # test if requested format is supported
6494         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6495         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6496         $d->{format} = $defFormat if !$supported;
6497
6498         my $name;
6499         if ($d->{is_cloudinit}) {
6500             $name = "vm-$vmid-cloudinit";
6501             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6502             if ($scfg->{path}) {
6503                 $name .= ".$d->{format}";
6504             }
6505         }
6506
6507         my $volid = PVE::Storage::vdisk_alloc(
6508             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6509
6510         print STDERR "new volume ID is '$volid'\n";
6511         $d->{volid} = $volid;
6512
6513         PVE::Storage::activate_volumes($storecfg, [$volid]);
6514
6515         $map->{$virtdev} = $volid;
6516     }
6517
6518     return $map;
6519 };
6520
6521 sub restore_update_config_line {
6522     my ($cookie, $map, $line, $unique) = @_;
6523
6524     return '' if $line =~ m/^\#qmdump\#/;
6525     return '' if $line =~ m/^\#vzdump\#/;
6526     return '' if $line =~ m/^lock:/;
6527     return '' if $line =~ m/^unused\d+:/;
6528     return '' if $line =~ m/^parent:/;
6529
6530     my $res = '';
6531
6532     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6533     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6534         # try to convert old 1.X settings
6535         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6536         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6537             my ($model, $macaddr) = split(/\=/, $devconfig);
6538             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6539             my $net = {
6540                 model => $model,
6541                 bridge => "vmbr$ind",
6542                 macaddr => $macaddr,
6543             };
6544             my $netstr = print_net($net);
6545
6546             $res .= "net$cookie->{netcount}: $netstr\n";
6547             $cookie->{netcount}++;
6548         }
6549     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6550         my ($id, $netstr) = ($1, $2);
6551         my $net = parse_net($netstr);
6552         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6553         $netstr = print_net($net);
6554         $res .= "$id: $netstr\n";
6555     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6556         my $virtdev = $1;
6557         my $value = $3;
6558         my $di = parse_drive($virtdev, $value);
6559         if (defined($di->{backup}) && !$di->{backup}) {
6560             $res .= "#$line";
6561         } elsif ($map->{$virtdev}) {
6562             delete $di->{format}; # format can change on restore
6563             $di->{file} = $map->{$virtdev};
6564             $value = print_drive($di);
6565             $res .= "$virtdev: $value\n";
6566         } else {
6567             $res .= $line;
6568         }
6569     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6570         my $vmgenid = $1;
6571         if ($vmgenid ne '0') {
6572             # always generate a new vmgenid if there was a valid one setup
6573             $vmgenid = generate_uuid();
6574         }
6575         $res .= "vmgenid: $vmgenid\n";
6576     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6577         my ($uuid, $uuid_str);
6578         UUID::generate($uuid);
6579         UUID::unparse($uuid, $uuid_str);
6580         my $smbios1 = parse_smbios1($2);
6581         $smbios1->{uuid} = $uuid_str;
6582         $res .= $1.print_smbios1($smbios1)."\n";
6583     } else {
6584         $res .= $line;
6585     }
6586
6587     return $res;
6588 }
6589
6590 my $restore_deactivate_volumes = sub {
6591     my ($storecfg, $virtdev_hash) = @_;
6592
6593     my $vollist = [];
6594     for my $dev (values $virtdev_hash->%*) {
6595         push $vollist->@*, $dev->{volid} if $dev->{volid};
6596     }
6597
6598     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6599     print STDERR $@ if $@;
6600 };
6601
6602 my $restore_destroy_volumes = sub {
6603     my ($storecfg, $virtdev_hash) = @_;
6604
6605     for my $dev (values $virtdev_hash->%*) {
6606         my $volid = $dev->{volid} or next;
6607         eval {
6608             PVE::Storage::vdisk_free($storecfg, $volid);
6609             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6610         };
6611         print STDERR "unable to cleanup '$volid' - $@" if $@;
6612     }
6613 };
6614
6615 my $restore_merge_config = sub {
6616     my ($filename, $backup_conf_raw, $override_conf) = @_;
6617
6618     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6619     for my $key (keys $override_conf->%*) {
6620         $backup_conf->{$key} = $override_conf->{$key};
6621     }
6622
6623     return $backup_conf;
6624 };
6625
6626 sub scan_volids {
6627     my ($cfg, $vmid) = @_;
6628
6629     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6630
6631     my $volid_hash = {};
6632     foreach my $storeid (keys %$info) {
6633         foreach my $item (@{$info->{$storeid}}) {
6634             next if !($item->{volid} && $item->{size});
6635             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6636             $volid_hash->{$item->{volid}} = $item;
6637         }
6638     }
6639
6640     return $volid_hash;
6641 }
6642
6643 sub update_disk_config {
6644     my ($vmid, $conf, $volid_hash) = @_;
6645
6646     my $changes;
6647     my $prefix = "VM $vmid";
6648
6649     # used and unused disks
6650     my $referenced = {};
6651
6652     # Note: it is allowed to define multiple storages with same path (alias), so
6653     # we need to check both 'volid' and real 'path' (two different volid can point
6654     # to the same path).
6655
6656     my $referencedpath = {};
6657
6658     # update size info
6659     PVE::QemuConfig->foreach_volume($conf, sub {
6660         my ($opt, $drive) = @_;
6661
6662         my $volid = $drive->{file};
6663         return if !$volid;
6664         my $volume = $volid_hash->{$volid};
6665
6666         # mark volid as "in-use" for next step
6667         $referenced->{$volid} = 1;
6668         if ($volume && (my $path = $volume->{path})) {
6669             $referencedpath->{$path} = 1;
6670         }
6671
6672         return if drive_is_cdrom($drive);
6673         return if !$volume;
6674
6675         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6676         if (defined($updated)) {
6677             $changes = 1;
6678             $conf->{$opt} = print_drive($updated);
6679             print "$prefix ($opt): $msg\n";
6680         }
6681     });
6682
6683     # remove 'unusedX' entry if volume is used
6684     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6685         my ($opt, $drive) = @_;
6686
6687         my $volid = $drive->{file};
6688         return if !$volid;
6689
6690         my $path;
6691         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6692         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6693             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6694             $changes = 1;
6695             delete $conf->{$opt};
6696         }
6697
6698         $referenced->{$volid} = 1;
6699         $referencedpath->{$path} = 1 if $path;
6700     });
6701
6702     foreach my $volid (sort keys %$volid_hash) {
6703         next if $volid =~ m/vm-$vmid-state-/;
6704         next if $referenced->{$volid};
6705         my $path = $volid_hash->{$volid}->{path};
6706         next if !$path; # just to be sure
6707         next if $referencedpath->{$path};
6708         $changes = 1;
6709         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6710         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6711         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6712     }
6713
6714     return $changes;
6715 }
6716
6717 sub rescan {
6718     my ($vmid, $nolock, $dryrun) = @_;
6719
6720     my $cfg = PVE::Storage::config();
6721
6722     print "rescan volumes...\n";
6723     my $volid_hash = scan_volids($cfg, $vmid);
6724
6725     my $updatefn =  sub {
6726         my ($vmid) = @_;
6727
6728         my $conf = PVE::QemuConfig->load_config($vmid);
6729
6730         PVE::QemuConfig->check_lock($conf);
6731
6732         my $vm_volids = {};
6733         foreach my $volid (keys %$volid_hash) {
6734             my $info = $volid_hash->{$volid};
6735             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6736         }
6737
6738         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6739
6740         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6741     };
6742
6743     if (defined($vmid)) {
6744         if ($nolock) {
6745             &$updatefn($vmid);
6746         } else {
6747             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6748         }
6749     } else {
6750         my $vmlist = config_list();
6751         foreach my $vmid (keys %$vmlist) {
6752             if ($nolock) {
6753                 &$updatefn($vmid);
6754             } else {
6755                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6756             }
6757         }
6758     }
6759 }
6760
6761 sub restore_proxmox_backup_archive {
6762     my ($archive, $vmid, $user, $options) = @_;
6763
6764     my $storecfg = PVE::Storage::config();
6765
6766     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6767     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6768
6769     my $fingerprint = $scfg->{fingerprint};
6770     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6771
6772     my $repo = PVE::PBSClient::get_repository($scfg);
6773     my $namespace = $scfg->{namespace};
6774
6775     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6776     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6777     local $ENV{PBS_PASSWORD} = $password;
6778     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6779
6780     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6781         PVE::Storage::parse_volname($storecfg, $archive);
6782
6783     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6784
6785     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6786
6787     my $tmpdir = "/var/tmp/vzdumptmp$$";
6788     rmtree $tmpdir;
6789     mkpath $tmpdir;
6790
6791     my $conffile = PVE::QemuConfig->config_file($vmid);
6792      # disable interrupts (always do cleanups)
6793     local $SIG{INT} =
6794         local $SIG{TERM} =
6795         local $SIG{QUIT} =
6796         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6797
6798     # Note: $oldconf is undef if VM does not exists
6799     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6800     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6801     my $new_conf_raw = '';
6802
6803     my $rpcenv = PVE::RPCEnvironment::get();
6804     my $devinfo = {}; # info about drives included in backup
6805     my $virtdev_hash = {}; # info about allocated drives
6806
6807     eval {
6808         # enable interrupts
6809         local $SIG{INT} =
6810             local $SIG{TERM} =
6811             local $SIG{QUIT} =
6812             local $SIG{HUP} =
6813             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6814
6815         my $cfgfn = "$tmpdir/qemu-server.conf";
6816         my $firewall_config_fn = "$tmpdir/fw.conf";
6817         my $index_fn = "$tmpdir/index.json";
6818
6819         my $cmd = "restore";
6820
6821         my $param = [$pbs_backup_name, "index.json", $index_fn];
6822         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6823         my $index = PVE::Tools::file_get_contents($index_fn);
6824         $index = decode_json($index);
6825
6826         foreach my $info (@{$index->{files}}) {
6827             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6828                 my $devname = $1;
6829                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6830                     $devinfo->{$devname}->{size} = $1;
6831                 } else {
6832                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6833                 }
6834             }
6835         }
6836
6837         my $is_qemu_server_backup = scalar(
6838             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6839         );
6840         if (!$is_qemu_server_backup) {
6841             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6842         }
6843         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6844
6845         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6846         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6847
6848         if ($has_firewall_config) {
6849             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6850             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6851
6852             my $pve_firewall_dir = '/etc/pve/firewall';
6853             mkdir $pve_firewall_dir; # make sure the dir exists
6854             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6855         }
6856
6857         my $fh = IO::File->new($cfgfn, "r") ||
6858             die "unable to read qemu-server.conf - $!\n";
6859
6860         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6861
6862         # fixme: rate limit?
6863
6864         # create empty/temp config
6865         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6866
6867         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6868
6869         # allocate volumes
6870         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6871
6872         foreach my $virtdev (sort keys %$virtdev_hash) {
6873             my $d = $virtdev_hash->{$virtdev};
6874             next if $d->{is_cloudinit}; # no need to restore cloudinit
6875
6876             # this fails if storage is unavailable
6877             my $volid = $d->{volid};
6878             my $path = PVE::Storage::path($storecfg, $volid);
6879
6880             # for live-restore we only want to preload the efidisk and TPM state
6881             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6882
6883             my @ns_arg;
6884             if (defined(my $ns = $scfg->{namespace})) {
6885                 @ns_arg = ('--ns', $ns);
6886             }
6887
6888             my $pbs_restore_cmd = [
6889                 '/usr/bin/pbs-restore',
6890                 '--repository', $repo,
6891                 @ns_arg,
6892                 $pbs_backup_name,
6893                 "$d->{devname}.img.fidx",
6894                 $path,
6895                 '--verbose',
6896                 ];
6897
6898             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6899             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6900
6901             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6902                 push @$pbs_restore_cmd, '--skip-zero';
6903             }
6904
6905             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6906             print "restore proxmox backup image: $dbg_cmdstring\n";
6907             run_command($pbs_restore_cmd);
6908         }
6909
6910         $fh->seek(0, 0) || die "seek failed - $!\n";
6911
6912         my $cookie = { netcount => 0 };
6913         while (defined(my $line = <$fh>)) {
6914             $new_conf_raw .= restore_update_config_line(
6915                 $cookie,
6916                 $map,
6917                 $line,
6918                 $options->{unique},
6919             );
6920         }
6921
6922         $fh->close();
6923     };
6924     my $err = $@;
6925
6926     if ($err || !$options->{live}) {
6927         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6928     }
6929
6930     rmtree $tmpdir;
6931
6932     if ($err) {
6933         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6934         die $err;
6935     }
6936
6937     if ($options->{live}) {
6938         # keep lock during live-restore
6939         $new_conf_raw .= "\nlock: create";
6940     }
6941
6942     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6943     PVE::QemuConfig->write_config($vmid, $new_conf);
6944
6945     eval { rescan($vmid, 1); };
6946     warn $@ if $@;
6947
6948     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6949
6950     if ($options->{live}) {
6951         # enable interrupts
6952         local $SIG{INT} =
6953             local $SIG{TERM} =
6954             local $SIG{QUIT} =
6955             local $SIG{HUP} =
6956             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6957
6958         my $conf = PVE::QemuConfig->load_config($vmid);
6959         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6960
6961         # these special drives are already restored before start
6962         delete $devinfo->{'drive-efidisk0'};
6963         delete $devinfo->{'drive-tpmstate0-backup'};
6964
6965         my $pbs_opts = {
6966             repo => $repo,
6967             keyfile => $keyfile,
6968             snapshot => $pbs_backup_name,
6969             namespace => $namespace,
6970         };
6971         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6972
6973         PVE::QemuConfig->remove_lock($vmid, "create");
6974     }
6975 }
6976
6977 sub pbs_live_restore {
6978     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6979
6980     print "starting VM for live-restore\n";
6981     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6982
6983     my $pbs_backing = {};
6984     for my $ds (keys %$restored_disks) {
6985         $ds =~ m/^drive-(.*)$/;
6986         my $confname = $1;
6987         $pbs_backing->{$confname} = {
6988             repository => $opts->{repo},
6989             snapshot => $opts->{snapshot},
6990             archive => "$ds.img.fidx",
6991         };
6992         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6993         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6994
6995         my $drive = parse_drive($confname, $conf->{$confname});
6996         print "restoring '$ds' to '$drive->{file}'\n";
6997     }
6998
6999     my $drives_streamed = 0;
7000     eval {
7001         # make sure HA doesn't interrupt our restore by stopping the VM
7002         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
7003             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
7004         }
7005
7006         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
7007         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
7008         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
7009
7010         my $qmeventd_fd = register_qmeventd_handle($vmid);
7011
7012         # begin streaming, i.e. data copy from PBS to target disk for every vol,
7013         # this will effectively collapse the backing image chain consisting of
7014         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
7015         # removes itself once all backing images vanish with 'auto-remove=on')
7016         my $jobs = {};
7017         for my $ds (sort keys %$restored_disks) {
7018             my $job_id = "restore-$ds";
7019             mon_cmd($vmid, 'block-stream',
7020                 'job-id' => $job_id,
7021                 device => "$ds",
7022             );
7023             $jobs->{$job_id} = {};
7024         }
7025
7026         mon_cmd($vmid, 'cont');
7027         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
7028
7029         print "restore-drive jobs finished successfully, removing all tracking block devices"
7030             ." to disconnect from Proxmox Backup Server\n";
7031
7032         for my $ds (sort keys %$restored_disks) {
7033             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
7034         }
7035
7036         close($qmeventd_fd);
7037     };
7038
7039     my $err = $@;
7040
7041     if ($err) {
7042         warn "An error occurred during live-restore: $err\n";
7043         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
7044         die "live-restore failed\n";
7045     }
7046 }
7047
7048 sub restore_vma_archive {
7049     my ($archive, $vmid, $user, $opts, $comp) = @_;
7050
7051     my $readfrom = $archive;
7052
7053     my $cfg = PVE::Storage::config();
7054     my $commands = [];
7055     my $bwlimit = $opts->{bwlimit};
7056
7057     my $dbg_cmdstring = '';
7058     my $add_pipe = sub {
7059         my ($cmd) = @_;
7060         push @$commands, $cmd;
7061         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
7062         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
7063         $readfrom = '-';
7064     };
7065
7066     my $input = undef;
7067     if ($archive eq '-') {
7068         $input = '<&STDIN';
7069     } else {
7070         # If we use a backup from a PVE defined storage we also consider that
7071         # storage's rate limit:
7072         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
7073         if (defined($volid)) {
7074             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
7075             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
7076             if ($readlimit) {
7077                 print STDERR "applying read rate limit: $readlimit\n";
7078                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
7079                 $add_pipe->($cstream);
7080             }
7081         }
7082     }
7083
7084     if ($comp) {
7085         my $info = PVE::Storage::decompressor_info('vma', $comp);
7086         my $cmd = $info->{decompressor};
7087         push @$cmd, $readfrom;
7088         $add_pipe->($cmd);
7089     }
7090
7091     my $tmpdir = "/var/tmp/vzdumptmp$$";
7092     rmtree $tmpdir;
7093
7094     # disable interrupts (always do cleanups)
7095     local $SIG{INT} =
7096         local $SIG{TERM} =
7097         local $SIG{QUIT} =
7098         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
7099
7100     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
7101     POSIX::mkfifo($mapfifo, 0600);
7102     my $fifofh;
7103     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
7104
7105     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
7106
7107     my $oldtimeout;
7108     my $timeout = 5;
7109
7110     my $devinfo = {}; # info about drives included in backup
7111     my $virtdev_hash = {}; # info about allocated drives
7112
7113     my $rpcenv = PVE::RPCEnvironment::get();
7114
7115     my $conffile = PVE::QemuConfig->config_file($vmid);
7116
7117     # Note: $oldconf is undef if VM does not exist
7118     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7119     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7120     my $new_conf_raw = '';
7121
7122     my %storage_limits;
7123
7124     my $print_devmap = sub {
7125         my $cfgfn = "$tmpdir/qemu-server.conf";
7126
7127         # we can read the config - that is already extracted
7128         my $fh = IO::File->new($cfgfn, "r") ||
7129             die "unable to read qemu-server.conf - $!\n";
7130
7131         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7132         if (-f $fwcfgfn) {
7133             my $pve_firewall_dir = '/etc/pve/firewall';
7134             mkdir $pve_firewall_dir; # make sure the dir exists
7135             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7136         }
7137
7138         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7139
7140         foreach my $info (values %{$virtdev_hash}) {
7141             my $storeid = $info->{storeid};
7142             next if defined($storage_limits{$storeid});
7143
7144             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7145             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7146             $storage_limits{$storeid} = $limit * 1024;
7147         }
7148
7149         foreach my $devname (keys %$devinfo) {
7150             die "found no device mapping information for device '$devname'\n"
7151                 if !$devinfo->{$devname}->{virtdev};
7152         }
7153
7154         # create empty/temp config
7155         if ($oldconf) {
7156             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7157             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7158         }
7159
7160         # allocate volumes
7161         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7162
7163         # print restore information to $fifofh
7164         foreach my $virtdev (sort keys %$virtdev_hash) {
7165             my $d = $virtdev_hash->{$virtdev};
7166             next if $d->{is_cloudinit}; # no need to restore cloudinit
7167
7168             my $storeid = $d->{storeid};
7169             my $volid = $d->{volid};
7170
7171             my $map_opts = '';
7172             if (my $limit = $storage_limits{$storeid}) {
7173                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7174             }
7175
7176             my $write_zeros = 1;
7177             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7178                 $write_zeros = 0;
7179             }
7180
7181             my $path = PVE::Storage::path($cfg, $volid);
7182
7183             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7184
7185             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7186         }
7187
7188         $fh->seek(0, 0) || die "seek failed - $!\n";
7189
7190         my $cookie = { netcount => 0 };
7191         while (defined(my $line = <$fh>)) {
7192             $new_conf_raw .= restore_update_config_line(
7193                 $cookie,
7194                 $map,
7195                 $line,
7196                 $opts->{unique},
7197             );
7198         }
7199
7200         $fh->close();
7201     };
7202
7203     eval {
7204         # enable interrupts
7205         local $SIG{INT} =
7206             local $SIG{TERM} =
7207             local $SIG{QUIT} =
7208             local $SIG{HUP} =
7209             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7210         local $SIG{ALRM} = sub { die "got timeout\n"; };
7211
7212         $oldtimeout = alarm($timeout);
7213
7214         my $parser = sub {
7215             my $line = shift;
7216
7217             print "$line\n";
7218
7219             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7220                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7221                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7222             } elsif ($line =~ m/^CTIME: /) {
7223                 # we correctly received the vma config, so we can disable
7224                 # the timeout now for disk allocation (set to 10 minutes, so
7225                 # that we always timeout if something goes wrong)
7226                 alarm(600);
7227                 &$print_devmap();
7228                 print $fifofh "done\n";
7229                 my $tmp = $oldtimeout || 0;
7230                 $oldtimeout = undef;
7231                 alarm($tmp);
7232                 close($fifofh);
7233                 $fifofh = undef;
7234             }
7235         };
7236
7237         print "restore vma archive: $dbg_cmdstring\n";
7238         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7239     };
7240     my $err = $@;
7241
7242     alarm($oldtimeout) if $oldtimeout;
7243
7244     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7245
7246     close($fifofh) if $fifofh;
7247     unlink $mapfifo;
7248     rmtree $tmpdir;
7249
7250     if ($err) {
7251         $restore_destroy_volumes->($cfg, $virtdev_hash);
7252         die $err;
7253     }
7254
7255     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7256     PVE::QemuConfig->write_config($vmid, $new_conf);
7257
7258     eval { rescan($vmid, 1); };
7259     warn $@ if $@;
7260
7261     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7262 }
7263
7264 sub restore_tar_archive {
7265     my ($archive, $vmid, $user, $opts) = @_;
7266
7267     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7268         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7269         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7270     }
7271
7272     if ($archive ne '-') {
7273         my $firstfile = tar_archive_read_firstfile($archive);
7274         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7275             if $firstfile ne 'qemu-server.conf';
7276     }
7277
7278     my $storecfg = PVE::Storage::config();
7279
7280     # avoid zombie disks when restoring over an existing VM -> cleanup first
7281     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7282     # skiplock=1 because qmrestore has set the 'create' lock itself already
7283     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7284     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7285
7286     my $tocmd = "/usr/lib/qemu-server/qmextract";
7287
7288     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7289     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7290     $tocmd .= ' --prealloc' if $opts->{prealloc};
7291     $tocmd .= ' --info' if $opts->{info};
7292
7293     # tar option "xf" does not autodetect compression when read from STDIN,
7294     # so we pipe to zcat
7295     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7296         PVE::Tools::shellquote("--to-command=$tocmd");
7297
7298     my $tmpdir = "/var/tmp/vzdumptmp$$";
7299     mkpath $tmpdir;
7300
7301     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7302     local $ENV{VZDUMP_VMID} = $vmid;
7303     local $ENV{VZDUMP_USER} = $user;
7304
7305     my $conffile = PVE::QemuConfig->config_file($vmid);
7306     my $new_conf_raw = '';
7307
7308     # disable interrupts (always do cleanups)
7309     local $SIG{INT} =
7310         local $SIG{TERM} =
7311         local $SIG{QUIT} =
7312         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7313
7314     eval {
7315         # enable interrupts
7316         local $SIG{INT} =
7317             local $SIG{TERM} =
7318             local $SIG{QUIT} =
7319             local $SIG{HUP} =
7320             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7321
7322         if ($archive eq '-') {
7323             print "extracting archive from STDIN\n";
7324             run_command($cmd, input => "<&STDIN");
7325         } else {
7326             print "extracting archive '$archive'\n";
7327             run_command($cmd);
7328         }
7329
7330         return if $opts->{info};
7331
7332         # read new mapping
7333         my $map = {};
7334         my $statfile = "$tmpdir/qmrestore.stat";
7335         if (my $fd = IO::File->new($statfile, "r")) {
7336             while (defined (my $line = <$fd>)) {
7337                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7338                     $map->{$1} = $2 if $1;
7339                 } else {
7340                     print STDERR "unable to parse line in statfile - $line\n";
7341                 }
7342             }
7343             $fd->close();
7344         }
7345
7346         my $confsrc = "$tmpdir/qemu-server.conf";
7347
7348         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7349
7350         my $cookie = { netcount => 0 };
7351         while (defined (my $line = <$srcfd>)) {
7352             $new_conf_raw .= restore_update_config_line(
7353                 $cookie,
7354                 $map,
7355                 $line,
7356                 $opts->{unique},
7357             );
7358         }
7359
7360         $srcfd->close();
7361     };
7362     if (my $err = $@) {
7363         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7364         die $err;
7365     }
7366
7367     rmtree $tmpdir;
7368
7369     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7370
7371     PVE::Cluster::cfs_update(); # make sure we read new file
7372
7373     eval { rescan($vmid, 1); };
7374     warn $@ if $@;
7375 };
7376
7377 sub foreach_storage_used_by_vm {
7378     my ($conf, $func) = @_;
7379
7380     my $sidhash = {};
7381
7382     PVE::QemuConfig->foreach_volume($conf, sub {
7383         my ($ds, $drive) = @_;
7384         return if drive_is_cdrom($drive);
7385
7386         my $volid = $drive->{file};
7387
7388         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7389         $sidhash->{$sid} = $sid if $sid;
7390     });
7391
7392     foreach my $sid (sort keys %$sidhash) {
7393         &$func($sid);
7394     }
7395 }
7396
7397 my $qemu_snap_storage = {
7398     rbd => 1,
7399 };
7400 sub do_snapshots_with_qemu {
7401     my ($storecfg, $volid, $deviceid) = @_;
7402
7403     return if $deviceid =~ m/tpmstate0/;
7404
7405     my $storage_name = PVE::Storage::parse_volume_id($volid);
7406     my $scfg = $storecfg->{ids}->{$storage_name};
7407     die "could not find storage '$storage_name'\n" if !defined($scfg);
7408
7409     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7410         return 1;
7411     }
7412
7413     if ($volid =~ m/\.(qcow2|qed)$/){
7414         return 1;
7415     }
7416
7417     return;
7418 }
7419
7420 sub qga_check_running {
7421     my ($vmid, $nowarn) = @_;
7422
7423     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7424     if ($@) {
7425         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7426         return 0;
7427     }
7428     return 1;
7429 }
7430
7431 sub template_create {
7432     my ($vmid, $conf, $disk) = @_;
7433
7434     my $storecfg = PVE::Storage::config();
7435
7436     PVE::QemuConfig->foreach_volume($conf, sub {
7437         my ($ds, $drive) = @_;
7438
7439         return if drive_is_cdrom($drive);
7440         return if $disk && $ds ne $disk;
7441
7442         my $volid = $drive->{file};
7443         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7444
7445         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7446         $drive->{file} = $voliddst;
7447         $conf->{$ds} = print_drive($drive);
7448         PVE::QemuConfig->write_config($vmid, $conf);
7449     });
7450 }
7451
7452 sub convert_iscsi_path {
7453     my ($path) = @_;
7454
7455     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7456         my $portal = $1;
7457         my $target = $2;
7458         my $lun = $3;
7459
7460         my $initiator_name = get_initiator_name();
7461
7462         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7463                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7464     }
7465
7466     die "cannot convert iscsi path '$path', unkown format\n";
7467 }
7468
7469 sub qemu_img_convert {
7470     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7471
7472     my $storecfg = PVE::Storage::config();
7473     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7474     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7475
7476     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7477
7478     my $cachemode;
7479     my $src_path;
7480     my $src_is_iscsi = 0;
7481     my $src_format;
7482
7483     if ($src_storeid) {
7484         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7485         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7486         $src_format = qemu_img_format($src_scfg, $src_volname);
7487         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7488         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7489         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7490     } elsif (-f $src_volid || -b $src_volid) {
7491         $src_path = $src_volid;
7492         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7493             $src_format = $1;
7494         }
7495     }
7496
7497     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7498
7499     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7500     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7501     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7502     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7503
7504     my $cmd = [];
7505     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7506     push @$cmd, '-l', "snapshot.name=$snapname"
7507         if $snapname && $src_format && $src_format eq "qcow2";
7508     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7509     push @$cmd, '-T', $cachemode if defined($cachemode);
7510
7511     if ($src_is_iscsi) {
7512         push @$cmd, '--image-opts';
7513         $src_path = convert_iscsi_path($src_path);
7514     } elsif ($src_format) {
7515         push @$cmd, '-f', $src_format;
7516     }
7517
7518     if ($dst_is_iscsi) {
7519         push @$cmd, '--target-image-opts';
7520         $dst_path = convert_iscsi_path($dst_path);
7521     } else {
7522         push @$cmd, '-O', $dst_format;
7523     }
7524
7525     push @$cmd, $src_path;
7526
7527     if (!$dst_is_iscsi && $is_zero_initialized) {
7528         push @$cmd, "zeroinit:$dst_path";
7529     } else {
7530         push @$cmd, $dst_path;
7531     }
7532
7533     my $parser = sub {
7534         my $line = shift;
7535         if($line =~ m/\((\S+)\/100\%\)/){
7536             my $percent = $1;
7537             my $transferred = int($size * $percent / 100);
7538             my $total_h = render_bytes($size, 1);
7539             my $transferred_h = render_bytes($transferred, 1);
7540
7541             print "transferred $transferred_h of $total_h ($percent%)\n";
7542         }
7543
7544     };
7545
7546     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7547     my $err = $@;
7548     die "copy failed: $err" if $err;
7549 }
7550
7551 sub qemu_img_format {
7552     my ($scfg, $volname) = @_;
7553
7554     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7555         return $1;
7556     } else {
7557         return "raw";
7558     }
7559 }
7560
7561 sub qemu_drive_mirror {
7562     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7563
7564     $jobs = {} if !$jobs;
7565
7566     my $qemu_target;
7567     my $format;
7568     $jobs->{"drive-$drive"} = {};
7569
7570     if ($dst_volid =~ /^nbd:/) {
7571         $qemu_target = $dst_volid;
7572         $format = "nbd";
7573     } else {
7574         my $storecfg = PVE::Storage::config();
7575         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7576
7577         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7578
7579         $format = qemu_img_format($dst_scfg, $dst_volname);
7580
7581         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7582
7583         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7584     }
7585
7586     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7587     $opts->{format} = $format if $format;
7588
7589     if (defined($src_bitmap)) {
7590         $opts->{sync} = 'incremental';
7591         $opts->{bitmap} = $src_bitmap;
7592         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7593     }
7594
7595     if (defined($bwlimit)) {
7596         $opts->{speed} = $bwlimit * 1024;
7597         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7598     } else {
7599         print "drive mirror is starting for drive-$drive\n";
7600     }
7601
7602     # if a job already runs for this device we get an error, catch it for cleanup
7603     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7604     if (my $err = $@) {
7605         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7606         warn "$@\n" if $@;
7607         die "mirroring error: $err\n";
7608     }
7609
7610     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7611 }
7612
7613 # $completion can be either
7614 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7615 # 'cancel': wait until all jobs are ready, block-job-cancel them
7616 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7617 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7618 sub qemu_drive_mirror_monitor {
7619     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7620
7621     $completion //= 'complete';
7622     $op //= "mirror";
7623
7624     eval {
7625         my $err_complete = 0;
7626
7627         my $starttime = time ();
7628         while (1) {
7629             die "block job ('$op') timed out\n" if $err_complete > 300;
7630
7631             my $stats = mon_cmd($vmid, "query-block-jobs");
7632             my $ctime = time();
7633
7634             my $running_jobs = {};
7635             for my $stat (@$stats) {
7636                 next if $stat->{type} ne $op;
7637                 $running_jobs->{$stat->{device}} = $stat;
7638             }
7639
7640             my $readycounter = 0;
7641
7642             for my $job_id (sort keys %$jobs) {
7643                 my $job = $running_jobs->{$job_id};
7644
7645                 my $vanished = !defined($job);
7646                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7647                 if($complete || ($vanished && $completion eq 'auto')) {
7648                     print "$job_id: $op-job finished\n";
7649                     delete $jobs->{$job_id};
7650                     next;
7651                 }
7652
7653                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7654
7655                 my $busy = $job->{busy};
7656                 my $ready = $job->{ready};
7657                 if (my $total = $job->{len}) {
7658                     my $transferred = $job->{offset} || 0;
7659                     my $remaining = $total - $transferred;
7660                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7661
7662                     my $duration = $ctime - $starttime;
7663                     my $total_h = render_bytes($total, 1);
7664                     my $transferred_h = render_bytes($transferred, 1);
7665
7666                     my $status = sprintf(
7667                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7668                         render_duration($duration),
7669                     );
7670
7671                     if ($ready) {
7672                         if ($busy) {
7673                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7674                         } else {
7675                             $status .= ", ready";
7676                         }
7677                     }
7678                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7679                     $jobs->{$job_id}->{ready} = $ready;
7680                 }
7681
7682                 $readycounter++ if $job->{ready};
7683             }
7684
7685             last if scalar(keys %$jobs) == 0;
7686
7687             if ($readycounter == scalar(keys %$jobs)) {
7688                 print "all '$op' jobs are ready\n";
7689
7690                 # do the complete later (or has already been done)
7691                 last if $completion eq 'skip' || $completion eq 'auto';
7692
7693                 if ($vmiddst && $vmiddst != $vmid) {
7694                     my $agent_running = $qga && qga_check_running($vmid);
7695                     if ($agent_running) {
7696                         print "freeze filesystem\n";
7697                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7698                         warn $@ if $@;
7699                     } else {
7700                         print "suspend vm\n";
7701                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7702                         warn $@ if $@;
7703                     }
7704
7705                     # if we clone a disk for a new target vm, we don't switch the disk
7706                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7707
7708                     if ($agent_running) {
7709                         print "unfreeze filesystem\n";
7710                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7711                         warn $@ if $@;
7712                     } else {
7713                         print "resume vm\n";
7714                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7715                         warn $@ if $@;
7716                     }
7717
7718                     last;
7719                 } else {
7720
7721                     for my $job_id (sort keys %$jobs) {
7722                         # try to switch the disk if source and destination are on the same guest
7723                         print "$job_id: Completing block job_id...\n";
7724
7725                         my $op;
7726                         if ($completion eq 'complete') {
7727                             $op = 'block-job-complete';
7728                         } elsif ($completion eq 'cancel') {
7729                             $op = 'block-job-cancel';
7730                         } else {
7731                             die "invalid completion value: $completion\n";
7732                         }
7733                         eval { mon_cmd($vmid, $op, device => $job_id) };
7734                         if ($@ =~ m/cannot be completed/) {
7735                             print "$job_id: block job cannot be completed, trying again.\n";
7736                             $err_complete++;
7737                         }else {
7738                             print "$job_id: Completed successfully.\n";
7739                             $jobs->{$job_id}->{complete} = 1;
7740                         }
7741                     }
7742                 }
7743             }
7744             sleep 1;
7745         }
7746     };
7747     my $err = $@;
7748
7749     if ($err) {
7750         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7751         die "block job ($op) error: $err";
7752     }
7753 }
7754
7755 sub qemu_blockjobs_cancel {
7756     my ($vmid, $jobs) = @_;
7757
7758     foreach my $job (keys %$jobs) {
7759         print "$job: Cancelling block job\n";
7760         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7761         $jobs->{$job}->{cancel} = 1;
7762     }
7763
7764     while (1) {
7765         my $stats = mon_cmd($vmid, "query-block-jobs");
7766
7767         my $running_jobs = {};
7768         foreach my $stat (@$stats) {
7769             $running_jobs->{$stat->{device}} = $stat;
7770         }
7771
7772         foreach my $job (keys %$jobs) {
7773
7774             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7775                 print "$job: Done.\n";
7776                 delete $jobs->{$job};
7777             }
7778         }
7779
7780         last if scalar(keys %$jobs) == 0;
7781
7782         sleep 1;
7783     }
7784 }
7785
7786 sub clone_disk {
7787     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7788
7789     my ($vmid, $running) = $source->@{qw(vmid running)};
7790     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7791
7792     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7793     my ($storage, $format) = $dest->@{qw(storage format)};
7794
7795     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7796
7797     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7798         die "cloning from/to EFI disk requires EFI disk\n"
7799             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7800         die "cloning from/to TPM state requires TPM state\n"
7801             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7802
7803         # This would lead to two device nodes in QEMU pointing to the same backing image!
7804         die "cannot change drive name when cloning disk from/to the same VM\n"
7805             if $use_drive_mirror && $vmid == $newvmid;
7806     }
7807
7808     die "cannot move TPM state while VM is running\n"
7809         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7810
7811     my $newvolid;
7812
7813     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7814     print "$src_drivename " if $src_drivename;
7815     print "($drive->{file})\n";
7816
7817     if (!$full) {
7818         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7819         push @$newvollist, $newvolid;
7820     } else {
7821
7822         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7823         $storeid = $storage if $storage;
7824
7825         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7826
7827         my $name = undef;
7828         my $size = undef;
7829         if (drive_is_cloudinit($drive)) {
7830             $name = "vm-$newvmid-cloudinit";
7831             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7832             if ($scfg->{path}) {
7833                 $name .= ".$dst_format";
7834             }
7835             $snapname = undef;
7836             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7837         } elsif ($dst_drivename eq 'efidisk0') {
7838             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7839         } elsif ($dst_drivename eq 'tpmstate0') {
7840             $dst_format = 'raw';
7841             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7842         } else {
7843             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7844         }
7845         $newvolid = PVE::Storage::vdisk_alloc(
7846             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7847         );
7848         push @$newvollist, $newvolid;
7849
7850         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7851
7852         if (drive_is_cloudinit($drive)) {
7853             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7854             # if this is the case, we have to complete any block-jobs still there from
7855             # previous drive-mirrors
7856             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7857                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7858             }
7859             goto no_data_clone;
7860         }
7861
7862         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7863         if ($use_drive_mirror) {
7864             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7865                 $completion, $qga, $bwlimit);
7866         } else {
7867             # TODO: handle bwlimits
7868             if ($dst_drivename eq 'efidisk0') {
7869                 # the relevant data on the efidisk may be smaller than the source
7870                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7871                 # that is given by the OVMF_VARS.fd
7872                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7873                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7874
7875                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7876
7877                 # better for Ceph if block size is not too small, see bug #3324
7878                 my $bs = 1024*1024;
7879
7880                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7881
7882                 if ($src_format eq 'qcow2' && $snapname) {
7883                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7884                         if !min_version(kvm_user_version(), 6, 2);
7885                     push $cmd->@*, '-l', $snapname;
7886                 }
7887                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7888                 run_command($cmd);
7889             } else {
7890                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7891             }
7892         }
7893     }
7894
7895 no_data_clone:
7896     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7897
7898     my $disk = dclone($drive);
7899     delete $disk->{format};
7900     $disk->{file} = $newvolid;
7901     $disk->{size} = $size if defined($size);
7902
7903     return $disk;
7904 }
7905
7906 sub get_running_qemu_version {
7907     my ($vmid) = @_;
7908     my $res = mon_cmd($vmid, "query-version");
7909     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7910 }
7911
7912 sub qemu_use_old_bios_files {
7913     my ($machine_type) = @_;
7914
7915     return if !$machine_type;
7916
7917     my $use_old_bios_files = undef;
7918
7919     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7920         $machine_type = $1;
7921         $use_old_bios_files = 1;
7922     } else {
7923         my $version = extract_version($machine_type, kvm_user_version());
7924         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7925         # load new efi bios files on migration. So this hack is required to allow
7926         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7927         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7928         $use_old_bios_files = !min_version($version, 2, 4);
7929     }
7930
7931     return ($use_old_bios_files, $machine_type);
7932 }
7933
7934 sub get_efivars_size {
7935     my ($conf, $efidisk) = @_;
7936
7937     my $arch = get_vm_arch($conf);
7938     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7939     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7940     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7941     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7942     return -s $ovmf_vars;
7943 }
7944
7945 sub update_efidisk_size {
7946     my ($conf) = @_;
7947
7948     return if !defined($conf->{efidisk0});
7949
7950     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7951     $disk->{size} = get_efivars_size($conf);
7952     $conf->{efidisk0} = print_drive($disk);
7953
7954     return;
7955 }
7956
7957 sub update_tpmstate_size {
7958     my ($conf) = @_;
7959
7960     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7961     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7962     $conf->{tpmstate0} = print_drive($disk);
7963 }
7964
7965 sub create_efidisk($$$$$$$) {
7966     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7967
7968     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7969     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7970
7971     my $vars_size_b = -s $ovmf_vars;
7972     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7973     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7974     PVE::Storage::activate_volumes($storecfg, [$volid]);
7975
7976     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7977     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7978
7979     return ($volid, $size/1024);
7980 }
7981
7982 sub vm_iothreads_list {
7983     my ($vmid) = @_;
7984
7985     my $res = mon_cmd($vmid, 'query-iothreads');
7986
7987     my $iothreads = {};
7988     foreach my $iothread (@$res) {
7989         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7990     }
7991
7992     return $iothreads;
7993 }
7994
7995 sub scsihw_infos {
7996     my ($conf, $drive) = @_;
7997
7998     my $maxdev = 0;
7999
8000     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
8001         $maxdev = 7;
8002     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
8003         $maxdev = 1;
8004     } else {
8005         $maxdev = 256;
8006     }
8007
8008     my $controller = int($drive->{index} / $maxdev);
8009     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
8010         ? "virtioscsi"
8011         : "scsihw";
8012
8013     return ($maxdev, $controller, $controller_prefix);
8014 }
8015
8016 sub resolve_dst_disk_format {
8017         my ($storecfg, $storeid, $src_volname, $format) = @_;
8018         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
8019
8020         if (!$format) {
8021             # if no target format is specified, use the source disk format as hint
8022             if ($src_volname) {
8023                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8024                 $format = qemu_img_format($scfg, $src_volname);
8025             } else {
8026                 return $defFormat;
8027             }
8028         }
8029
8030         # test if requested format is supported - else use default
8031         my $supported = grep { $_ eq $format } @$validFormats;
8032         $format = $defFormat if !$supported;
8033         return $format;
8034 }
8035
8036 # NOTE: if this logic changes, please update docs & possibly gui logic
8037 sub find_vmstate_storage {
8038     my ($conf, $storecfg) = @_;
8039
8040     # first, return storage from conf if set
8041     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
8042
8043     my ($target, $shared, $local);
8044
8045     foreach_storage_used_by_vm($conf, sub {
8046         my ($sid) = @_;
8047         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
8048         my $dst = $scfg->{shared} ? \$shared : \$local;
8049         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
8050     });
8051
8052     # second, use shared storage where VM has at least one disk
8053     # third, use local storage where VM has at least one disk
8054     # fall back to local storage
8055     $target = $shared // $local // 'local';
8056
8057     return $target;
8058 }
8059
8060 sub generate_uuid {
8061     my ($uuid, $uuid_str);
8062     UUID::generate($uuid);
8063     UUID::unparse($uuid, $uuid_str);
8064     return $uuid_str;
8065 }
8066
8067 sub generate_smbios1_uuid {
8068     return "uuid=".generate_uuid();
8069 }
8070
8071 sub nbd_stop {
8072     my ($vmid) = @_;
8073
8074     mon_cmd($vmid, 'nbd-server-stop');
8075 }
8076
8077 sub create_reboot_request {
8078     my ($vmid) = @_;
8079     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
8080         or die "failed to create reboot trigger file: $!\n";
8081     close($fh);
8082 }
8083
8084 sub clear_reboot_request {
8085     my ($vmid) = @_;
8086     my $path = "/run/qemu-server/$vmid.reboot";
8087     my $res = 0;
8088
8089     $res = unlink($path);
8090     die "could not remove reboot request for $vmid: $!"
8091         if !$res && $! != POSIX::ENOENT;
8092
8093     return $res;
8094 }
8095
8096 sub bootorder_from_legacy {
8097     my ($conf, $bootcfg) = @_;
8098
8099     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8100     my $bootindex_hash = {};
8101     my $i = 1;
8102     foreach my $o (split(//, $boot)) {
8103         $bootindex_hash->{$o} = $i*100;
8104         $i++;
8105     }
8106
8107     my $bootorder = {};
8108
8109     PVE::QemuConfig->foreach_volume($conf, sub {
8110         my ($ds, $drive) = @_;
8111
8112         if (drive_is_cdrom ($drive, 1)) {
8113             if ($bootindex_hash->{d}) {
8114                 $bootorder->{$ds} = $bootindex_hash->{d};
8115                 $bootindex_hash->{d} += 1;
8116             }
8117         } elsif ($bootindex_hash->{c}) {
8118             $bootorder->{$ds} = $bootindex_hash->{c}
8119                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8120             $bootindex_hash->{c} += 1;
8121         }
8122     });
8123
8124     if ($bootindex_hash->{n}) {
8125         for (my $i = 0; $i < $MAX_NETS; $i++) {
8126             my $netname = "net$i";
8127             next if !$conf->{$netname};
8128             $bootorder->{$netname} = $bootindex_hash->{n};
8129             $bootindex_hash->{n} += 1;
8130         }
8131     }
8132
8133     return $bootorder;
8134 }
8135
8136 # Generate default device list for 'boot: order=' property. Matches legacy
8137 # default boot order, but with explicit device names. This is important, since
8138 # the fallback for when neither 'order' nor the old format is specified relies
8139 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8140 sub get_default_bootdevices {
8141     my ($conf) = @_;
8142
8143     my @ret = ();
8144
8145     # harddisk
8146     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8147     push @ret, $first if $first;
8148
8149     # cdrom
8150     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8151     push @ret, $first if $first;
8152
8153     # network
8154     for (my $i = 0; $i < $MAX_NETS; $i++) {
8155         my $netname = "net$i";
8156         next if !$conf->{$netname};
8157         push @ret, $netname;
8158         last;
8159     }
8160
8161     return \@ret;
8162 }
8163
8164 sub device_bootorder {
8165     my ($conf) = @_;
8166
8167     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8168
8169     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8170
8171     my $bootorder = {};
8172     if (!defined($boot) || $boot->{legacy}) {
8173         $bootorder = bootorder_from_legacy($conf, $boot);
8174     } elsif ($boot->{order}) {
8175         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8176         for my $dev (PVE::Tools::split_list($boot->{order})) {
8177             $bootorder->{$dev} = $i++;
8178         }
8179     }
8180
8181     return $bootorder;
8182 }
8183
8184 sub register_qmeventd_handle {
8185     my ($vmid) = @_;
8186
8187     my $fh;
8188     my $peer = "/var/run/qmeventd.sock";
8189     my $count = 0;
8190
8191     for (;;) {
8192         $count++;
8193         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8194         last if $fh;
8195         if ($! != EINTR && $! != EAGAIN) {
8196             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8197         }
8198         if ($count > 4) {
8199             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8200               . "after $count retries\n";
8201         }
8202         usleep(25000);
8203     }
8204
8205     # send handshake to mark VM as backing up
8206     print $fh to_json({vzdump => {vmid => "$vmid"}});
8207
8208     # return handle to be closed later when inhibit is no longer required
8209     return $fh;
8210 }
8211
8212 # bash completion helper
8213
8214 sub complete_backup_archives {
8215     my ($cmdname, $pname, $cvalue) = @_;
8216
8217     my $cfg = PVE::Storage::config();
8218
8219     my $storeid;
8220
8221     if ($cvalue =~ m/^([^:]+):/) {
8222         $storeid = $1;
8223     }
8224
8225     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8226
8227     my $res = [];
8228     foreach my $id (keys %$data) {
8229         foreach my $item (@{$data->{$id}}) {
8230             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8231             push @$res, $item->{volid} if defined($item->{volid});
8232         }
8233     }
8234
8235     return $res;
8236 }
8237
8238 my $complete_vmid_full = sub {
8239     my ($running) = @_;
8240
8241     my $idlist = vmstatus();
8242
8243     my $res = [];
8244
8245     foreach my $id (keys %$idlist) {
8246         my $d = $idlist->{$id};
8247         if (defined($running)) {
8248             next if $d->{template};
8249             next if $running && $d->{status} ne 'running';
8250             next if !$running && $d->{status} eq 'running';
8251         }
8252         push @$res, $id;
8253
8254     }
8255     return $res;
8256 };
8257
8258 sub complete_vmid {
8259     return &$complete_vmid_full();
8260 }
8261
8262 sub complete_vmid_stopped {
8263     return &$complete_vmid_full(0);
8264 }
8265
8266 sub complete_vmid_running {
8267     return &$complete_vmid_full(1);
8268 }
8269
8270 sub complete_storage {
8271
8272     my $cfg = PVE::Storage::config();
8273     my $ids = $cfg->{ids};
8274
8275     my $res = [];
8276     foreach my $sid (keys %$ids) {
8277         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8278         next if !$ids->{$sid}->{content}->{images};
8279         push @$res, $sid;
8280     }
8281
8282     return $res;
8283 }
8284
8285 sub complete_migration_storage {
8286     my ($cmd, $param, $current_value, $all_args) = @_;
8287
8288     my $targetnode = @$all_args[1];
8289
8290     my $cfg = PVE::Storage::config();
8291     my $ids = $cfg->{ids};
8292
8293     my $res = [];
8294     foreach my $sid (keys %$ids) {
8295         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8296         next if !$ids->{$sid}->{content}->{images};
8297         push @$res, $sid;
8298     }
8299
8300     return $res;
8301 }
8302
8303 sub vm_is_paused {
8304     my ($vmid) = @_;
8305     my $qmpstatus = eval {
8306         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8307         mon_cmd($vmid, "query-status");
8308     };
8309     warn "$@\n" if $@;
8310     return $qmpstatus && $qmpstatus->{status} eq "paused";
8311 }
8312
8313 sub check_volume_storage_type {
8314     my ($storecfg, $vol) = @_;
8315
8316     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8317     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8318     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8319
8320     die "storage '$storeid' does not support content-type '$vtype'\n"
8321         if !$scfg->{content}->{$vtype};
8322
8323     return 1;
8324 }
8325
8326 sub add_nets_bridge_fdb {
8327     my ($conf, $vmid) = @_;
8328
8329     for my $opt (keys %$conf) {
8330         next if $opt !~ m/^net(\d+)$/;
8331         my $iface = "tap${vmid}i$1";
8332         # NOTE: expect setups with learning off to *not* use auto-random-generation of MAC on start
8333         my $net = parse_net($conf->{$opt}, 1) or next;
8334
8335         my $mac = $net->{macaddr};
8336         if (!$mac) {
8337             log_warn("MAC learning disabled, but vNIC '$iface' has no static MAC to add to forwarding DB!")
8338                 if !file_read_firstline("/sys/class/net/$iface/brport/learning");
8339             next;
8340         }
8341
8342         if ($have_sdn) {
8343             PVE::Network::SDN::Zones::add_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
8344         } else {
8345             PVE::Network::add_bridge_fdb($iface, $mac, $net->{firewall});
8346         }
8347     }
8348 }
8349
8350 sub del_nets_bridge_fdb {
8351     my ($conf, $vmid) = @_;
8352
8353     for my $opt (keys %$conf) {
8354         next if $opt !~ m/^net(\d+)$/;
8355         my $iface = "tap${vmid}i$1";
8356
8357         my $net = parse_net($conf->{$opt}) or next;
8358         my $mac = $net->{macaddr} or next;
8359
8360         if ($have_sdn) {
8361             PVE::Network::SDN::Zones::del_bridge_fdb($iface, $mac, $net->{bridge}, $net->{firewall});
8362         } else {
8363             PVE::Network::del_bridge_fdb($iface, $mac, $net->{firewall});
8364         }
8365     }
8366 }
8367
8368 1;