PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. This option is generally not required and"
 371         ." is often better handled from within the guest OS.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.",
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.",
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1040
1041     return verify_volume_id_or_absolute_path($volid, $noerr);
1042 }
1043
1044 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1045 sub verify_volume_id_or_absolute_path {
1046     my ($volid, $noerr) = @_;
1047
1048     return $volid if $volid =~ m|^/|;
1049
1050     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1051     if ($@) {
1052         return if $noerr;
1053         die $@;
1054     }
1055     return $volid;
1056 }
1057
1058 my $usb_fmt = {
1059     host => {
1060         default_key => 1,
1061         type => 'string', format => 'pve-qm-usb-device',
1062         format_description => 'HOSTUSBDEVICE|spice',
1063         description => <<EODESCR,
1064 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1065
1066  'bus-port(.port)*' (decimal numbers) or
1067  'vendor_id:product_id' (hexadeciaml numbers) or
1068  'spice'
1069
1070 You can use the 'lsusb -t' command to list existing usb devices.
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1073 machines - use with special care.
1074
1075 The value 'spice' can be used to add a usb redirection devices for spice.
1076 EODESCR
1077     },
1078     usb3 => {
1079         optional => 1,
1080         type => 'boolean',
1081         description => "Specifies whether if given host option is a USB3 device or port.",
1082         default => 0,
1083     },
1084 };
1085
1086 my $usbdesc = {
1087     optional => 1,
1088     type => 'string', format => $usb_fmt,
1089     description => "Configure an USB device (n is 0 to 4).",
1090 };
1091 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1092
1093 my $serialdesc = {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '(/dev/.+|socket)',
1097         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1098         verbose_description =>  <<EODESCR,
1099 Create a serial device inside the VM (n is 0 to 3), and pass through a
1100 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1101 host side (use 'qm terminal' to open a terminal connection).
1102
1103 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1104 use with special care.
1105
1106 CAUTION: Experimental! User reported problems with this option.
1107 EODESCR
1108 };
1109
1110 my $paralleldesc= {
1111         optional => 1,
1112         type => 'string',
1113         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1114         description =>  "Map host parallel devices (n is 0 to 2).",
1115         verbose_description =>  <<EODESCR,
1116 Map host parallel devices (n is 0 to 2).
1117
1118 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1119 machines - use with special care.
1120
1121 CAUTION: Experimental! User reported problems with this option.
1122 EODESCR
1123 };
1124
1125 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1126     $confdesc->{"parallel$i"} = $paralleldesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1130     $confdesc->{"serial$i"} = $serialdesc;
1131 }
1132
1133 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1134     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1135 }
1136
1137 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1138     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1139 }
1140
1141 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1142     $confdesc->{"usb$i"} = $usbdesc;
1143 }
1144
1145 my $boot_fmt = {
1146     legacy => {
1147         optional => 1,
1148         default_key => 1,
1149         type => 'string',
1150         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1151                      . " Deprecated, use 'order=' instead.",
1152         pattern => '[acdn]{1,4}',
1153         format_description => "[acdn]{1,4}",
1154
1155         # note: this is also the fallback if boot: is not given at all
1156         default => 'cdn',
1157     },
1158     order => {
1159         optional => 1,
1160         type => 'string',
1161         format => 'pve-qm-bootdev-list',
1162         format_description => "device[;device...]",
1163         description => <<EODESC,
1164 The guest will attempt to boot from devices in the order they appear here.
1165
1166 Disks, optical drives and passed-through storage USB devices will be directly
1167 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1168 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1169
1170 Note that only devices in this list will be marked as bootable and thus loaded
1171 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1172 (e.g. software-raid), you need to specify all of them here.
1173
1174 Overrides the deprecated 'legacy=[acdn]*' value when given.
1175 EODESC
1176     },
1177 };
1178 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1179
1180 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1181 sub verify_bootdev {
1182     my ($dev, $noerr) = @_;
1183
1184     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1185     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1186
1187     my $check = sub {
1188         my ($base) = @_;
1189         return 0 if $dev !~ m/^$base\d+$/;
1190         return 0 if !$confdesc->{$dev};
1191         return 1;
1192     };
1193
1194     return $dev if $check->("net");
1195     return $dev if $check->("usb");
1196     return $dev if $check->("hostpci");
1197
1198     return if $noerr;
1199     die "invalid boot device '$dev'\n";
1200 }
1201
1202 sub print_bootorder {
1203     my ($devs) = @_;
1204     return "" if !@$devs;
1205     my $data = { order => join(';', @$devs) };
1206     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1207 }
1208
1209 my $kvm_api_version = 0;
1210
1211 sub kvm_version {
1212     return $kvm_api_version if $kvm_api_version;
1213
1214     open my $fh, '<', '/dev/kvm' or return;
1215
1216     # 0xae00 => KVM_GET_API_VERSION
1217     $kvm_api_version = ioctl($fh, 0xae00, 0);
1218     close($fh);
1219
1220     return $kvm_api_version;
1221 }
1222
1223 my $kvm_user_version = {};
1224 my $kvm_mtime = {};
1225
1226 sub kvm_user_version {
1227     my ($binary) = @_;
1228
1229     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1230     my $st = stat($binary);
1231
1232     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1233     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1234         $cachedmtime == $st->mtime;
1235
1236     $kvm_user_version->{$binary} = 'unknown';
1237     $kvm_mtime->{$binary} = $st->mtime;
1238
1239     my $code = sub {
1240         my $line = shift;
1241         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1242             $kvm_user_version->{$binary} = $2;
1243         }
1244     };
1245
1246     eval { run_command([$binary, '--version'], outfunc => $code); };
1247     warn $@ if $@;
1248
1249     return $kvm_user_version->{$binary};
1250
1251 }
1252 my sub extract_version {
1253     my ($machine_type, $version) = @_;
1254     $version = kvm_user_version() if !defined($version);
1255     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1256 }
1257
1258 sub kernel_has_vhost_net {
1259     return -c '/dev/vhost-net';
1260 }
1261
1262 sub option_exists {
1263     my $key = shift;
1264     return defined($confdesc->{$key});
1265 }
1266
1267 my $cdrom_path;
1268 sub get_cdrom_path {
1269
1270     return  $cdrom_path if $cdrom_path;
1271
1272     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1273     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1274     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1275 }
1276
1277 sub get_iso_path {
1278     my ($storecfg, $vmid, $cdrom) = @_;
1279
1280     if ($cdrom eq 'cdrom') {
1281         return get_cdrom_path();
1282     } elsif ($cdrom eq 'none') {
1283         return '';
1284     } elsif ($cdrom =~ m|^/|) {
1285         return $cdrom;
1286     } else {
1287         return PVE::Storage::path($storecfg, $cdrom);
1288     }
1289 }
1290
1291 # try to convert old style file names to volume IDs
1292 sub filename_to_volume_id {
1293     my ($vmid, $file, $media) = @_;
1294
1295      if (!($file eq 'none' || $file eq 'cdrom' ||
1296           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1297
1298         return if $file =~ m|/|;
1299
1300         if ($media && $media eq 'cdrom') {
1301             $file = "local:iso/$file";
1302         } else {
1303             $file = "local:$vmid/$file";
1304         }
1305     }
1306
1307     return $file;
1308 }
1309
1310 sub verify_media_type {
1311     my ($opt, $vtype, $media) = @_;
1312
1313     return if !$media;
1314
1315     my $etype;
1316     if ($media eq 'disk') {
1317         $etype = 'images';
1318     } elsif ($media eq 'cdrom') {
1319         $etype = 'iso';
1320     } else {
1321         die "internal error";
1322     }
1323
1324     return if ($vtype eq $etype);
1325
1326     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1327 }
1328
1329 sub cleanup_drive_path {
1330     my ($opt, $storecfg, $drive) = @_;
1331
1332     # try to convert filesystem paths to volume IDs
1333
1334     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1335         ($drive->{file} !~ m|^/dev/.+|) &&
1336         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1337         ($drive->{file} !~ m/^\d+$/)) {
1338         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1339         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1340             if !$vtype;
1341         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1342         verify_media_type($opt, $vtype, $drive->{media});
1343         $drive->{file} = $volid;
1344     }
1345
1346     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1347 }
1348
1349 sub parse_hotplug_features {
1350     my ($data) = @_;
1351
1352     my $res = {};
1353
1354     return $res if $data eq '0';
1355
1356     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1357
1358     foreach my $feature (PVE::Tools::split_list($data)) {
1359         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1360             $res->{$1} = 1;
1361         } else {
1362             die "invalid hotplug feature '$feature'\n";
1363         }
1364     }
1365     return $res;
1366 }
1367
1368 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1369 sub pve_verify_hotplug_features {
1370     my ($value, $noerr) = @_;
1371
1372     return $value if parse_hotplug_features($value);
1373
1374     return if $noerr;
1375
1376     die "unable to parse hotplug option\n";
1377 }
1378
1379 sub scsi_inquiry {
1380     my($fh, $noerr) = @_;
1381
1382     my $SG_IO = 0x2285;
1383     my $SG_GET_VERSION_NUM = 0x2282;
1384
1385     my $versionbuf = "\x00" x 8;
1386     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1387     if (!$ret) {
1388         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1389         return;
1390     }
1391     my $version = unpack("I", $versionbuf);
1392     if ($version < 30000) {
1393         die "scsi generic interface too old\n"  if !$noerr;
1394         return;
1395     }
1396
1397     my $buf = "\x00" x 36;
1398     my $sensebuf = "\x00" x 8;
1399     my $cmd = pack("C x3 C x1", 0x12, 36);
1400
1401     # see /usr/include/scsi/sg.h
1402     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1403
1404     my $packet = pack(
1405         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1406     );
1407
1408     $ret = ioctl($fh, $SG_IO, $packet);
1409     if (!$ret) {
1410         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my @res = unpack($sg_io_hdr_t, $packet);
1415     if ($res[17] || $res[18]) {
1416         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1417         return;
1418     }
1419
1420     my $res = {};
1421     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1422
1423     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1424     $res->{type} &= 0x1F;
1425
1426     return $res;
1427 }
1428
1429 sub path_is_scsi {
1430     my ($path) = @_;
1431
1432     my $fh = IO::File->new("+<$path") || return;
1433     my $res = scsi_inquiry($fh, 1);
1434     close($fh);
1435
1436     return $res;
1437 }
1438
1439 sub print_tabletdevice_full {
1440     my ($conf, $arch) = @_;
1441
1442     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1443
1444     # we use uhci for old VMs because tablet driver was buggy in older qemu
1445     my $usbbus;
1446     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1447         $usbbus = 'ehci';
1448     } else {
1449         $usbbus = 'uhci';
1450     }
1451
1452     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1453 }
1454
1455 sub print_keyboarddevice_full {
1456     my ($conf, $arch) = @_;
1457
1458     return if $arch ne 'aarch64';
1459
1460     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1461 }
1462
1463 my sub get_drive_id {
1464     my ($drive) = @_;
1465     return "$drive->{interface}$drive->{index}";
1466 }
1467
1468 sub print_drivedevice_full {
1469     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1470
1471     my $device = '';
1472     my $maxdev = 0;
1473
1474     my $drive_id = get_drive_id($drive);
1475     if ($drive->{interface} eq 'virtio') {
1476         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1477         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1478         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1479     } elsif ($drive->{interface} eq 'scsi') {
1480
1481         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1482         my $unit = $drive->{index} % $maxdev;
1483         my $devicetype = 'hd';
1484         my $path = '';
1485         if (drive_is_cdrom($drive)) {
1486             $devicetype = 'cd';
1487         } else {
1488             if ($drive->{file} =~ m|^/|) {
1489                 $path = $drive->{file};
1490                 if (my $info = path_is_scsi($path)) {
1491                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1492                         $devicetype = 'block';
1493                     } elsif ($info->{type} == 1) { # tape
1494                         $devicetype = 'generic';
1495                     }
1496                 }
1497             } else {
1498                  $path = PVE::Storage::path($storecfg, $drive->{file});
1499             }
1500
1501             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1502             my $version = extract_version($machine_type, kvm_user_version());
1503             if ($path =~ m/^iscsi\:\/\// &&
1504                !min_version($version, 4, 1)) {
1505                 $devicetype = 'generic';
1506             }
1507         }
1508
1509         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1510             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1511         } else {
1512             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1513                 .",lun=$drive->{index}";
1514         }
1515         $device .= ",drive=drive-$drive_id,id=$drive_id";
1516
1517         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1518             $device .= ",rotation_rate=1";
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521
1522     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1523         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1524         my $controller = int($drive->{index} / $maxdev);
1525         my $unit = $drive->{index} % $maxdev;
1526         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1527
1528         $device = "ide-$devicetype";
1529         if ($drive->{interface} eq 'ide') {
1530             $device .= ",bus=ide.$controller,unit=$unit";
1531         } else {
1532             $device .= ",bus=ahci$controller.$unit";
1533         }
1534         $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536         if ($devicetype eq 'hd') {
1537             if (my $model = $drive->{model}) {
1538                 $model = URI::Escape::uri_unescape($model);
1539                 $device .= ",model=$model";
1540             }
1541             if ($drive->{ssd}) {
1542                 $device .= ",rotation_rate=1";
1543             }
1544         }
1545         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1546     } elsif ($drive->{interface} eq 'usb') {
1547         die "implement me";
1548         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1549     } else {
1550         die "unsupported interface type";
1551     }
1552
1553     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1554
1555     if (my $serial = $drive->{serial}) {
1556         $serial = URI::Escape::uri_unescape($serial);
1557         $device .= ",serial=$serial";
1558     }
1559
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_commandline_full {
1579     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format = $drive->{format};
1584     my $drive_id = get_drive_id($drive);
1585
1586     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1587     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1588
1589     if (drive_is_cdrom($drive)) {
1590         $path = get_iso_path($storecfg, $vmid, $volid);
1591         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1592     } else {
1593         if ($storeid) {
1594             $path = PVE::Storage::path($storecfg, $volid);
1595             $format //= qemu_img_format($scfg, $volname);
1596         } else {
1597             $path = $volid;
1598             $format //= "raw";
1599         }
1600    }
1601
1602    my $is_rbd = $path =~ m/^rbd:/;
1603
1604     my $opts = '';
1605     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1606     foreach my $o (@qemu_drive_options) {
1607         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1608     }
1609
1610     # snapshot only accepts on|off
1611     if (defined($drive->{snapshot})) {
1612         my $v = $drive->{snapshot} ? 'on' : 'off';
1613         $opts .= ",snapshot=$v";
1614     }
1615
1616     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1617         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1618     }
1619
1620     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1621         my ($dir, $qmpname) = @$type;
1622         if (my $v = $drive->{"mbps$dir"}) {
1623             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1624         }
1625         if (my $v = $drive->{"mbps${dir}_max"}) {
1626             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1627         }
1628         if (my $v = $drive->{"bps${dir}_max_length"}) {
1629             $opts .= ",throttling.bps$qmpname-max-length=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}"}) {
1632             $opts .= ",throttling.iops$qmpname=$v";
1633         }
1634         if (my $v = $drive->{"iops${dir}_max"}) {
1635             $opts .= ",throttling.iops$qmpname-max=$v";
1636         }
1637         if (my $v = $drive->{"iops${dir}_max_length"}) {
1638             $opts .= ",throttling.iops$qmpname-max-length=$v";
1639         }
1640     }
1641
1642     if ($pbs_name) {
1643         $format = "rbd" if $is_rbd;
1644         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1645             if !$format;
1646         $opts .= ",format=alloc-track,file.driver=$format";
1647     } elsif ($format) {
1648         $opts .= ",format=$format";
1649     }
1650
1651     my $cache_direct = 0;
1652
1653     if (my $cache = $drive->{cache}) {
1654         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1655     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1656         $opts .= ",cache=none";
1657         $cache_direct = 1;
1658     }
1659
1660     # io_uring with cache mode writeback or writethrough on krbd will hang...
1661     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1662
1663     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1664     # sometimes, just plain disable...
1665     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1666
1667     if (!$drive->{aio}) {
1668         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1669             # io_uring supports all cache modes
1670             $opts .= ",aio=io_uring";
1671         } else {
1672             # aio native works only with O_DIRECT
1673             if($cache_direct) {
1674                 $opts .= ",aio=native";
1675             } else {
1676                 $opts .= ",aio=threads";
1677             }
1678         }
1679     }
1680
1681     if (!drive_is_cdrom($drive)) {
1682         my $detectzeroes;
1683         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1684             $detectzeroes = 'off';
1685         } elsif ($drive->{discard}) {
1686             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1687         } else {
1688             # This used to be our default with discard not being specified:
1689             $detectzeroes = 'on';
1690         }
1691
1692         # note: 'detect-zeroes' works per blockdev and we want it to persist
1693         # after the alloc-track is removed, so put it on 'file' directly
1694         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1695         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1696     }
1697
1698     if ($pbs_name) {
1699         $opts .= ",backing=$pbs_name";
1700         $opts .= ",auto-remove=on";
1701     }
1702
1703     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1704     my $file_param = "file";
1705     if ($pbs_name) {
1706         # non-rbd drivers require the underlying file to be a seperate block
1707         # node, so add a second .file indirection
1708         $file_param .= ".file" if !$is_rbd;
1709         $file_param .= ".filename";
1710     }
1711     my $pathinfo = $path ? "$file_param=$path," : '';
1712
1713     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1714 }
1715
1716 sub print_pbs_blockdev {
1717     my ($pbs_conf, $pbs_name) = @_;
1718     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1719     $blockdev .= ",repository=$pbs_conf->{repository}";
1720     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1721     $blockdev .= ",archive=$pbs_conf->{archive}";
1722     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1723     return $blockdev;
1724 }
1725
1726 sub print_netdevice_full {
1727     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1728
1729     my $device = $net->{model};
1730     if ($net->{model} eq 'virtio') {
1731          $device = 'virtio-net-pci';
1732      };
1733
1734     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1735     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1736     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1737         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1738         # and out of each queue plus one config interrupt and control vector queue
1739         my $vectors = $net->{queues} * 2 + 2;
1740         $tmpstr .= ",vectors=$vectors,mq=on";
1741     }
1742     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1743
1744     if (my $mtu = $net->{mtu}) {
1745         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1746             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1747             if ($mtu == 1) {
1748                  $mtu = $bridge_mtu;
1749             } elsif ($mtu < 576) {
1750                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1751             } elsif ($mtu > $bridge_mtu) {
1752                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1753             }
1754             $tmpstr .= ",host_mtu=$mtu";
1755         } else {
1756             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1757         }
1758     }
1759
1760     if ($use_old_bios_files) {
1761         my $romfile;
1762         if ($device eq 'virtio-net-pci') {
1763             $romfile = 'pxe-virtio.rom';
1764         } elsif ($device eq 'e1000') {
1765             $romfile = 'pxe-e1000.rom';
1766         } elsif ($device eq 'e1000e') {
1767             $romfile = 'pxe-e1000e.rom';
1768         } elsif ($device eq 'ne2k') {
1769             $romfile = 'pxe-ne2k_pci.rom';
1770         } elsif ($device eq 'pcnet') {
1771             $romfile = 'pxe-pcnet.rom';
1772         } elsif ($device eq 'rtl8139') {
1773             $romfile = 'pxe-rtl8139.rom';
1774         }
1775         $tmpstr .= ",romfile=$romfile" if $romfile;
1776     }
1777
1778     return $tmpstr;
1779 }
1780
1781 sub print_netdev_full {
1782     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1783
1784     my $i = '';
1785     if ($netid =~ m/^net(\d+)$/) {
1786         $i = int($1);
1787     }
1788
1789     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1790
1791     my $ifname = "tap${vmid}i$i";
1792
1793     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1794     die "interface name '$ifname' is too long (max 15 character)\n"
1795         if length($ifname) >= 16;
1796
1797     my $vhostparam = '';
1798     if (is_native($arch)) {
1799         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1800     }
1801
1802     my $vmname = $conf->{name} || "vm$vmid";
1803
1804     my $netdev = "";
1805     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1806
1807     if ($net->{bridge}) {
1808         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1809             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1810     } else {
1811         $netdev = "type=user,id=$netid,hostname=$vmname";
1812     }
1813
1814     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1815
1816     return $netdev;
1817 }
1818
1819 my $vga_map = {
1820     'cirrus' => 'cirrus-vga',
1821     'std' => 'VGA',
1822     'vmware' => 'vmware-svga',
1823     'virtio' => 'virtio-vga',
1824     'virtio-gl' => 'virtio-vga-gl',
1825 };
1826
1827 sub print_vga_device {
1828     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1829
1830     my $type = $vga_map->{$vga->{type}};
1831     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1832         $type = 'virtio-gpu';
1833     }
1834     my $vgamem_mb = $vga->{memory};
1835
1836     my $max_outputs = '';
1837     if ($qxlnum) {
1838         $type = $id ? 'qxl' : 'qxl-vga';
1839
1840         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1841             # set max outputs so linux can have up to 4 qxl displays with one device
1842             if (min_version($machine_version, 4, 1)) {
1843                 $max_outputs = ",max_outputs=4";
1844             }
1845         }
1846     }
1847
1848     die "no devicetype for $vga->{type}\n" if !$type;
1849
1850     my $memory = "";
1851     if ($vgamem_mb) {
1852         if ($vga->{type} =~ /^virtio/) {
1853             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1854             $memory = ",max_hostmem=$bytes";
1855         } elsif ($qxlnum) {
1856             # from https://www.spice-space.org/multiple-monitors.html
1857             $memory = ",vgamem_mb=$vga->{memory}";
1858             my $ram = $vgamem_mb * 4;
1859             my $vram = $vgamem_mb * 2;
1860             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1861         } else {
1862             $memory = ",vgamem_mb=$vga->{memory}";
1863         }
1864     } elsif ($qxlnum && $id) {
1865         $memory = ",ram_size=67108864,vram_size=33554432";
1866     }
1867
1868     my $edidoff = "";
1869     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1870         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1871     }
1872
1873     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1874     my $vgaid = "vga" . ($id // '');
1875     my $pciaddr;
1876     if ($q35 && $vgaid eq 'vga') {
1877         # the first display uses pcie.0 bus on q35 machines
1878         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1879     } else {
1880         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1881     }
1882
1883     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1884 }
1885
1886 sub parse_number_sets {
1887     my ($set) = @_;
1888     my $res = [];
1889     foreach my $part (split(/;/, $set)) {
1890         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1891             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1892             push @$res, [ $1, $2 ];
1893         } else {
1894             die "invalid range: $part\n";
1895         }
1896     }
1897     return $res;
1898 }
1899
1900 sub parse_numa {
1901     my ($data) = @_;
1902
1903     my $res = parse_property_string($numa_fmt, $data);
1904     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1905     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1906     return $res;
1907 }
1908
1909 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1910 sub parse_net {
1911     my ($data) = @_;
1912
1913     my $res = eval { parse_property_string($net_fmt, $data) };
1914     if ($@) {
1915         warn $@;
1916         return;
1917     }
1918     if (!defined($res->{macaddr})) {
1919         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1920         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1921     }
1922     return $res;
1923 }
1924
1925 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1926 sub parse_ipconfig {
1927     my ($data) = @_;
1928
1929     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1930     if ($@) {
1931         warn $@;
1932         return;
1933     }
1934
1935     if ($res->{gw} && !$res->{ip}) {
1936         warn 'gateway specified without specifying an IP address';
1937         return;
1938     }
1939     if ($res->{gw6} && !$res->{ip6}) {
1940         warn 'IPv6 gateway specified without specifying an IPv6 address';
1941         return;
1942     }
1943     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1944         warn 'gateway specified together with DHCP';
1945         return;
1946     }
1947     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1948         # gw6 + auto/dhcp
1949         warn "IPv6 gateway specified together with $res->{ip6} address";
1950         return;
1951     }
1952
1953     if (!$res->{ip} && !$res->{ip6}) {
1954         return { ip => 'dhcp', ip6 => 'dhcp' };
1955     }
1956
1957     return $res;
1958 }
1959
1960 sub print_net {
1961     my $net = shift;
1962
1963     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1964 }
1965
1966 sub add_random_macs {
1967     my ($settings) = @_;
1968
1969     foreach my $opt (keys %$settings) {
1970         next if $opt !~ m/^net(\d+)$/;
1971         my $net = parse_net($settings->{$opt});
1972         next if !$net;
1973         $settings->{$opt} = print_net($net);
1974     }
1975 }
1976
1977 sub vm_is_volid_owner {
1978     my ($storecfg, $vmid, $volid) = @_;
1979
1980     if ($volid !~  m|^/|) {
1981         my ($path, $owner);
1982         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1983         if ($owner && ($owner == $vmid)) {
1984             return 1;
1985         }
1986     }
1987
1988     return;
1989 }
1990
1991 sub vmconfig_register_unused_drive {
1992     my ($storecfg, $vmid, $conf, $drive) = @_;
1993
1994     if (drive_is_cloudinit($drive)) {
1995         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1996         warn $@ if $@;
1997     } elsif (!drive_is_cdrom($drive)) {
1998         my $volid = $drive->{file};
1999         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2000             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2001         }
2002     }
2003 }
2004
2005 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2006 my $smbios1_fmt = {
2007     uuid => {
2008         type => 'string',
2009         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2010         format_description => 'UUID',
2011         description => "Set SMBIOS1 UUID.",
2012         optional => 1,
2013     },
2014     version => {
2015         type => 'string',
2016         pattern => '[A-Za-z0-9+\/]+={0,2}',
2017         format_description => 'Base64 encoded string',
2018         description => "Set SMBIOS1 version.",
2019         optional => 1,
2020     },
2021     serial => {
2022         type => 'string',
2023         pattern => '[A-Za-z0-9+\/]+={0,2}',
2024         format_description => 'Base64 encoded string',
2025         description => "Set SMBIOS1 serial number.",
2026         optional => 1,
2027     },
2028     manufacturer => {
2029         type => 'string',
2030         pattern => '[A-Za-z0-9+\/]+={0,2}',
2031         format_description => 'Base64 encoded string',
2032         description => "Set SMBIOS1 manufacturer.",
2033         optional => 1,
2034     },
2035     product => {
2036         type => 'string',
2037         pattern => '[A-Za-z0-9+\/]+={0,2}',
2038         format_description => 'Base64 encoded string',
2039         description => "Set SMBIOS1 product ID.",
2040         optional => 1,
2041     },
2042     sku => {
2043         type => 'string',
2044         pattern => '[A-Za-z0-9+\/]+={0,2}',
2045         format_description => 'Base64 encoded string',
2046         description => "Set SMBIOS1 SKU string.",
2047         optional => 1,
2048     },
2049     family => {
2050         type => 'string',
2051         pattern => '[A-Za-z0-9+\/]+={0,2}',
2052         format_description => 'Base64 encoded string',
2053         description => "Set SMBIOS1 family string.",
2054         optional => 1,
2055     },
2056     base64 => {
2057         type => 'boolean',
2058         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2059         optional => 1,
2060     },
2061 };
2062
2063 sub parse_smbios1 {
2064     my ($data) = @_;
2065
2066     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2067     warn $@ if $@;
2068     return $res;
2069 }
2070
2071 sub print_smbios1 {
2072     my ($smbios1) = @_;
2073     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2074 }
2075
2076 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2077
2078 sub parse_watchdog {
2079     my ($value) = @_;
2080
2081     return if !$value;
2082
2083     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2084     warn $@ if $@;
2085     return $res;
2086 }
2087
2088 sub parse_guest_agent {
2089     my ($conf) = @_;
2090
2091     return {} if !defined($conf->{agent});
2092
2093     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2094     warn $@ if $@;
2095
2096     # if the agent is disabled ignore the other potentially set properties
2097     return {} if !$res->{enabled};
2098     return $res;
2099 }
2100
2101 sub get_qga_key {
2102     my ($conf, $key) = @_;
2103     return undef if !defined($conf->{agent});
2104
2105     my $agent = parse_guest_agent($conf);
2106     return $agent->{$key};
2107 }
2108
2109 sub parse_vga {
2110     my ($value) = @_;
2111
2112     return {} if !$value;
2113     my $res = eval { parse_property_string($vga_fmt, $value) };
2114     warn $@ if $@;
2115     return $res;
2116 }
2117
2118 sub parse_rng {
2119     my ($value) = @_;
2120
2121     return if !$value;
2122
2123     my $res = eval { parse_property_string($rng_fmt, $value) };
2124     warn $@ if $@;
2125     return $res;
2126 }
2127
2128 sub parse_meta_info {
2129     my ($value) = @_;
2130
2131     return if !$value;
2132
2133     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2134     warn $@ if $@;
2135     return $res;
2136 }
2137
2138 sub new_meta_info_string {
2139     my () = @_; # for now do not allow to override any value
2140
2141     return PVE::JSONSchema::print_property_string(
2142         {
2143             'creation-qemu' => kvm_user_version(),
2144             ctime => "". int(time()),
2145         },
2146         $meta_info_fmt
2147     );
2148 }
2149
2150 sub qemu_created_version_fixups {
2151     my ($conf, $forcemachine, $kvmver) = @_;
2152
2153     my $meta = parse_meta_info($conf->{meta}) // {};
2154     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2155
2156     # check if we need to apply some handling for VMs that always use the latest machine version but
2157     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2158     # would be required (we do not want to pin machine version for non-windows OS type)
2159     if (
2160         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2161         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2162         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2163         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2164     ) {
2165         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2166         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2167             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2168             # and thus with the predictable interface naming of systemd
2169             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2170         }
2171     }
2172     return;
2173 }
2174
2175 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2176 sub verify_usb_device {
2177     my ($value, $noerr) = @_;
2178
2179     return $value if parse_usb_device($value);
2180
2181     return if $noerr;
2182
2183     die "unable to parse usb device\n";
2184 }
2185
2186 # add JSON properties for create and set function
2187 sub json_config_properties {
2188     my ($prop, $with_disk_alloc) = @_;
2189
2190     my $skip_json_config_opts = {
2191         parent => 1,
2192         snaptime => 1,
2193         vmstate => 1,
2194         runningmachine => 1,
2195         runningcpu => 1,
2196         meta => 1,
2197     };
2198
2199     foreach my $opt (keys %$confdesc) {
2200         next if $skip_json_config_opts->{$opt};
2201
2202         if ($with_disk_alloc && is_valid_drivename($opt)) {
2203             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2204         } else {
2205             $prop->{$opt} = $confdesc->{$opt};
2206         }
2207     }
2208
2209     return $prop;
2210 }
2211
2212 # Properties that we can read from an OVF file
2213 sub json_ovf_properties {
2214     my $prop = {};
2215
2216     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2217         $prop->{$device} = {
2218             type => 'string',
2219             format => 'pve-volume-id-or-absolute-path',
2220             description => "Disk image that gets imported to $device",
2221             optional => 1,
2222         };
2223     }
2224
2225     $prop->{cores} = {
2226         type => 'integer',
2227         description => "The number of CPU cores.",
2228         optional => 1,
2229     };
2230     $prop->{memory} = {
2231         type => 'integer',
2232         description => "Amount of RAM for the VM in MB.",
2233         optional => 1,
2234     };
2235     $prop->{name} = {
2236         type => 'string',
2237         description => "Name of the VM.",
2238         optional => 1,
2239     };
2240
2241     return $prop;
2242 }
2243
2244 # return copy of $confdesc_cloudinit to generate documentation
2245 sub cloudinit_config_properties {
2246
2247     return dclone($confdesc_cloudinit);
2248 }
2249
2250 sub check_type {
2251     my ($key, $value) = @_;
2252
2253     die "unknown setting '$key'\n" if !$confdesc->{$key};
2254
2255     my $type = $confdesc->{$key}->{type};
2256
2257     if (!defined($value)) {
2258         die "got undefined value\n";
2259     }
2260
2261     if ($value =~ m/[\n\r]/) {
2262         die "property contains a line feed\n";
2263     }
2264
2265     if ($type eq 'boolean') {
2266         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2267         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2268         die "type check ('boolean') failed - got '$value'\n";
2269     } elsif ($type eq 'integer') {
2270         return int($1) if $value =~ m/^(\d+)$/;
2271         die "type check ('integer') failed - got '$value'\n";
2272     } elsif ($type eq 'number') {
2273         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2274         die "type check ('number') failed - got '$value'\n";
2275     } elsif ($type eq 'string') {
2276         if (my $fmt = $confdesc->{$key}->{format}) {
2277             PVE::JSONSchema::check_format($fmt, $value);
2278             return $value;
2279         }
2280         $value =~ s/^\"(.*)\"$/$1/;
2281         return $value;
2282     } else {
2283         die "internal error"
2284     }
2285 }
2286
2287 sub destroy_vm {
2288     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2289
2290     my $conf = PVE::QemuConfig->load_config($vmid);
2291
2292     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2293
2294     if ($conf->{template}) {
2295         # check if any base image is still used by a linked clone
2296         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2297                 my ($ds, $drive) = @_;
2298                 return if drive_is_cdrom($drive);
2299
2300                 my $volid = $drive->{file};
2301                 return if !$volid || $volid =~ m|^/|;
2302
2303                 die "base volume '$volid' is still in use by linked cloned\n"
2304                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2305
2306         });
2307     }
2308
2309     my $volids = {};
2310     my $remove_owned_drive = sub {
2311         my ($ds, $drive) = @_;
2312         return if drive_is_cdrom($drive, 1);
2313
2314         my $volid = $drive->{file};
2315         return if !$volid || $volid =~ m|^/|;
2316         return if $volids->{$volid};
2317
2318         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2319         return if !$path || !$owner || ($owner != $vmid);
2320
2321         $volids->{$volid} = 1;
2322         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2323         warn "Could not remove disk '$volid', check manually: $@" if $@;
2324     };
2325
2326     # only remove disks owned by this VM (referenced in the config)
2327     my $include_opts = {
2328         include_unused => 1,
2329         extra_keys => ['vmstate'],
2330     };
2331     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2332
2333     for my $snap (values %{$conf->{snapshots}}) {
2334         next if !defined($snap->{vmstate});
2335         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2336         next if !defined($drive);
2337         $remove_owned_drive->('vmstate', $drive);
2338     }
2339
2340     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2341
2342     if ($purge_unreferenced) { # also remove unreferenced disk
2343         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2344         PVE::Storage::foreach_volid($vmdisks, sub {
2345             my ($volid, $sid, $volname, $d) = @_;
2346             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2347             warn $@ if $@;
2348         });
2349     }
2350
2351     if (defined $replacement_conf) {
2352         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2353     } else {
2354         PVE::QemuConfig->destroy_config($vmid);
2355     }
2356 }
2357
2358 sub parse_vm_config {
2359     my ($filename, $raw, $strict) = @_;
2360
2361     return if !defined($raw);
2362
2363     my $res = {
2364         digest => Digest::SHA::sha1_hex($raw),
2365         snapshots => {},
2366         pending => {},
2367     };
2368
2369     my $handle_error = sub {
2370         my ($msg) = @_;
2371
2372         if ($strict) {
2373             die $msg;
2374         } else {
2375             warn $msg;
2376         }
2377     };
2378
2379     $filename =~ m|/qemu-server/(\d+)\.conf$|
2380         || die "got strange filename '$filename'";
2381
2382     my $vmid = $1;
2383
2384     my $conf = $res;
2385     my $descr;
2386     my $section = '';
2387
2388     my @lines = split(/\n/, $raw);
2389     foreach my $line (@lines) {
2390         next if $line =~ m/^\s*$/;
2391
2392         if ($line =~ m/^\[PENDING\]\s*$/i) {
2393             $section = 'pending';
2394             if (defined($descr)) {
2395                 $descr =~ s/\s+$//;
2396                 $conf->{description} = $descr;
2397             }
2398             $descr = undef;
2399             $conf = $res->{$section} = {};
2400             next;
2401
2402         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2403             $section = $1;
2404             if (defined($descr)) {
2405                 $descr =~ s/\s+$//;
2406                 $conf->{description} = $descr;
2407             }
2408             $descr = undef;
2409             $conf = $res->{snapshots}->{$section} = {};
2410             next;
2411         }
2412
2413         if ($line =~ m/^\#(.*)$/) {
2414             $descr = '' if !defined($descr);
2415             $descr .= PVE::Tools::decode_text($1) . "\n";
2416             next;
2417         }
2418
2419         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2420             $descr = '' if !defined($descr);
2421             $descr .= PVE::Tools::decode_text($2);
2422         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2423             $conf->{snapstate} = $1;
2424         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2425             my $key = $1;
2426             my $value = $2;
2427             $conf->{$key} = $value;
2428         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2429             my $value = $1;
2430             if ($section eq 'pending') {
2431                 $conf->{delete} = $value; # we parse this later
2432             } else {
2433                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2434             }
2435         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2436             my $key = $1;
2437             my $value = $2;
2438             eval { $value = check_type($key, $value); };
2439             if ($@) {
2440                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2441             } else {
2442                 $key = 'ide2' if $key eq 'cdrom';
2443                 my $fmt = $confdesc->{$key}->{format};
2444                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2445                     my $v = parse_drive($key, $value);
2446                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2447                         $v->{file} = $volid;
2448                         $value = print_drive($v);
2449                     } else {
2450                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2451                         next;
2452                     }
2453                 }
2454
2455                 $conf->{$key} = $value;
2456             }
2457         } else {
2458             $handle_error->("vm $vmid - unable to parse config: $line\n");
2459         }
2460     }
2461
2462     if (defined($descr)) {
2463         $descr =~ s/\s+$//;
2464         $conf->{description} = $descr;
2465     }
2466     delete $res->{snapstate}; # just to be sure
2467
2468     return $res;
2469 }
2470
2471 sub write_vm_config {
2472     my ($filename, $conf) = @_;
2473
2474     delete $conf->{snapstate}; # just to be sure
2475
2476     if ($conf->{cdrom}) {
2477         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2478         $conf->{ide2} = $conf->{cdrom};
2479         delete $conf->{cdrom};
2480     }
2481
2482     # we do not use 'smp' any longer
2483     if ($conf->{sockets}) {
2484         delete $conf->{smp};
2485     } elsif ($conf->{smp}) {
2486         $conf->{sockets} = $conf->{smp};
2487         delete $conf->{cores};
2488         delete $conf->{smp};
2489     }
2490
2491     my $used_volids = {};
2492
2493     my $cleanup_config = sub {
2494         my ($cref, $pending, $snapname) = @_;
2495
2496         foreach my $key (keys %$cref) {
2497             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2498                 $key eq 'snapstate' || $key eq 'pending';
2499             my $value = $cref->{$key};
2500             if ($key eq 'delete') {
2501                 die "propertry 'delete' is only allowed in [PENDING]\n"
2502                     if !$pending;
2503                 # fixme: check syntax?
2504                 next;
2505             }
2506             eval { $value = check_type($key, $value); };
2507             die "unable to parse value of '$key' - $@" if $@;
2508
2509             $cref->{$key} = $value;
2510
2511             if (!$snapname && is_valid_drivename($key)) {
2512                 my $drive = parse_drive($key, $value);
2513                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2514             }
2515         }
2516     };
2517
2518     &$cleanup_config($conf);
2519
2520     &$cleanup_config($conf->{pending}, 1);
2521
2522     foreach my $snapname (keys %{$conf->{snapshots}}) {
2523         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2524         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2525     }
2526
2527     # remove 'unusedX' settings if we re-add a volume
2528     foreach my $key (keys %$conf) {
2529         my $value = $conf->{$key};
2530         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2531             delete $conf->{$key};
2532         }
2533     }
2534
2535     my $generate_raw_config = sub {
2536         my ($conf, $pending) = @_;
2537
2538         my $raw = '';
2539
2540         # add description as comment to top of file
2541         if (defined(my $descr = $conf->{description})) {
2542             if ($descr) {
2543                 foreach my $cl (split(/\n/, $descr)) {
2544                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2545                 }
2546             } else {
2547                 $raw .= "#\n" if $pending;
2548             }
2549         }
2550
2551         foreach my $key (sort keys %$conf) {
2552             next if $key =~ /^(digest|description|pending|snapshots)$/;
2553             $raw .= "$key: $conf->{$key}\n";
2554         }
2555         return $raw;
2556     };
2557
2558     my $raw = &$generate_raw_config($conf);
2559
2560     if (scalar(keys %{$conf->{pending}})){
2561         $raw .= "\n[PENDING]\n";
2562         $raw .= &$generate_raw_config($conf->{pending}, 1);
2563     }
2564
2565     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2566         $raw .= "\n[$snapname]\n";
2567         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2568     }
2569
2570     return $raw;
2571 }
2572
2573 sub load_defaults {
2574
2575     my $res = {};
2576
2577     # we use static defaults from our JSON schema configuration
2578     foreach my $key (keys %$confdesc) {
2579         if (defined(my $default = $confdesc->{$key}->{default})) {
2580             $res->{$key} = $default;
2581         }
2582     }
2583
2584     return $res;
2585 }
2586
2587 sub config_list {
2588     my $vmlist = PVE::Cluster::get_vmlist();
2589     my $res = {};
2590     return $res if !$vmlist || !$vmlist->{ids};
2591     my $ids = $vmlist->{ids};
2592     my $nodename = nodename();
2593
2594     foreach my $vmid (keys %$ids) {
2595         my $d = $ids->{$vmid};
2596         next if !$d->{node} || $d->{node} ne $nodename;
2597         next if !$d->{type} || $d->{type} ne 'qemu';
2598         $res->{$vmid}->{exists} = 1;
2599     }
2600     return $res;
2601 }
2602
2603 # test if VM uses local resources (to prevent migration)
2604 sub check_local_resources {
2605     my ($conf, $noerr) = @_;
2606
2607     my @loc_res = ();
2608
2609     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2610     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2611
2612     push @loc_res, "ivshmem" if $conf->{ivshmem};
2613
2614     foreach my $k (keys %$conf) {
2615         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2616         # sockets are safe: they will recreated be on the target side post-migrate
2617         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2618         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2619     }
2620
2621     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2622
2623     return \@loc_res;
2624 }
2625
2626 # check if used storages are available on all nodes (use by migrate)
2627 sub check_storage_availability {
2628     my ($storecfg, $conf, $node) = @_;
2629
2630     PVE::QemuConfig->foreach_volume($conf, sub {
2631         my ($ds, $drive) = @_;
2632
2633         my $volid = $drive->{file};
2634         return if !$volid;
2635
2636         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2637         return if !$sid;
2638
2639         # check if storage is available on both nodes
2640         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2641         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2642
2643         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2644
2645         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2646             if !$scfg->{content}->{$vtype};
2647    });
2648 }
2649
2650 # list nodes where all VM images are available (used by has_feature API)
2651 sub shared_nodes {
2652     my ($conf, $storecfg) = @_;
2653
2654     my $nodelist = PVE::Cluster::get_nodelist();
2655     my $nodehash = { map { $_ => 1 } @$nodelist };
2656     my $nodename = nodename();
2657
2658     PVE::QemuConfig->foreach_volume($conf, sub {
2659         my ($ds, $drive) = @_;
2660
2661         my $volid = $drive->{file};
2662         return if !$volid;
2663
2664         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2665         if ($storeid) {
2666             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2667             if ($scfg->{disable}) {
2668                 $nodehash = {};
2669             } elsif (my $avail = $scfg->{nodes}) {
2670                 foreach my $node (keys %$nodehash) {
2671                     delete $nodehash->{$node} if !$avail->{$node};
2672                 }
2673             } elsif (!$scfg->{shared}) {
2674                 foreach my $node (keys %$nodehash) {
2675                     delete $nodehash->{$node} if $node ne $nodename
2676                 }
2677             }
2678         }
2679     });
2680
2681     return $nodehash
2682 }
2683
2684 sub check_local_storage_availability {
2685     my ($conf, $storecfg) = @_;
2686
2687     my $nodelist = PVE::Cluster::get_nodelist();
2688     my $nodehash = { map { $_ => {} } @$nodelist };
2689
2690     PVE::QemuConfig->foreach_volume($conf, sub {
2691         my ($ds, $drive) = @_;
2692
2693         my $volid = $drive->{file};
2694         return if !$volid;
2695
2696         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2697         if ($storeid) {
2698             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2699
2700             if ($scfg->{disable}) {
2701                 foreach my $node (keys %$nodehash) {
2702                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2703                 }
2704             } elsif (my $avail = $scfg->{nodes}) {
2705                 foreach my $node (keys %$nodehash) {
2706                     if (!$avail->{$node}) {
2707                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2708                     }
2709                 }
2710             }
2711         }
2712     });
2713
2714     foreach my $node (values %$nodehash) {
2715         if (my $unavail = $node->{unavailable_storages}) {
2716             $node->{unavailable_storages} = [ sort keys %$unavail ];
2717         }
2718     }
2719
2720     return $nodehash
2721 }
2722
2723 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2724 sub check_running {
2725     my ($vmid, $nocheck, $node) = @_;
2726
2727     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2728     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2729 }
2730
2731 sub vzlist {
2732
2733     my $vzlist = config_list();
2734
2735     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2736
2737     while (defined(my $de = $fd->read)) {
2738         next if $de !~ m/^(\d+)\.pid$/;
2739         my $vmid = $1;
2740         next if !defined($vzlist->{$vmid});
2741         if (my $pid = check_running($vmid)) {
2742             $vzlist->{$vmid}->{pid} = $pid;
2743         }
2744     }
2745
2746     return $vzlist;
2747 }
2748
2749 our $vmstatus_return_properties = {
2750     vmid => get_standard_option('pve-vmid'),
2751     status => {
2752         description => "Qemu process status.",
2753         type => 'string',
2754         enum => ['stopped', 'running'],
2755     },
2756     maxmem => {
2757         description => "Maximum memory in bytes.",
2758         type => 'integer',
2759         optional => 1,
2760         renderer => 'bytes',
2761     },
2762     maxdisk => {
2763         description => "Root disk size in bytes.",
2764         type => 'integer',
2765         optional => 1,
2766         renderer => 'bytes',
2767     },
2768     name => {
2769         description => "VM name.",
2770         type => 'string',
2771         optional => 1,
2772     },
2773     qmpstatus => {
2774         description => "Qemu QMP agent status.",
2775         type => 'string',
2776         optional => 1,
2777     },
2778     pid => {
2779         description => "PID of running qemu process.",
2780         type => 'integer',
2781         optional => 1,
2782     },
2783     uptime => {
2784         description => "Uptime.",
2785         type => 'integer',
2786         optional => 1,
2787         renderer => 'duration',
2788     },
2789     cpus => {
2790         description => "Maximum usable CPUs.",
2791         type => 'number',
2792         optional => 1,
2793     },
2794     lock => {
2795         description => "The current config lock, if any.",
2796         type => 'string',
2797         optional => 1,
2798     },
2799     tags => {
2800         description  => "The current configured tags, if any",
2801         type => 'string',
2802         optional => 1,
2803     },
2804     'running-machine' => {
2805         description => "The currently running machine type (if running).",
2806         type => 'string',
2807         optional => 1,
2808     },
2809     'running-qemu' => {
2810         description => "The currently running QEMU version (if running).",
2811         type => 'string',
2812         optional => 1,
2813     },
2814 };
2815
2816 my $last_proc_pid_stat;
2817
2818 # get VM status information
2819 # This must be fast and should not block ($full == false)
2820 # We only query KVM using QMP if $full == true (this can be slow)
2821 sub vmstatus {
2822     my ($opt_vmid, $full) = @_;
2823
2824     my $res = {};
2825
2826     my $storecfg = PVE::Storage::config();
2827
2828     my $list = vzlist();
2829     my $defaults = load_defaults();
2830
2831     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2832
2833     my $cpucount = $cpuinfo->{cpus} || 1;
2834
2835     foreach my $vmid (keys %$list) {
2836         next if $opt_vmid && ($vmid ne $opt_vmid);
2837
2838         my $conf = PVE::QemuConfig->load_config($vmid);
2839
2840         my $d = { vmid => int($vmid) };
2841         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2842
2843         # fixme: better status?
2844         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2845
2846         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2847         if (defined($size)) {
2848             $d->{disk} = 0; # no info available
2849             $d->{maxdisk} = $size;
2850         } else {
2851             $d->{disk} = 0;
2852             $d->{maxdisk} = 0;
2853         }
2854
2855         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2856             * ($conf->{cores} || $defaults->{cores});
2857         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2858         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2859
2860         $d->{name} = $conf->{name} || "VM $vmid";
2861         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2862             : $defaults->{memory}*(1024*1024);
2863
2864         if ($conf->{balloon}) {
2865             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2866             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2867                 : $defaults->{shares};
2868         }
2869
2870         $d->{uptime} = 0;
2871         $d->{cpu} = 0;
2872         $d->{mem} = 0;
2873
2874         $d->{netout} = 0;
2875         $d->{netin} = 0;
2876
2877         $d->{diskread} = 0;
2878         $d->{diskwrite} = 0;
2879
2880         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2881
2882         $d->{serial} = 1 if conf_has_serial($conf);
2883         $d->{lock} = $conf->{lock} if $conf->{lock};
2884         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2885
2886         $res->{$vmid} = $d;
2887     }
2888
2889     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2890     foreach my $dev (keys %$netdev) {
2891         next if $dev !~ m/^tap([1-9]\d*)i/;
2892         my $vmid = $1;
2893         my $d = $res->{$vmid};
2894         next if !$d;
2895
2896         $d->{netout} += $netdev->{$dev}->{receive};
2897         $d->{netin} += $netdev->{$dev}->{transmit};
2898
2899         if ($full) {
2900             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2901             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2902         }
2903
2904     }
2905
2906     my $ctime = gettimeofday;
2907
2908     foreach my $vmid (keys %$list) {
2909
2910         my $d = $res->{$vmid};
2911         my $pid = $d->{pid};
2912         next if !$pid;
2913
2914         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2915         next if !$pstat; # not running
2916
2917         my $used = $pstat->{utime} + $pstat->{stime};
2918
2919         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2920
2921         if ($pstat->{vsize}) {
2922             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2923         }
2924
2925         my $old = $last_proc_pid_stat->{$pid};
2926         if (!$old) {
2927             $last_proc_pid_stat->{$pid} = {
2928                 time => $ctime,
2929                 used => $used,
2930                 cpu => 0,
2931             };
2932             next;
2933         }
2934
2935         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2936
2937         if ($dtime > 1000) {
2938             my $dutime = $used -  $old->{used};
2939
2940             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2941             $last_proc_pid_stat->{$pid} = {
2942                 time => $ctime,
2943                 used => $used,
2944                 cpu => $d->{cpu},
2945             };
2946         } else {
2947             $d->{cpu} = $old->{cpu};
2948         }
2949     }
2950
2951     return $res if !$full;
2952
2953     my $qmpclient = PVE::QMPClient->new();
2954
2955     my $ballooncb = sub {
2956         my ($vmid, $resp) = @_;
2957
2958         my $info = $resp->{'return'};
2959         return if !$info->{max_mem};
2960
2961         my $d = $res->{$vmid};
2962
2963         # use memory assigned to VM
2964         $d->{maxmem} = $info->{max_mem};
2965         $d->{balloon} = $info->{actual};
2966
2967         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2968             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2969             $d->{freemem} = $info->{free_mem};
2970         }
2971
2972         $d->{ballooninfo} = $info;
2973     };
2974
2975     my $blockstatscb = sub {
2976         my ($vmid, $resp) = @_;
2977         my $data = $resp->{'return'} || [];
2978         my $totalrdbytes = 0;
2979         my $totalwrbytes = 0;
2980
2981         for my $blockstat (@$data) {
2982             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2983             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2984
2985             $blockstat->{device} =~ s/drive-//;
2986             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2987         }
2988         $res->{$vmid}->{diskread} = $totalrdbytes;
2989         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2990     };
2991
2992     my $machinecb = sub {
2993         my ($vmid, $resp) = @_;
2994         my $data = $resp->{'return'} || [];
2995
2996         $res->{$vmid}->{'running-machine'} =
2997             PVE::QemuServer::Machine::current_from_query_machines($data);
2998     };
2999
3000     my $versioncb = sub {
3001         my ($vmid, $resp) = @_;
3002         my $data = $resp->{'return'} // {};
3003         my $version = 'unknown';
3004
3005         if (my $v = $data->{qemu}) {
3006             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3007         }
3008
3009         $res->{$vmid}->{'running-qemu'} = $version;
3010     };
3011
3012     my $statuscb = sub {
3013         my ($vmid, $resp) = @_;
3014
3015         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3016         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3017         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3018         # this fails if ballon driver is not loaded, so this must be
3019         # the last commnand (following command are aborted if this fails).
3020         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3021
3022         my $status = 'unknown';
3023         if (!defined($status = $resp->{'return'}->{status})) {
3024             warn "unable to get VM status\n";
3025             return;
3026         }
3027
3028         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3029     };
3030
3031     foreach my $vmid (keys %$list) {
3032         next if $opt_vmid && ($vmid ne $opt_vmid);
3033         next if !$res->{$vmid}->{pid}; # not running
3034         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3035     }
3036
3037     $qmpclient->queue_execute(undef, 2);
3038
3039     foreach my $vmid (keys %$list) {
3040         next if $opt_vmid && ($vmid ne $opt_vmid);
3041         next if !$res->{$vmid}->{pid}; #not running
3042
3043         # we can't use the $qmpclient since it might have already aborted on
3044         # 'query-balloon', but this might also fail for older versions...
3045         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3046         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3047     }
3048
3049     foreach my $vmid (keys %$list) {
3050         next if $opt_vmid && ($vmid ne $opt_vmid);
3051         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3052     }
3053
3054     return $res;
3055 }
3056
3057 sub conf_has_serial {
3058     my ($conf) = @_;
3059
3060     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3061         if ($conf->{"serial$i"}) {
3062             return 1;
3063         }
3064     }
3065
3066     return 0;
3067 }
3068
3069 sub conf_has_audio {
3070     my ($conf, $id) = @_;
3071
3072     $id //= 0;
3073     my $audio = $conf->{"audio$id"};
3074     return if !defined($audio);
3075
3076     my $audioproperties = parse_property_string($audio_fmt, $audio);
3077     my $audiodriver = $audioproperties->{driver} // 'spice';
3078
3079     return {
3080         dev => $audioproperties->{device},
3081         dev_id => "audiodev$id",
3082         backend => $audiodriver,
3083         backend_id => "$audiodriver-backend${id}",
3084     };
3085 }
3086
3087 sub audio_devs {
3088     my ($audio, $audiopciaddr, $machine_version) = @_;
3089
3090     my $devs = [];
3091
3092     my $id = $audio->{dev_id};
3093     my $audiodev = "";
3094     if (min_version($machine_version, 4, 2)) {
3095         $audiodev = ",audiodev=$audio->{backend_id}";
3096     }
3097
3098     if ($audio->{dev} eq 'AC97') {
3099         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3100     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3101         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3102         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3103         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3104     } else {
3105         die "unkown audio device '$audio->{dev}', implement me!";
3106     }
3107
3108     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3109
3110     return $devs;
3111 }
3112
3113 sub get_tpm_paths {
3114     my ($vmid) = @_;
3115     return {
3116         socket => "/var/run/qemu-server/$vmid.swtpm",
3117         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3118     };
3119 }
3120
3121 sub add_tpm_device {
3122     my ($vmid, $devices, $conf) = @_;
3123
3124     return if !$conf->{tpmstate0};
3125
3126     my $paths = get_tpm_paths($vmid);
3127
3128     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3129     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3130     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3131 }
3132
3133 sub start_swtpm {
3134     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3135
3136     return if !$tpmdrive;
3137
3138     my $state;
3139     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3140     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3141     if ($storeid) {
3142         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3143     } else {
3144         $state = $tpm->{file};
3145     }
3146
3147     my $paths = get_tpm_paths($vmid);
3148
3149     # during migration, we will get state from remote
3150     #
3151     if (!$migration) {
3152         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3153         my $setup_cmd = [
3154             "swtpm_setup",
3155             "--tpmstate",
3156             "file://$state",
3157             "--createek",
3158             "--create-ek-cert",
3159             "--create-platform-cert",
3160             "--lock-nvram",
3161             "--config",
3162             "/etc/swtpm_setup.conf", # do not use XDG configs
3163             "--runas",
3164             "0", # force creation as root, error if not possible
3165             "--not-overwrite", # ignore existing state, do not modify
3166         ];
3167
3168         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3169         # TPM 2.0 supports ECC crypto, use if possible
3170         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3171
3172         run_command($setup_cmd, outfunc => sub {
3173             print "swtpm_setup: $1\n";
3174         });
3175     }
3176
3177     my $emulator_cmd = [
3178         "swtpm",
3179         "socket",
3180         "--tpmstate",
3181         "backend-uri=file://$state,mode=0600",
3182         "--ctrl",
3183         "type=unixio,path=$paths->{socket},mode=0600",
3184         "--pid",
3185         "file=$paths->{pid}",
3186         "--terminate", # terminate on QEMU disconnect
3187         "--daemon",
3188     ];
3189     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3190     run_command($emulator_cmd, outfunc => sub { print $1; });
3191
3192     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3193     while (! -e $paths->{pid}) {
3194         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3195         usleep(50_000);
3196     }
3197
3198     # return untainted PID of swtpm daemon so it can be killed on error
3199     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3200     return $1;
3201 }
3202
3203 sub vga_conf_has_spice {
3204     my ($vga) = @_;
3205
3206     my $vgaconf = parse_vga($vga);
3207     my $vgatype = $vgaconf->{type};
3208     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3209
3210     return $1 || 1;
3211 }
3212
3213 sub is_native($) {
3214     my ($arch) = @_;
3215     return get_host_arch() eq $arch;
3216 }
3217
3218 sub get_vm_arch {
3219     my ($conf) = @_;
3220     return $conf->{arch} // get_host_arch();
3221 }
3222
3223 my $default_machines = {
3224     x86_64 => 'pc',
3225     aarch64 => 'virt',
3226 };
3227
3228 sub get_installed_machine_version {
3229     my ($kvmversion) = @_;
3230     $kvmversion = kvm_user_version() if !defined($kvmversion);
3231     $kvmversion =~ m/^(\d+\.\d+)/;
3232     return $1;
3233 }
3234
3235 sub windows_get_pinned_machine_version {
3236     my ($machine, $base_version, $kvmversion) = @_;
3237
3238     my $pin_version = $base_version;
3239     if (!defined($base_version) ||
3240         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3241     ) {
3242         $pin_version = get_installed_machine_version($kvmversion);
3243     }
3244     if (!$machine || $machine eq 'pc') {
3245         $machine = "pc-i440fx-$pin_version";
3246     } elsif ($machine eq 'q35') {
3247         $machine = "pc-q35-$pin_version";
3248     } elsif ($machine eq 'virt') {
3249         $machine = "virt-$pin_version";
3250     } else {
3251         warn "unknown machine type '$machine', not touching that!\n";
3252     }
3253
3254     return $machine;
3255 }
3256
3257 sub get_vm_machine {
3258     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3259
3260     my $machine = $forcemachine || $conf->{machine};
3261
3262     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3263         $kvmversion //= kvm_user_version();
3264         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3265         # layout which confuses windows quite a bit and may result in various regressions..
3266         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3267         if (windows_version($conf->{ostype})) {
3268             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3269         }
3270         $arch //= 'x86_64';
3271         $machine ||= $default_machines->{$arch};
3272         if ($add_pve_version) {
3273             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3274             $machine .= "+pve$pvever";
3275         }
3276     }
3277
3278     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3279         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3280         $machine = $1 if $is_pxe;
3281
3282         # for version-pinned machines that do not include a pve-version (e.g.
3283         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3284         $machine .= '+pve0';
3285
3286         $machine .= '.pxe' if $is_pxe;
3287     }
3288
3289     return $machine;
3290 }
3291
3292 sub get_ovmf_files($$$) {
3293     my ($arch, $efidisk, $smm) = @_;
3294
3295     my $types = $OVMF->{$arch}
3296         or die "no OVMF images known for architecture '$arch'\n";
3297
3298     my $type = 'default';
3299     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3300         $type = $smm ? "4m" : "4m-no-smm";
3301         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3302     }
3303
3304     return $types->{$type}->@*;
3305 }
3306
3307 my $Arch2Qemu = {
3308     aarch64 => '/usr/bin/qemu-system-aarch64',
3309     x86_64 => '/usr/bin/qemu-system-x86_64',
3310 };
3311 sub get_command_for_arch($) {
3312     my ($arch) = @_;
3313     return '/usr/bin/kvm' if is_native($arch);
3314
3315     my $cmd = $Arch2Qemu->{$arch}
3316         or die "don't know how to emulate architecture '$arch'\n";
3317     return $cmd;
3318 }
3319
3320 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3321 # to use in a QEMU command line (-cpu element), first array_intersect the result
3322 # of query_supported_ with query_understood_. This is necessary because:
3323 #
3324 # a) query_understood_ returns flags the host cannot use and
3325 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3326 #    flags, but CPU settings - with most of them being flags. Those settings
3327 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3328 #
3329 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3330 # expensive. If you need the value returned from this, you can get it much
3331 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3332 # $accel being 'kvm' or 'tcg'.
3333 #
3334 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3335 # changes, automatically populating pmxcfs.
3336 #
3337 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3338 # since kvm and tcg machines support different flags
3339 #
3340 sub query_supported_cpu_flags {
3341     my ($arch) = @_;
3342
3343     $arch //= get_host_arch();
3344     my $default_machine = $default_machines->{$arch};
3345
3346     my $flags = {};
3347
3348     # FIXME: Once this is merged, the code below should work for ARM as well:
3349     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3350     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3351         $arch eq "aarch64";
3352
3353     my $kvm_supported = defined(kvm_version());
3354     my $qemu_cmd = get_command_for_arch($arch);
3355     my $fakevmid = -1;
3356     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3357
3358     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3359     my $query_supported_run_qemu = sub {
3360         my ($kvm) = @_;
3361
3362         my $flags = {};
3363         my $cmd = [
3364             $qemu_cmd,
3365             '-machine', $default_machine,
3366             '-display', 'none',
3367             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3368             '-mon', 'chardev=qmp,mode=control',
3369             '-pidfile', $pidfile,
3370             '-S', '-daemonize'
3371         ];
3372
3373         if (!$kvm) {
3374             push @$cmd, '-accel', 'tcg';
3375         }
3376
3377         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3378         die "QEMU flag querying VM exited with code " . $rc if $rc;
3379
3380         eval {
3381             my $cmd_result = mon_cmd(
3382                 $fakevmid,
3383                 'query-cpu-model-expansion',
3384                 type => 'full',
3385                 model => { name => 'host' }
3386             );
3387
3388             my $props = $cmd_result->{model}->{props};
3389             foreach my $prop (keys %$props) {
3390                 next if $props->{$prop} ne '1';
3391                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3392                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3393                 # We only keep those with underscores, to match /proc/cpuinfo
3394                 $prop =~ s/\.|-/_/g;
3395                 $flags->{$prop} = 1;
3396             }
3397         };
3398         my $err = $@;
3399
3400         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3401         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3402
3403         die $err if $err;
3404
3405         return [ sort keys %$flags ];
3406     };
3407
3408     # We need to query QEMU twice, since KVM and TCG have different supported flags
3409     PVE::QemuConfig->lock_config($fakevmid, sub {
3410         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3411         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3412
3413         if ($kvm_supported) {
3414             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3415             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3416         }
3417     });
3418
3419     return $flags;
3420 }
3421
3422 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3423 my $understood_cpu_flag_dir = "/usr/share/kvm";
3424 sub query_understood_cpu_flags {
3425     my $arch = get_host_arch();
3426     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3427
3428     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3429         if ! -e $filepath;
3430
3431     my $raw = file_get_contents($filepath);
3432     $raw =~ s/^\s+|\s+$//g;
3433     my @flags = split(/\s+/, $raw);
3434
3435     return \@flags;
3436 }
3437
3438 my sub get_cpuunits {
3439     my ($conf) = @_;
3440     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3441
3442     my $cpuunits = $conf->{cpuunits};
3443     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3444
3445     if ($is_cgroupv2) {
3446         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3447     } else {
3448         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3449     }
3450     return $cpuunits;
3451 }
3452
3453 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3454 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3455 my sub should_disable_smm {
3456     my ($conf, $vga) = @_;
3457
3458     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3459         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3460 }
3461
3462 sub config_to_command {
3463     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3464         $pbs_backing) = @_;
3465
3466     my $cmd = [];
3467     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3468     my $devices = [];
3469     my $bridges = {};
3470     my $ostype = $conf->{ostype};
3471     my $winversion = windows_version($ostype);
3472     my $kvm = $conf->{kvm};
3473     my $nodename = nodename();
3474
3475     my $arch = get_vm_arch($conf);
3476     my $kvm_binary = get_command_for_arch($arch);
3477     my $kvmver = kvm_user_version($kvm_binary);
3478
3479     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3480         $kvmver //= "undefined";
3481         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3482     }
3483
3484     my $add_pve_version = min_version($kvmver, 4, 1);
3485
3486     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3487     my $machine_version = extract_version($machine_type, $kvmver);
3488     $kvm //= 1 if is_native($arch);
3489
3490     $machine_version =~ m/(\d+)\.(\d+)/;
3491     my ($machine_major, $machine_minor) = ($1, $2);
3492
3493     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3494         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3495     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3496         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3497             ." please upgrade node '$nodename'\n"
3498     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3499         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3500         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3501             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3502             ." node '$nodename'\n";
3503     }
3504
3505     # if a specific +pve version is required for a feature, use $version_guard
3506     # instead of min_version to allow machines to be run with the minimum
3507     # required version
3508     my $required_pve_version = 0;
3509     my $version_guard = sub {
3510         my ($major, $minor, $pve) = @_;
3511         return 0 if !min_version($machine_version, $major, $minor, $pve);
3512         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3513         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3514         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3515         return 1;
3516     };
3517
3518     if ($kvm && !defined kvm_version()) {
3519         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3520             ." or enable in BIOS.\n";
3521     }
3522
3523     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3524     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3525     my $use_old_bios_files = undef;
3526     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3527
3528     push @$cmd, $kvm_binary;
3529
3530     push @$cmd, '-id', $vmid;
3531
3532     my $vmname = $conf->{name} || "vm$vmid";
3533
3534     push @$cmd, '-name', $vmname;
3535
3536     push @$cmd, '-no-shutdown';
3537
3538     my $use_virtio = 0;
3539
3540     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3541     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3542     push @$cmd, '-mon', "chardev=qmp,mode=control";
3543
3544     if (min_version($machine_version, 2, 12)) {
3545         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3546         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3547     }
3548
3549     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3550
3551     push @$cmd, '-daemonize';
3552
3553     if ($conf->{smbios1}) {
3554         my $smbios_conf = parse_smbios1($conf->{smbios1});
3555         if ($smbios_conf->{base64}) {
3556             # Do not pass base64 flag to qemu
3557             delete $smbios_conf->{base64};
3558             my $smbios_string = "";
3559             foreach my $key (keys %$smbios_conf) {
3560                 my $value;
3561                 if ($key eq "uuid") {
3562                     $value = $smbios_conf->{uuid}
3563                 } else {
3564                     $value = decode_base64($smbios_conf->{$key});
3565                 }
3566                 # qemu accepts any binary data, only commas need escaping by double comma
3567                 $value =~ s/,/,,/g;
3568                 $smbios_string .= "," . $key . "=" . $value if $value;
3569             }
3570             push @$cmd, '-smbios', "type=1" . $smbios_string;
3571         } else {
3572             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3573         }
3574     }
3575
3576     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3577         my $d;
3578         if (my $efidisk = $conf->{efidisk0}) {
3579             $d = parse_drive('efidisk0', $efidisk);
3580         }
3581
3582         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3583         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3584
3585         my ($path, $format);
3586         my $read_only_str = '';
3587         if ($d) {
3588             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3589             $format = $d->{format};
3590             if ($storeid) {
3591                 $path = PVE::Storage::path($storecfg, $d->{file});
3592                 if (!defined($format)) {
3593                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3594                     $format = qemu_img_format($scfg, $volname);
3595                 }
3596             } else {
3597                 $path = $d->{file};
3598                 die "efidisk format must be specified\n"
3599                     if !defined($format);
3600             }
3601
3602             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3603         } else {
3604             warn "no efidisk configured! Using temporary efivars disk.\n";
3605             $path = "/tmp/$vmid-ovmf.fd";
3606             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3607             $format = 'raw';
3608         }
3609
3610         my $size_str = "";
3611
3612         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3613             $size_str = ",size=" . (-s $ovmf_vars);
3614         }
3615
3616         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3617         my $cache = "";
3618         if ($path =~ m/^rbd:/) {
3619                 $cache = ',cache=writeback';
3620                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3621         }
3622
3623         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3624         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3625     }
3626
3627     if ($q35) { # tell QEMU to load q35 config early
3628         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3629         if (min_version($machine_version, 4, 0)) {
3630             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3631         } else {
3632             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3633         }
3634     }
3635
3636     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3637         push @$cmd, $fixups->@*;
3638     }
3639
3640     if ($conf->{vmgenid}) {
3641         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3642     }
3643
3644     # add usb controllers
3645     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3646         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3647     push @$devices, @usbcontrollers if @usbcontrollers;
3648     my $vga = parse_vga($conf->{vga});
3649
3650     my $qxlnum = vga_conf_has_spice($conf->{vga});
3651     $vga->{type} = 'qxl' if $qxlnum;
3652
3653     if (!$vga->{type}) {
3654         if ($arch eq 'aarch64') {
3655             $vga->{type} = 'virtio';
3656         } elsif (min_version($machine_version, 2, 9)) {
3657             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3658         } else {
3659             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3660         }
3661     }
3662
3663     # enable absolute mouse coordinates (needed by vnc)
3664     my $tablet = $conf->{tablet};
3665     if (!defined($tablet)) {
3666         $tablet = $defaults->{tablet};
3667         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3668         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3669     }
3670
3671     if ($tablet) {
3672         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3673         my $kbd = print_keyboarddevice_full($conf, $arch);
3674         push @$devices, '-device', $kbd if defined($kbd);
3675     }
3676
3677     my $bootorder = device_bootorder($conf);
3678
3679     # host pci device passthrough
3680     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3681         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3682
3683     # usb devices
3684     my $usb_dev_features = {};
3685     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3686
3687     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3688         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3689     push @$devices, @usbdevices if @usbdevices;
3690
3691     # serial devices
3692     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3693         my $path = $conf->{"serial$i"} or next;
3694         if ($path eq 'socket') {
3695             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3696             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3697             # On aarch64, serial0 is the UART device. Qemu only allows
3698             # connecting UART devices via the '-serial' command line, as
3699             # the device has a fixed slot on the hardware...
3700             if ($arch eq 'aarch64' && $i == 0) {
3701                 push @$devices, '-serial', "chardev:serial$i";
3702             } else {
3703                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3704             }
3705         } else {
3706             die "no such serial device\n" if ! -c $path;
3707             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3708             push @$devices, '-device', "isa-serial,chardev=serial$i";
3709         }
3710     }
3711
3712     # parallel devices
3713     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3714         if (my $path = $conf->{"parallel$i"}) {
3715             die "no such parallel device\n" if ! -c $path;
3716             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3717             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3718             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3719         }
3720     }
3721
3722     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3723         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3724         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3725         push @$devices, @$audio_devs;
3726     }
3727
3728     add_tpm_device($vmid, $devices, $conf);
3729
3730     my $sockets = 1;
3731     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3732     $sockets = $conf->{sockets} if  $conf->{sockets};
3733
3734     my $cores = $conf->{cores} || 1;
3735
3736     my $maxcpus = $sockets * $cores;
3737
3738     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3739
3740     my $allowed_vcpus = $cpuinfo->{cpus};
3741
3742     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3743
3744     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3745         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3746         for (my $i = 2; $i <= $vcpus; $i++)  {
3747             my $cpustr = print_cpu_device($conf,$i);
3748             push @$cmd, '-device', $cpustr;
3749         }
3750
3751     } else {
3752
3753         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3754     }
3755     push @$cmd, '-nodefaults';
3756
3757     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3758
3759     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3760
3761     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3762
3763     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3764         push @$devices, '-device', print_vga_device(
3765             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3766
3767         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3768
3769         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3770         push @$cmd,  '-vnc', "unix:$socket,password=on";
3771     } else {
3772         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3773         push @$cmd, '-nographic';
3774     }
3775
3776     # time drift fix
3777     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3778     my $useLocaltime = $conf->{localtime};
3779
3780     if ($winversion >= 5) { # windows
3781         $useLocaltime = 1 if !defined($conf->{localtime});
3782
3783         # use time drift fix when acpi is enabled
3784         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3785             $tdf = 1 if !defined($conf->{tdf});
3786         }
3787     }
3788
3789     if ($winversion >= 6) {
3790         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3791         push @$cmd, '-no-hpet';
3792     }
3793
3794     push @$rtcFlags, 'driftfix=slew' if $tdf;
3795
3796     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3797         push @$rtcFlags, "base=$conf->{startdate}";
3798     } elsif ($useLocaltime) {
3799         push @$rtcFlags, 'base=localtime';
3800     }
3801
3802     if ($forcecpu) {
3803         push @$cmd, '-cpu', $forcecpu;
3804     } else {
3805         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3806     }
3807
3808     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3809
3810     push @$cmd, '-S' if $conf->{freeze};
3811
3812     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3813
3814     my $guest_agent = parse_guest_agent($conf);
3815
3816     if ($guest_agent->{enabled}) {
3817         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3818         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3819
3820         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3821             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3822             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3823             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3824         } elsif ($guest_agent->{type} eq 'isa') {
3825             push @$devices, '-device', "isa-serial,chardev=qga0";
3826         }
3827     }
3828
3829     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3830     if ($rng && $version_guard->(4, 1, 2)) {
3831         check_rng_source($rng->{source});
3832
3833         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3834         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3835         my $limiter_str = "";
3836         if ($max_bytes) {
3837             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3838         }
3839
3840         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3841         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3842         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3843     }
3844
3845     my $spice_port;
3846
3847     if ($qxlnum) {
3848         if ($qxlnum > 1) {
3849             if ($winversion){
3850                 for (my $i = 1; $i < $qxlnum; $i++){
3851                     push @$devices, '-device', print_vga_device(
3852                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3853                 }
3854             } else {
3855                 # assume other OS works like Linux
3856                 my ($ram, $vram) = ("134217728", "67108864");
3857                 if ($vga->{memory}) {
3858                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3859                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3860                 }
3861                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3862                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3863             }
3864         }
3865
3866         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3867
3868         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3869         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3870         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3871
3872         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3873         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3874         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3875
3876         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3877         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3878
3879         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3880         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3881         if ($spice_enhancement->{foldersharing}) {
3882             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3883             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3884         }
3885
3886         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3887         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3888             if $spice_enhancement->{videostreaming};
3889
3890         push @$devices, '-spice', "$spice_opts";
3891     }
3892
3893     # enable balloon by default, unless explicitly disabled
3894     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3895         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3896         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3897     }
3898
3899     if ($conf->{watchdog}) {
3900         my $wdopts = parse_watchdog($conf->{watchdog});
3901         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3902         my $watchdog = $wdopts->{model} || 'i6300esb';
3903         push @$devices, '-device', "$watchdog$pciaddr";
3904         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3905     }
3906
3907     my $vollist = [];
3908     my $scsicontroller = {};
3909     my $ahcicontroller = {};
3910     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3911
3912     # Add iscsi initiator name if available
3913     if (my $initiator = get_initiator_name()) {
3914         push @$devices, '-iscsi', "initiator-name=$initiator";
3915     }
3916
3917     PVE::QemuConfig->foreach_volume($conf, sub {
3918         my ($ds, $drive) = @_;
3919
3920         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3921             check_volume_storage_type($storecfg, $drive->{file});
3922             push @$vollist, $drive->{file};
3923         }
3924
3925         # ignore efidisk here, already added in bios/fw handling code above
3926         return if $drive->{interface} eq 'efidisk';
3927         # similar for TPM
3928         return if $drive->{interface} eq 'tpmstate';
3929
3930         $use_virtio = 1 if $ds =~ m/^virtio/;
3931
3932         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3933
3934         if ($drive->{interface} eq 'virtio'){
3935            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3936         }
3937
3938         if ($drive->{interface} eq 'scsi') {
3939
3940             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3941
3942             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3943                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3944
3945             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3946             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3947
3948             my $iothread = '';
3949             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3950                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3951                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3952             } elsif ($drive->{iothread}) {
3953                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3954             }
3955
3956             my $queues = '';
3957             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3958                 $queues = ",num_queues=$drive->{queues}";
3959             }
3960
3961             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3962                 if !$scsicontroller->{$controller};
3963             $scsicontroller->{$controller}=1;
3964         }
3965
3966         if ($drive->{interface} eq 'sata') {
3967             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3968             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3969             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3970                 if !$ahcicontroller->{$controller};
3971             $ahcicontroller->{$controller}=1;
3972         }
3973
3974         my $pbs_conf = $pbs_backing->{$ds};
3975         my $pbs_name = undef;
3976         if ($pbs_conf) {
3977             $pbs_name = "drive-$ds-pbs";
3978             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3979         }
3980
3981         my $drive_cmd = print_drive_commandline_full(
3982             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3983
3984         # extra protection for templates, but SATA and IDE don't support it..
3985         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3986
3987         push @$devices, '-drive',$drive_cmd;
3988         push @$devices, '-device', print_drivedevice_full(
3989             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3990     });
3991
3992     for (my $i = 0; $i < $MAX_NETS; $i++) {
3993         my $netname = "net$i";
3994
3995         next if !$conf->{$netname};
3996         my $d = parse_net($conf->{$netname});
3997         next if !$d;
3998
3999         $use_virtio = 1 if $d->{model} eq 'virtio';
4000
4001         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4002
4003         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4004         push @$devices, '-netdev', $netdevfull;
4005
4006         my $netdevicefull = print_netdevice_full(
4007             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4008
4009         push @$devices, '-device', $netdevicefull;
4010     }
4011
4012     if ($conf->{ivshmem}) {
4013         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4014
4015         my $bus;
4016         if ($q35) {
4017             $bus = print_pcie_addr("ivshmem");
4018         } else {
4019             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4020         }
4021
4022         my $ivshmem_name = $ivshmem->{name} // $vmid;
4023         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4024
4025         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4026         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4027             .",size=$ivshmem->{size}M";
4028     }
4029
4030     # pci.4 is nested in pci.1
4031     $bridges->{1} = 1 if $bridges->{4};
4032
4033     if (!$q35) { # add pci bridges
4034         if (min_version($machine_version, 2, 3)) {
4035            $bridges->{1} = 1;
4036            $bridges->{2} = 1;
4037         }
4038         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4039     }
4040
4041     for my $k (sort {$b cmp $a} keys %$bridges) {
4042         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4043
4044         my $k_name = $k;
4045         if ($k == 2 && $legacy_igd) {
4046             $k_name = "$k-igd";
4047         }
4048         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4049         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4050
4051         if ($q35) { # add after -readconfig pve-q35.cfg
4052             splice @$devices, 2, 0, '-device', $devstr;
4053         } else {
4054             unshift @$devices, '-device', $devstr if $k > 0;
4055         }
4056     }
4057
4058     if (!$kvm) {
4059         push @$machineFlags, 'accel=tcg';
4060     }
4061
4062     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4063
4064     my $machine_type_min = $machine_type;
4065     if ($add_pve_version) {
4066         $machine_type_min =~ s/\+pve\d+$//;
4067         $machine_type_min .= "+pve$required_pve_version";
4068     }
4069     push @$machineFlags, "type=${machine_type_min}";
4070
4071     push @$cmd, @$devices;
4072     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4073     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4074     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4075
4076     if (my $vmstate = $conf->{vmstate}) {
4077         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4078         push @$vollist, $vmstate;
4079         push @$cmd, '-loadstate', $statepath;
4080         print "activating and using '$vmstate' as vmstate\n";
4081     }
4082
4083     if (PVE::QemuConfig->is_template($conf)) {
4084         # needed to workaround base volumes being read-only
4085         push @$cmd, '-snapshot';
4086     }
4087
4088     # add custom args
4089     if ($conf->{args}) {
4090         my $aa = PVE::Tools::split_args($conf->{args});
4091         push @$cmd, @$aa;
4092     }
4093
4094     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4095 }
4096
4097 sub check_rng_source {
4098     my ($source) = @_;
4099
4100     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4101     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4102         if ! -e $source;
4103
4104     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4105     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4106         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4107         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4108         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4109             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4110             ." to the host.\n";
4111     }
4112 }
4113
4114 sub spice_port {
4115     my ($vmid) = @_;
4116
4117     my $res = mon_cmd($vmid, 'query-spice');
4118
4119     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4120 }
4121
4122 sub vm_devices_list {
4123     my ($vmid) = @_;
4124
4125     my $res = mon_cmd($vmid, 'query-pci');
4126     my $devices_to_check = [];
4127     my $devices = {};
4128     foreach my $pcibus (@$res) {
4129         push @$devices_to_check, @{$pcibus->{devices}},
4130     }
4131
4132     while (@$devices_to_check) {
4133         my $to_check = [];
4134         for my $d (@$devices_to_check) {
4135             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4136             next if !$d->{'pci_bridge'};
4137
4138             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4139             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4140         }
4141         $devices_to_check = $to_check;
4142     }
4143
4144     my $resblock = mon_cmd($vmid, 'query-block');
4145     foreach my $block (@$resblock) {
4146         if($block->{device} =~ m/^drive-(\S+)/){
4147                 $devices->{$1} = 1;
4148         }
4149     }
4150
4151     my $resmice = mon_cmd($vmid, 'query-mice');
4152     foreach my $mice (@$resmice) {
4153         if ($mice->{name} eq 'QEMU HID Tablet') {
4154             $devices->{tablet} = 1;
4155             last;
4156         }
4157     }
4158
4159     # for usb devices there is no query-usb
4160     # but we can iterate over the entries in
4161     # qom-list path=/machine/peripheral
4162     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4163     foreach my $per (@$resperipheral) {
4164         if ($per->{name} =~ m/^usb\d+$/) {
4165             $devices->{$per->{name}} = 1;
4166         }
4167     }
4168
4169     return $devices;
4170 }
4171
4172 sub vm_deviceplug {
4173     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4174
4175     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4176
4177     my $devices_list = vm_devices_list($vmid);
4178     return 1 if defined($devices_list->{$deviceid});
4179
4180     # add PCI bridge if we need it for the device
4181     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4182
4183     if ($deviceid eq 'tablet') {
4184         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4185     } elsif ($deviceid eq 'keyboard') {
4186         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4187     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4188         die "usb hotplug currently not reliable\n";
4189         # since we can't reliably hot unplug all added usb devices and usb
4190         # passthrough breaks live migration we disable usb hotplugging for now
4191         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4192     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4193         qemu_iothread_add($vmid, $deviceid, $device);
4194
4195         qemu_driveadd($storecfg, $vmid, $device);
4196         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4197
4198         qemu_deviceadd($vmid, $devicefull);
4199         eval { qemu_deviceaddverify($vmid, $deviceid); };
4200         if (my $err = $@) {
4201             eval { qemu_drivedel($vmid, $deviceid); };
4202             warn $@ if $@;
4203             die $err;
4204         }
4205     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4206         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4207         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4208         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4209
4210         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4211
4212         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4213             qemu_iothread_add($vmid, $deviceid, $device);
4214             $devicefull .= ",iothread=iothread-$deviceid";
4215         }
4216
4217         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4218             $devicefull .= ",num_queues=$device->{queues}";
4219         }
4220
4221         qemu_deviceadd($vmid, $devicefull);
4222         qemu_deviceaddverify($vmid, $deviceid);
4223     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4224         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4225         qemu_driveadd($storecfg, $vmid, $device);
4226
4227         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4228         eval { qemu_deviceadd($vmid, $devicefull); };
4229         if (my $err = $@) {
4230             eval { qemu_drivedel($vmid, $deviceid); };
4231             warn $@ if $@;
4232             die $err;
4233         }
4234     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4235         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4236
4237         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4238         my $use_old_bios_files = undef;
4239         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4240
4241         my $netdevicefull = print_netdevice_full(
4242             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4243         qemu_deviceadd($vmid, $netdevicefull);
4244         eval {
4245             qemu_deviceaddverify($vmid, $deviceid);
4246             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4247         };
4248         if (my $err = $@) {
4249             eval { qemu_netdevdel($vmid, $deviceid); };
4250             warn $@ if $@;
4251             die $err;
4252         }
4253     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4254         my $bridgeid = $2;
4255         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4256         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4257
4258         qemu_deviceadd($vmid, $devicefull);
4259         qemu_deviceaddverify($vmid, $deviceid);
4260     } else {
4261         die "can't hotplug device '$deviceid'\n";
4262     }
4263
4264     return 1;
4265 }
4266
4267 # fixme: this should raise exceptions on error!
4268 sub vm_deviceunplug {
4269     my ($vmid, $conf, $deviceid) = @_;
4270
4271     my $devices_list = vm_devices_list($vmid);
4272     return 1 if !defined($devices_list->{$deviceid});
4273
4274     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4275     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4276
4277     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4278         qemu_devicedel($vmid, $deviceid);
4279     } elsif ($deviceid =~ m/^usb\d+$/) {
4280         die "usb hotplug currently not reliable\n";
4281         # when unplugging usb devices this way, there may be remaining usb
4282         # controllers/hubs so we disable it for now
4283         #qemu_devicedel($vmid, $deviceid);
4284         #qemu_devicedelverify($vmid, $deviceid);
4285     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4286         my $device = parse_drive($deviceid, $conf->{$deviceid});
4287
4288         qemu_devicedel($vmid, $deviceid);
4289         qemu_devicedelverify($vmid, $deviceid);
4290         qemu_drivedel($vmid, $deviceid);
4291         qemu_iothread_del($vmid, $deviceid, $device);
4292     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4293         qemu_devicedel($vmid, $deviceid);
4294         qemu_devicedelverify($vmid, $deviceid);
4295     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4296         my $device = parse_drive($deviceid, $conf->{$deviceid});
4297
4298         qemu_devicedel($vmid, $deviceid);
4299         qemu_devicedelverify($vmid, $deviceid);
4300         qemu_drivedel($vmid, $deviceid);
4301         qemu_deletescsihw($conf, $vmid, $deviceid);
4302
4303         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4304             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4305     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4306         qemu_devicedel($vmid, $deviceid);
4307         qemu_devicedelverify($vmid, $deviceid);
4308         qemu_netdevdel($vmid, $deviceid);
4309     } else {
4310         die "can't unplug device '$deviceid'\n";
4311     }
4312
4313     return 1;
4314 }
4315
4316 sub qemu_deviceadd {
4317     my ($vmid, $devicefull) = @_;
4318
4319     $devicefull = "driver=".$devicefull;
4320     my %options =  split(/[=,]/, $devicefull);
4321
4322     mon_cmd($vmid, "device_add" , %options);
4323 }
4324
4325 sub qemu_devicedel {
4326     my ($vmid, $deviceid) = @_;
4327
4328     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4329 }
4330
4331 sub qemu_iothread_add {
4332     my ($vmid, $deviceid, $device) = @_;
4333
4334     if ($device->{iothread}) {
4335         my $iothreads = vm_iothreads_list($vmid);
4336         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4337     }
4338 }
4339
4340 sub qemu_iothread_del {
4341     my ($vmid, $deviceid, $device) = @_;
4342
4343     if ($device->{iothread}) {
4344         my $iothreads = vm_iothreads_list($vmid);
4345         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4346     }
4347 }
4348
4349 sub qemu_objectadd {
4350     my ($vmid, $objectid, $qomtype) = @_;
4351
4352     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4353
4354     return 1;
4355 }
4356
4357 sub qemu_objectdel {
4358     my ($vmid, $objectid) = @_;
4359
4360     mon_cmd($vmid, "object-del", id => $objectid);
4361
4362     return 1;
4363 }
4364
4365 sub qemu_driveadd {
4366     my ($storecfg, $vmid, $device) = @_;
4367
4368     my $kvmver = get_running_qemu_version($vmid);
4369     my $io_uring = min_version($kvmver, 6, 0);
4370     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4371     $drive =~ s/\\/\\\\/g;
4372     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4373
4374     # If the command succeeds qemu prints: "OK"
4375     return 1 if $ret =~ m/OK/s;
4376
4377     die "adding drive failed: $ret\n";
4378 }
4379
4380 sub qemu_drivedel {
4381     my ($vmid, $deviceid) = @_;
4382
4383     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4384     $ret =~ s/^\s+//;
4385
4386     return 1 if $ret eq "";
4387
4388     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4389     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4390
4391     die "deleting drive $deviceid failed : $ret\n";
4392 }
4393
4394 sub qemu_deviceaddverify {
4395     my ($vmid, $deviceid) = @_;
4396
4397     for (my $i = 0; $i <= 5; $i++) {
4398          my $devices_list = vm_devices_list($vmid);
4399          return 1 if defined($devices_list->{$deviceid});
4400          sleep 1;
4401     }
4402
4403     die "error on hotplug device '$deviceid'\n";
4404 }
4405
4406
4407 sub qemu_devicedelverify {
4408     my ($vmid, $deviceid) = @_;
4409
4410     # need to verify that the device is correctly removed as device_del
4411     # is async and empty return is not reliable
4412
4413     for (my $i = 0; $i <= 5; $i++) {
4414          my $devices_list = vm_devices_list($vmid);
4415          return 1 if !defined($devices_list->{$deviceid});
4416          sleep 1;
4417     }
4418
4419     die "error on hot-unplugging device '$deviceid'\n";
4420 }
4421
4422 sub qemu_findorcreatescsihw {
4423     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4424
4425     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4426
4427     my $scsihwid="$controller_prefix$controller";
4428     my $devices_list = vm_devices_list($vmid);
4429
4430     if (!defined($devices_list->{$scsihwid})) {
4431         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4432     }
4433
4434     return 1;
4435 }
4436
4437 sub qemu_deletescsihw {
4438     my ($conf, $vmid, $opt) = @_;
4439
4440     my $device = parse_drive($opt, $conf->{$opt});
4441
4442     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4443         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4444         return 1;
4445     }
4446
4447     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4448
4449     my $devices_list = vm_devices_list($vmid);
4450     foreach my $opt (keys %{$devices_list}) {
4451         if (is_valid_drivename($opt)) {
4452             my $drive = parse_drive($opt, $conf->{$opt});
4453             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4454                 return 1;
4455             }
4456         }
4457     }
4458
4459     my $scsihwid="scsihw$controller";
4460
4461     vm_deviceunplug($vmid, $conf, $scsihwid);
4462
4463     return 1;
4464 }
4465
4466 sub qemu_add_pci_bridge {
4467     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4468
4469     my $bridges = {};
4470
4471     my $bridgeid;
4472
4473     print_pci_addr($device, $bridges, $arch, $machine_type);
4474
4475     while (my ($k, $v) = each %$bridges) {
4476         $bridgeid = $k;
4477     }
4478     return 1 if !defined($bridgeid) || $bridgeid < 1;
4479
4480     my $bridge = "pci.$bridgeid";
4481     my $devices_list = vm_devices_list($vmid);
4482
4483     if (!defined($devices_list->{$bridge})) {
4484         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4485     }
4486
4487     return 1;
4488 }
4489
4490 sub qemu_set_link_status {
4491     my ($vmid, $device, $up) = @_;
4492
4493     mon_cmd($vmid, "set_link", name => $device,
4494                up => $up ? JSON::true : JSON::false);
4495 }
4496
4497 sub qemu_netdevadd {
4498     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4499
4500     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4501     my %options =  split(/[=,]/, $netdev);
4502
4503     if (defined(my $vhost = $options{vhost})) {
4504         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4505     }
4506
4507     if (defined(my $queues = $options{queues})) {
4508         $options{queues} = $queues + 0;
4509     }
4510
4511     mon_cmd($vmid, "netdev_add",  %options);
4512     return 1;
4513 }
4514
4515 sub qemu_netdevdel {
4516     my ($vmid, $deviceid) = @_;
4517
4518     mon_cmd($vmid, "netdev_del", id => $deviceid);
4519 }
4520
4521 sub qemu_usb_hotplug {
4522     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4523
4524     return if !$device;
4525
4526     # remove the old one first
4527     vm_deviceunplug($vmid, $conf, $deviceid);
4528
4529     # check if xhci controller is necessary and available
4530     if ($device->{usb3}) {
4531
4532         my $devicelist = vm_devices_list($vmid);
4533
4534         if (!$devicelist->{xhci}) {
4535             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4536             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4537         }
4538     }
4539     my $d = parse_usb_device($device->{host});
4540     $d->{usb3} = $device->{usb3};
4541
4542     # add the new one
4543     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4544 }
4545
4546 sub qemu_cpu_hotplug {
4547     my ($vmid, $conf, $vcpus) = @_;
4548
4549     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4550
4551     my $sockets = 1;
4552     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4553     $sockets = $conf->{sockets} if  $conf->{sockets};
4554     my $cores = $conf->{cores} || 1;
4555     my $maxcpus = $sockets * $cores;
4556
4557     $vcpus = $maxcpus if !$vcpus;
4558
4559     die "you can't add more vcpus than maxcpus\n"
4560         if $vcpus > $maxcpus;
4561
4562     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4563
4564     if ($vcpus < $currentvcpus) {
4565
4566         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4567
4568             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4569                 qemu_devicedel($vmid, "cpu$i");
4570                 my $retry = 0;
4571                 my $currentrunningvcpus = undef;
4572                 while (1) {
4573                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4574                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4575                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4576                     $retry++;
4577                     sleep 1;
4578                 }
4579                 #update conf after each succesfull cpu unplug
4580                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4581                 PVE::QemuConfig->write_config($vmid, $conf);
4582             }
4583         } else {
4584             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4585         }
4586
4587         return;
4588     }
4589
4590     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4591     die "vcpus in running vm does not match its configuration\n"
4592         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4593
4594     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4595
4596         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4597             my $cpustr = print_cpu_device($conf, $i);
4598             qemu_deviceadd($vmid, $cpustr);
4599
4600             my $retry = 0;
4601             my $currentrunningvcpus = undef;
4602             while (1) {
4603                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4604                 last if scalar(@{$currentrunningvcpus}) == $i;
4605                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4606                 sleep 1;
4607                 $retry++;
4608             }
4609             #update conf after each succesfull cpu hotplug
4610             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4611             PVE::QemuConfig->write_config($vmid, $conf);
4612         }
4613     } else {
4614
4615         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4616             mon_cmd($vmid, "cpu-add", id => int($i));
4617         }
4618     }
4619 }
4620
4621 sub qemu_block_set_io_throttle {
4622     my ($vmid, $deviceid,
4623         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4624         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4625         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4626         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4627
4628     return if !check_running($vmid) ;
4629
4630     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4631         bps => int($bps),
4632         bps_rd => int($bps_rd),
4633         bps_wr => int($bps_wr),
4634         iops => int($iops),
4635         iops_rd => int($iops_rd),
4636         iops_wr => int($iops_wr),
4637         bps_max => int($bps_max),
4638         bps_rd_max => int($bps_rd_max),
4639         bps_wr_max => int($bps_wr_max),
4640         iops_max => int($iops_max),
4641         iops_rd_max => int($iops_rd_max),
4642         iops_wr_max => int($iops_wr_max),
4643         bps_max_length => int($bps_max_length),
4644         bps_rd_max_length => int($bps_rd_max_length),
4645         bps_wr_max_length => int($bps_wr_max_length),
4646         iops_max_length => int($iops_max_length),
4647         iops_rd_max_length => int($iops_rd_max_length),
4648         iops_wr_max_length => int($iops_wr_max_length),
4649     );
4650
4651 }
4652
4653 sub qemu_block_resize {
4654     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4655
4656     my $running = check_running($vmid);
4657
4658     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4659
4660     return if !$running;
4661
4662     my $padding = (1024 - $size % 1024) % 1024;
4663     $size = $size + $padding;
4664
4665     mon_cmd(
4666         $vmid,
4667         "block_resize",
4668         device => $deviceid,
4669         size => int($size),
4670         timeout => 60,
4671     );
4672 }
4673
4674 sub qemu_volume_snapshot {
4675     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4676
4677     my $running = check_running($vmid);
4678
4679     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4680         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4681     } else {
4682         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4683     }
4684 }
4685
4686 sub qemu_volume_snapshot_delete {
4687     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4688
4689     my $running = check_running($vmid);
4690
4691     if($running) {
4692
4693         $running = undef;
4694         my $conf = PVE::QemuConfig->load_config($vmid);
4695         PVE::QemuConfig->foreach_volume($conf, sub {
4696             my ($ds, $drive) = @_;
4697             $running = 1 if $drive->{file} eq $volid;
4698         });
4699     }
4700
4701     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4702         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4703     } else {
4704         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4705     }
4706 }
4707
4708 sub set_migration_caps {
4709     my ($vmid, $savevm) = @_;
4710
4711     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4712
4713     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4714     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4715
4716     my $cap_ref = [];
4717
4718     my $enabled_cap = {
4719         "auto-converge" => 1,
4720         "xbzrle" => 1,
4721         "x-rdma-pin-all" => 0,
4722         "zero-blocks" => 0,
4723         "compress" => 0,
4724         "dirty-bitmaps" => $dirty_bitmaps,
4725     };
4726
4727     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4728
4729     for my $supported_capability (@$supported_capabilities) {
4730         push @$cap_ref, {
4731             capability => $supported_capability->{capability},
4732             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4733         };
4734     }
4735
4736     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4737 }
4738
4739 sub foreach_volid {
4740     my ($conf, $func, @param) = @_;
4741
4742     my $volhash = {};
4743
4744     my $test_volid = sub {
4745         my ($key, $drive, $snapname) = @_;
4746
4747         my $volid = $drive->{file};
4748         return if !$volid;
4749
4750         $volhash->{$volid}->{cdrom} //= 1;
4751         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4752
4753         my $replicate = $drive->{replicate} // 1;
4754         $volhash->{$volid}->{replicate} //= 0;
4755         $volhash->{$volid}->{replicate} = 1 if $replicate;
4756
4757         $volhash->{$volid}->{shared} //= 0;
4758         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4759
4760         $volhash->{$volid}->{referenced_in_config} //= 0;
4761         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4762
4763         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4764             if defined($snapname);
4765
4766         my $size = $drive->{size};
4767         $volhash->{$volid}->{size} //= $size if $size;
4768
4769         $volhash->{$volid}->{is_vmstate} //= 0;
4770         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4771
4772         $volhash->{$volid}->{is_tpmstate} //= 0;
4773         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4774
4775         $volhash->{$volid}->{is_unused} //= 0;
4776         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4777
4778         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4779     };
4780
4781     my $include_opts = {
4782         extra_keys => ['vmstate'],
4783         include_unused => 1,
4784     };
4785
4786     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4787     foreach my $snapname (keys %{$conf->{snapshots}}) {
4788         my $snap = $conf->{snapshots}->{$snapname};
4789         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4790     }
4791
4792     foreach my $volid (keys %$volhash) {
4793         &$func($volid, $volhash->{$volid}, @param);
4794     }
4795 }
4796
4797 my $fast_plug_option = {
4798     'lock' => 1,
4799     'name' => 1,
4800     'onboot' => 1,
4801     'shares' => 1,
4802     'startup' => 1,
4803     'description' => 1,
4804     'protection' => 1,
4805     'vmstatestorage' => 1,
4806     'hookscript' => 1,
4807     'tags' => 1,
4808 };
4809
4810 # hotplug changes in [PENDING]
4811 # $selection hash can be used to only apply specified options, for
4812 # example: { cores => 1 } (only apply changed 'cores')
4813 # $errors ref is used to return error messages
4814 sub vmconfig_hotplug_pending {
4815     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4816
4817     my $defaults = load_defaults();
4818     my $arch = get_vm_arch($conf);
4819     my $machine_type = get_vm_machine($conf, undef, $arch);
4820
4821     # commit values which do not have any impact on running VM first
4822     # Note: those option cannot raise errors, we we do not care about
4823     # $selection and always apply them.
4824
4825     my $add_error = sub {
4826         my ($opt, $msg) = @_;
4827         $errors->{$opt} = "hotplug problem - $msg";
4828     };
4829
4830     my $changes = 0;
4831     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4832         if ($fast_plug_option->{$opt}) {
4833             $conf->{$opt} = $conf->{pending}->{$opt};
4834             delete $conf->{pending}->{$opt};
4835             $changes = 1;
4836         }
4837     }
4838
4839     if ($changes) {
4840         PVE::QemuConfig->write_config($vmid, $conf);
4841     }
4842
4843     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4844
4845     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4846     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4847     foreach my $opt (sort keys %$pending_delete_hash) {
4848         next if $selection && !$selection->{$opt};
4849         my $force = $pending_delete_hash->{$opt}->{force};
4850         eval {
4851             if ($opt eq 'hotplug') {
4852                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4853             } elsif ($opt eq 'tablet') {
4854                 die "skip\n" if !$hotplug_features->{usb};
4855                 if ($defaults->{tablet}) {
4856                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4857                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4858                         if $arch eq 'aarch64';
4859                 } else {
4860                     vm_deviceunplug($vmid, $conf, 'tablet');
4861                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4862                 }
4863             } elsif ($opt =~ m/^usb\d+/) {
4864                 die "skip\n";
4865                 # since we cannot reliably hot unplug usb devices we are disabling it
4866                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4867                 #vm_deviceunplug($vmid, $conf, $opt);
4868             } elsif ($opt eq 'vcpus') {
4869                 die "skip\n" if !$hotplug_features->{cpu};
4870                 qemu_cpu_hotplug($vmid, $conf, undef);
4871             } elsif ($opt eq 'balloon') {
4872                 # enable balloon device is not hotpluggable
4873                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4874                 # here we reset the ballooning value to memory
4875                 my $balloon = $conf->{memory} || $defaults->{memory};
4876                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4877             } elsif ($fast_plug_option->{$opt}) {
4878                 # do nothing
4879             } elsif ($opt =~ m/^net(\d+)$/) {
4880                 die "skip\n" if !$hotplug_features->{network};
4881                 vm_deviceunplug($vmid, $conf, $opt);
4882             } elsif (is_valid_drivename($opt)) {
4883                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4884                 vm_deviceunplug($vmid, $conf, $opt);
4885                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4886             } elsif ($opt =~ m/^memory$/) {
4887                 die "skip\n" if !$hotplug_features->{memory};
4888                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4889             } elsif ($opt eq 'cpuunits') {
4890                 $cgroup->change_cpu_shares(undef, 1024);
4891             } elsif ($opt eq 'cpulimit') {
4892                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4893             } else {
4894                 die "skip\n";
4895             }
4896         };
4897         if (my $err = $@) {
4898             &$add_error($opt, $err) if $err ne "skip\n";
4899         } else {
4900             delete $conf->{$opt};
4901             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4902         }
4903     }
4904
4905     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4906     $apply_pending_cloudinit = sub {
4907         return if $apply_pending_cloudinit_done; # once is enough
4908         $apply_pending_cloudinit_done = 1; # once is enough
4909
4910         my ($key, $value) = @_;
4911
4912         my @cloudinit_opts = keys %$confdesc_cloudinit;
4913         foreach my $opt (keys %{$conf->{pending}}) {
4914             next if !grep { $_ eq $opt } @cloudinit_opts;
4915             $conf->{$opt} = delete $conf->{pending}->{$opt};
4916         }
4917
4918         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4919         foreach my $opt (sort keys %$pending_delete_hash) {
4920             next if !grep { $_ eq $opt } @cloudinit_opts;
4921             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4922             delete $conf->{$opt};
4923         }
4924
4925         my $new_conf = { %$conf };
4926         $new_conf->{$key} = $value;
4927         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4928     };
4929
4930     foreach my $opt (keys %{$conf->{pending}}) {
4931         next if $selection && !$selection->{$opt};
4932         my $value = $conf->{pending}->{$opt};
4933         eval {
4934             if ($opt eq 'hotplug') {
4935                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4936             } elsif ($opt eq 'tablet') {
4937                 die "skip\n" if !$hotplug_features->{usb};
4938                 if ($value == 1) {
4939                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4940                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4941                         if $arch eq 'aarch64';
4942                 } elsif ($value == 0) {
4943                     vm_deviceunplug($vmid, $conf, 'tablet');
4944                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4945                 }
4946             } elsif ($opt =~ m/^usb\d+$/) {
4947                 die "skip\n";
4948                 # since we cannot reliably hot unplug usb devices we disable it for now
4949                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4950                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4951                 #die "skip\n" if !$d;
4952                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4953             } elsif ($opt eq 'vcpus') {
4954                 die "skip\n" if !$hotplug_features->{cpu};
4955                 qemu_cpu_hotplug($vmid, $conf, $value);
4956             } elsif ($opt eq 'balloon') {
4957                 # enable/disable balloning device is not hotpluggable
4958                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4959                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4960                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4961
4962                 # allow manual ballooning if shares is set to zero
4963                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4964                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4965                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4966                 }
4967             } elsif ($opt =~ m/^net(\d+)$/) {
4968                 # some changes can be done without hotplug
4969                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4970                                     $vmid, $opt, $value, $arch, $machine_type);
4971             } elsif (is_valid_drivename($opt)) {
4972                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4973                 # some changes can be done without hotplug
4974                 my $drive = parse_drive($opt, $value);
4975                 if (drive_is_cloudinit($drive)) {
4976                     &$apply_pending_cloudinit($opt, $value);
4977                 }
4978                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4979                                      $vmid, $opt, $value, $arch, $machine_type);
4980             } elsif ($opt =~ m/^memory$/) { #dimms
4981                 die "skip\n" if !$hotplug_features->{memory};
4982                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4983             } elsif ($opt eq 'cpuunits') {
4984                 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4985                 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4986             } elsif ($opt eq 'cpulimit') {
4987                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4988                 $cgroup->change_cpu_quota($cpulimit, 100000);
4989             } elsif ($opt eq 'agent') {
4990                 vmconfig_update_agent($conf, $opt, $value);
4991             } else {
4992                 die "skip\n";  # skip non-hot-pluggable options
4993             }
4994         };
4995         if (my $err = $@) {
4996             &$add_error($opt, $err) if $err ne "skip\n";
4997         } else {
4998             $conf->{$opt} = $value;
4999             delete $conf->{pending}->{$opt};
5000         }
5001     }
5002
5003     PVE::QemuConfig->write_config($vmid, $conf);
5004 }
5005
5006 sub try_deallocate_drive {
5007     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5008
5009     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5010         my $volid = $drive->{file};
5011         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5012             my $sid = PVE::Storage::parse_volume_id($volid);
5013             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5014
5015             # check if the disk is really unused
5016             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5017                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5018             PVE::Storage::vdisk_free($storecfg, $volid);
5019             return 1;
5020         } else {
5021             # If vm is not owner of this disk remove from config
5022             return 1;
5023         }
5024     }
5025
5026     return;
5027 }
5028
5029 sub vmconfig_delete_or_detach_drive {
5030     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5031
5032     my $drive = parse_drive($opt, $conf->{$opt});
5033
5034     my $rpcenv = PVE::RPCEnvironment::get();
5035     my $authuser = $rpcenv->get_user();
5036
5037     if ($force) {
5038         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5039         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5040     } else {
5041         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5042     }
5043 }
5044
5045
5046
5047 sub vmconfig_apply_pending {
5048     my ($vmid, $conf, $storecfg, $errors) = @_;
5049
5050     return if !scalar(keys %{$conf->{pending}});
5051
5052     my $add_apply_error = sub {
5053         my ($opt, $msg) = @_;
5054         my $err_msg = "unable to apply pending change $opt : $msg";
5055         $errors->{$opt} = $err_msg;
5056         warn $err_msg;
5057     };
5058
5059     # cold plug
5060
5061     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5062     foreach my $opt (sort keys %$pending_delete_hash) {
5063         my $force = $pending_delete_hash->{$opt}->{force};
5064         eval {
5065             if ($opt =~ m/^unused/) {
5066                 die "internal error";
5067             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5068                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5069             }
5070         };
5071         if (my $err = $@) {
5072             $add_apply_error->($opt, $err);
5073         } else {
5074             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5075             delete $conf->{$opt};
5076         }
5077     }
5078
5079     PVE::QemuConfig->cleanup_pending($conf);
5080
5081     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5082         next if $opt eq 'delete'; # just to be sure
5083         eval {
5084             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5085                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5086             }
5087         };
5088         if (my $err = $@) {
5089             $add_apply_error->($opt, $err);
5090         } else {
5091             $conf->{$opt} = delete $conf->{pending}->{$opt};
5092         }
5093     }
5094
5095     # write all changes at once to avoid unnecessary i/o
5096     PVE::QemuConfig->write_config($vmid, $conf);
5097 }
5098
5099 sub vmconfig_update_net {
5100     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5101
5102     my $newnet = parse_net($value);
5103
5104     if ($conf->{$opt}) {
5105         my $oldnet = parse_net($conf->{$opt});
5106
5107         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5108             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5109             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5110             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5111
5112             # for non online change, we try to hot-unplug
5113             die "skip\n" if !$hotplug;
5114             vm_deviceunplug($vmid, $conf, $opt);
5115         } else {
5116
5117             die "internal error" if $opt !~ m/net(\d+)/;
5118             my $iface = "tap${vmid}i$1";
5119
5120             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5121                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5122                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5123                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5124                 PVE::Network::tap_unplug($iface);
5125
5126                 if ($have_sdn) {
5127                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5128                 } else {
5129                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5130                 }
5131             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5132                 # Rate can be applied on its own but any change above needs to
5133                 # include the rate in tap_plug since OVS resets everything.
5134                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5135             }
5136
5137             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5138                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5139             }
5140
5141             return 1;
5142         }
5143     }
5144
5145     if ($hotplug) {
5146         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5147     } else {
5148         die "skip\n";
5149     }
5150 }
5151
5152 sub vmconfig_update_agent {
5153     my ($conf, $opt, $value) = @_;
5154
5155     die "skip\n" if !$conf->{$opt};
5156
5157     my $hotplug_options = { fstrim_cloned_disks => 1 };
5158
5159     my $old_agent = parse_guest_agent($conf);
5160     my $agent = parse_guest_agent({$opt => $value});
5161
5162     for my $option (keys %$agent) { # added/changed options
5163         next if defined($hotplug_options->{$option});
5164         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5165     }
5166
5167     for my $option (keys %$old_agent) { # removed options
5168         next if defined($hotplug_options->{$option});
5169         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5170     }
5171
5172     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5173 }
5174
5175 sub vmconfig_update_disk {
5176     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5177
5178     my $drive = parse_drive($opt, $value);
5179
5180     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5181         my $media = $drive->{media} || 'disk';
5182         my $oldmedia = $old_drive->{media} || 'disk';
5183         die "unable to change media type\n" if $media ne $oldmedia;
5184
5185         if (!drive_is_cdrom($old_drive)) {
5186
5187             if ($drive->{file} ne $old_drive->{file}) {
5188
5189                 die "skip\n" if !$hotplug;
5190
5191                 # unplug and register as unused
5192                 vm_deviceunplug($vmid, $conf, $opt);
5193                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5194
5195             } else {
5196                 # update existing disk
5197
5198                 # skip non hotpluggable value
5199                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5200                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5201                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5202                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5203                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5204                     die "skip\n";
5205                 }
5206
5207                 # apply throttle
5208                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5209                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5210                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5211                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5212                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5213                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5214                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5215                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5216                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5217                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5218                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5219                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5220                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5221                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5222                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5223                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5224                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5225                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5226
5227                     qemu_block_set_io_throttle(
5228                         $vmid,"drive-$opt",
5229                         ($drive->{mbps} || 0)*1024*1024,
5230                         ($drive->{mbps_rd} || 0)*1024*1024,
5231                         ($drive->{mbps_wr} || 0)*1024*1024,
5232                          $drive->{iops} || 0,
5233                          $drive->{iops_rd} || 0,
5234                          $drive->{iops_wr} || 0,
5235                         ($drive->{mbps_max} || 0)*1024*1024,
5236                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5237                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5238                          $drive->{iops_max} || 0,
5239                          $drive->{iops_rd_max} || 0,
5240                          $drive->{iops_wr_max} || 0,
5241                          $drive->{bps_max_length} || 1,
5242                          $drive->{bps_rd_max_length} || 1,
5243                          $drive->{bps_wr_max_length} || 1,
5244                          $drive->{iops_max_length} || 1,
5245                          $drive->{iops_rd_max_length} || 1,
5246                          $drive->{iops_wr_max_length} || 1,
5247                     );
5248
5249                 }
5250
5251                 return 1;
5252             }
5253
5254         } else { # cdrom
5255
5256             if ($drive->{file} eq 'none') {
5257                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5258                 if (drive_is_cloudinit($old_drive)) {
5259                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5260                 }
5261             } else {
5262                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5263
5264                 # force eject if locked
5265                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5266
5267                 if ($path) {
5268                     mon_cmd($vmid, "blockdev-change-medium",
5269                         id => "$opt", filename => "$path");
5270                 }
5271             }
5272
5273             return 1;
5274         }
5275     }
5276
5277     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5278     # hotplug new disks
5279     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5280     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5281 }
5282
5283 # called in locked context by incoming migration
5284 sub vm_migrate_get_nbd_disks {
5285     my ($storecfg, $conf, $replicated_volumes) = @_;
5286
5287     my $local_volumes = {};
5288     PVE::QemuConfig->foreach_volume($conf, sub {
5289         my ($ds, $drive) = @_;
5290
5291         return if drive_is_cdrom($drive);
5292         return if $ds eq 'tpmstate0';
5293
5294         my $volid = $drive->{file};
5295
5296         return if !$volid;
5297
5298         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5299
5300         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5301         return if $scfg->{shared};
5302
5303         # replicated disks re-use existing state via bitmap
5304         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5305         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5306     });
5307     return $local_volumes;
5308 }
5309
5310 # called in locked context by incoming migration
5311 sub vm_migrate_alloc_nbd_disks {
5312     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5313
5314     my $nbd = {};
5315     foreach my $opt (sort keys %$source_volumes) {
5316         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5317
5318         if ($use_existing) {
5319             $nbd->{$opt}->{drivestr} = print_drive($drive);
5320             $nbd->{$opt}->{volid} = $volid;
5321             $nbd->{$opt}->{replicated} = 1;
5322             next;
5323         }
5324
5325         # storage mapping + volname = regular migration
5326         # storage mapping + format = remote migration
5327         # order of precedence, filtered by whether storage supports it:
5328         # 1. explicit requested format
5329         # 2. format of current volume
5330         # 3. default format of storage
5331         if (!$storagemap->{identity}) {
5332             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5333             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5334             if (!$format || !grep { $format eq $_ } @$validFormats) {
5335                 if ($volname) {
5336                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5337                     my $fileFormat = qemu_img_format($scfg, $volname);
5338                     $format = $fileFormat
5339                         if grep { $fileFormat eq $_ } @$validFormats;
5340                 }
5341                 $format //= $defFormat;
5342             }
5343         } else {
5344             # can't happen for remote migration, so $volname is always defined
5345             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5346             $format = qemu_img_format($scfg, $volname);
5347         }
5348
5349         my $size = $drive->{size} / 1024;
5350         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5351         my $newdrive = $drive;
5352         $newdrive->{format} = $format;
5353         $newdrive->{file} = $newvolid;
5354         my $drivestr = print_drive($newdrive);
5355         $nbd->{$opt}->{drivestr} = $drivestr;
5356         $nbd->{$opt}->{volid} = $newvolid;
5357     }
5358
5359     return $nbd;
5360 }
5361
5362 # see vm_start_nolock for parameters, additionally:
5363 # migrate_opts:
5364 #   storagemap = parsed storage map for allocating NBD disks
5365 sub vm_start {
5366     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5367
5368     return PVE::QemuConfig->lock_config($vmid, sub {
5369         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5370
5371         die "you can't start a vm if it's a template\n"
5372             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5373
5374         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5375         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5376
5377         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5378
5379         if ($has_backup_lock && $running) {
5380             # a backup is currently running, attempt to start the guest in the
5381             # existing QEMU instance
5382             return vm_resume($vmid);
5383         }
5384
5385         PVE::QemuConfig->check_lock($conf)
5386             if !($params->{skiplock} || $has_suspended_lock);
5387
5388         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5389
5390         die "VM $vmid already running\n" if $running;
5391
5392         if (my $storagemap = $migrate_opts->{storagemap}) {
5393             my $replicated = $migrate_opts->{replicated_volumes};
5394             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5395             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5396
5397             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5398                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5399             }
5400         }
5401
5402         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5403     });
5404 }
5405
5406
5407 # params:
5408 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5409 #   skiplock => 0/1, skip checking for config lock
5410 #   skiptemplate => 0/1, skip checking whether VM is template
5411 #   forcemachine => to force Qemu machine (rollback/migration)
5412 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5413 #   timeout => in seconds
5414 #   paused => start VM in paused state (backup)
5415 #   resume => resume from hibernation
5416 #   pbs-backing => {
5417 #      sata0 => {
5418 #         repository
5419 #         snapshot
5420 #         keyfile
5421 #         archive
5422 #      },
5423 #      virtio2 => ...
5424 #   }
5425 # migrate_opts:
5426 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5427 #   migratedfrom => source node
5428 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5429 #   network => CIDR of migration network
5430 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5431 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5432 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5433 #   tpmstate_vol => new volid of tpmstate0, not yet contained in config
5434 sub vm_start_nolock {
5435     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5436
5437     my $statefile = $params->{statefile};
5438     my $resume = $params->{resume};
5439
5440     my $migratedfrom = $migrate_opts->{migratedfrom};
5441     my $migration_type = $migrate_opts->{type};
5442
5443     my $res = {};
5444
5445     # clean up leftover reboot request files
5446     eval { clear_reboot_request($vmid); };
5447     warn $@ if $@;
5448
5449     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5450         vmconfig_apply_pending($vmid, $conf, $storecfg);
5451         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5452     }
5453
5454     # don't regenerate the ISO if the VM is started as part of a live migration
5455     # this way we can reuse the old ISO with the correct config
5456     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5457
5458     # override TPM state vol if migrated, conf is out of date still
5459     if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
5460         my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
5461         $parsed->{file} = $tpmvol;
5462         $conf->{tpmstate0} = print_drive($parsed);
5463     }
5464
5465     my $defaults = load_defaults();
5466
5467     # set environment variable useful inside network script
5468     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5469
5470     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5471
5472     my $forcemachine = $params->{forcemachine};
5473     my $forcecpu = $params->{forcecpu};
5474     if ($resume) {
5475         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5476         $forcemachine = $conf->{runningmachine};
5477         $forcecpu = $conf->{runningcpu};
5478         print "Resuming suspended VM\n";
5479     }
5480
5481     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5482         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5483
5484     my $migration_ip;
5485     my $get_migration_ip = sub {
5486         my ($nodename) = @_;
5487
5488         return $migration_ip if defined($migration_ip);
5489
5490         my $cidr = $migrate_opts->{network};
5491
5492         if (!defined($cidr)) {
5493             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5494             $cidr = $dc_conf->{migration}->{network};
5495         }
5496
5497         if (defined($cidr)) {
5498             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5499
5500             die "could not get IP: no address configured on local " .
5501                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5502
5503             die "could not get IP: multiple addresses configured on local " .
5504                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5505
5506             $migration_ip = @$ips[0];
5507         }
5508
5509         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5510             if !defined($migration_ip);
5511
5512         return $migration_ip;
5513     };
5514
5515     my $migrate_uri;
5516     if ($statefile) {
5517         if ($statefile eq 'tcp') {
5518             my $localip = "localhost";
5519             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5520             my $nodename = nodename();
5521
5522             if (!defined($migration_type)) {
5523                 if (defined($datacenterconf->{migration}->{type})) {
5524                     $migration_type = $datacenterconf->{migration}->{type};
5525                 } else {
5526                     $migration_type = 'secure';
5527                 }
5528             }
5529
5530             if ($migration_type eq 'insecure') {
5531                 $localip = $get_migration_ip->($nodename);
5532                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5533             }
5534
5535             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5536             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5537             $migrate_uri = "tcp:${localip}:${migrate_port}";
5538             push @$cmd, '-incoming', $migrate_uri;
5539             push @$cmd, '-S';
5540
5541         } elsif ($statefile eq 'unix') {
5542             # should be default for secure migrations as a ssh TCP forward
5543             # tunnel is not deterministic reliable ready and fails regurarly
5544             # to set up in time, so use UNIX socket forwards
5545             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5546             unlink $socket_addr;
5547
5548             $migrate_uri = "unix:$socket_addr";
5549
5550             push @$cmd, '-incoming', $migrate_uri;
5551             push @$cmd, '-S';
5552
5553         } elsif (-e $statefile) {
5554             push @$cmd, '-loadstate', $statefile;
5555         } else {
5556             my $statepath = PVE::Storage::path($storecfg, $statefile);
5557             push @$vollist, $statefile;
5558             push @$cmd, '-loadstate', $statepath;
5559         }
5560     } elsif ($params->{paused}) {
5561         push @$cmd, '-S';
5562     }
5563
5564     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5565
5566     my $pci_devices = {}; # host pci devices
5567     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5568         my $dev = $conf->{"hostpci$i"} or next;
5569         $pci_devices->{$i} = parse_hostpci($dev);
5570     }
5571
5572     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5573     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5574
5575     # map to a flat list of pci ids
5576     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5577
5578     # reserve all PCI IDs before actually doing anything with them
5579     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5580
5581     eval {
5582         for my $id (sort keys %$pci_devices) {
5583             my $d = $pci_devices->{$id};
5584             for my $dev ($d->{pciid}->@*) {
5585                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5586             }
5587         }
5588     };
5589     if (my $err = $@) {
5590         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5591         warn $@ if $@;
5592         die $err;
5593     }
5594
5595     PVE::Storage::activate_volumes($storecfg, $vollist);
5596
5597     eval {
5598         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5599     };
5600     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5601     # timeout should be more than enough here...
5602     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5603
5604     my $cpuunits = get_cpuunits($conf);
5605
5606     my %run_params = (
5607         timeout => $statefile ? undef : $start_timeout,
5608         umask => 0077,
5609         noerr => 1,
5610     );
5611
5612     # when migrating, prefix QEMU output so other side can pick up any
5613     # errors that might occur and show the user
5614     if ($migratedfrom) {
5615         $run_params{quiet} = 1;
5616         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5617     }
5618
5619     my %systemd_properties = (
5620         Slice => 'qemu.slice',
5621         KillMode => 'process',
5622         SendSIGKILL => 0,
5623         TimeoutStopUSec => ULONG_MAX, # infinity
5624     );
5625
5626     if (PVE::CGroup::cgroup_mode() == 2) {
5627         $systemd_properties{CPUWeight} = $cpuunits;
5628     } else {
5629         $systemd_properties{CPUShares} = $cpuunits;
5630     }
5631
5632     if (my $cpulimit = $conf->{cpulimit}) {
5633         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5634     }
5635     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5636
5637     my $run_qemu = sub {
5638         PVE::Tools::run_fork sub {
5639             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5640
5641             my $tpmpid;
5642             if (my $tpm = $conf->{tpmstate0}) {
5643                 # start the TPM emulator so QEMU can connect on start
5644                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5645             }
5646
5647             my $exitcode = run_command($cmd, %run_params);
5648             if ($exitcode) {
5649                 if ($tpmpid) {
5650                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5651                     kill 'TERM', $tpmpid;
5652                 }
5653                 die "QEMU exited with code $exitcode\n";
5654             }
5655         };
5656     };
5657
5658     if ($conf->{hugepages}) {
5659
5660         my $code = sub {
5661             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5662             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5663
5664             PVE::QemuServer::Memory::hugepages_mount();
5665             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5666
5667             eval { $run_qemu->() };
5668             if (my $err = $@) {
5669                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5670                     if !$conf->{keephugepages};
5671                 die $err;
5672             }
5673
5674             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5675                 if !$conf->{keephugepages};
5676         };
5677         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5678
5679     } else {
5680         eval { $run_qemu->() };
5681     }
5682
5683     if (my $err = $@) {
5684         # deactivate volumes if start fails
5685         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5686         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5687
5688         die "start failed: $err";
5689     }
5690
5691     # re-reserve all PCI IDs now that we can know the actual VM PID
5692     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5693     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5694     warn $@ if $@;
5695
5696     print "migration listens on $migrate_uri\n" if $migrate_uri;
5697     $res->{migrate_uri} = $migrate_uri;
5698
5699     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5700         eval { mon_cmd($vmid, "cont"); };
5701         warn $@ if $@;
5702     }
5703
5704     #start nbd server for storage migration
5705     if (my $nbd = $migrate_opts->{nbd}) {
5706         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5707
5708         my $migrate_storage_uri;
5709         # nbd_protocol_version > 0 for unix socket support
5710         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5711             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5712             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5713             $migrate_storage_uri = "nbd:unix:$socket_path";
5714         } else {
5715             my $nodename = nodename();
5716             my $localip = $get_migration_ip->($nodename);
5717             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5718             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5719
5720             mon_cmd($vmid, "nbd-server-start", addr => {
5721                 type => 'inet',
5722                 data => {
5723                     host => "${localip}",
5724                     port => "${storage_migrate_port}",
5725                 },
5726             });
5727             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5728             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5729         }
5730
5731         $res->{migrate_storage_uri} = $migrate_storage_uri;
5732
5733         foreach my $opt (sort keys %$nbd) {
5734             my $drivestr = $nbd->{$opt}->{drivestr};
5735             my $volid = $nbd->{$opt}->{volid};
5736             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5737             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5738             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5739             print "re-using replicated volume: $opt - $volid\n"
5740                 if $nbd->{$opt}->{replicated};
5741
5742             $res->{drives}->{$opt} = $nbd->{$opt};
5743             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5744         }
5745     }
5746
5747     if ($migratedfrom) {
5748         eval {
5749             set_migration_caps($vmid);
5750         };
5751         warn $@ if $@;
5752
5753         if ($spice_port) {
5754             print "spice listens on port $spice_port\n";
5755             $res->{spice_port} = $spice_port;
5756             if ($migrate_opts->{spice_ticket}) {
5757                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5758                     $migrate_opts->{spice_ticket});
5759                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5760             }
5761         }
5762
5763     } else {
5764         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5765             if !$statefile && $conf->{balloon};
5766
5767         foreach my $opt (keys %$conf) {
5768             next if $opt !~  m/^net\d+$/;
5769             my $nicconf = parse_net($conf->{$opt});
5770             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5771         }
5772     }
5773
5774     mon_cmd($vmid, 'qom-set',
5775                 path => "machine/peripheral/balloon0",
5776                 property => "guest-stats-polling-interval",
5777                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5778
5779     if ($resume) {
5780         print "Resumed VM, removing state\n";
5781         if (my $vmstate = $conf->{vmstate}) {
5782             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5783             PVE::Storage::vdisk_free($storecfg, $vmstate);
5784         }
5785         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5786         PVE::QemuConfig->write_config($vmid, $conf);
5787     }
5788
5789     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5790
5791     return $res;
5792 }
5793
5794 sub vm_commandline {
5795     my ($storecfg, $vmid, $snapname) = @_;
5796
5797     my $conf = PVE::QemuConfig->load_config($vmid);
5798
5799     my ($forcemachine, $forcecpu);
5800     if ($snapname) {
5801         my $snapshot = $conf->{snapshots}->{$snapname};
5802         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5803
5804         # check for machine or CPU overrides in snapshot
5805         $forcemachine = $snapshot->{runningmachine};
5806         $forcecpu = $snapshot->{runningcpu};
5807
5808         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5809
5810         $conf = $snapshot;
5811     }
5812
5813     my $defaults = load_defaults();
5814
5815     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5816
5817     return PVE::Tools::cmd2string($cmd);
5818 }
5819
5820 sub vm_reset {
5821     my ($vmid, $skiplock) = @_;
5822
5823     PVE::QemuConfig->lock_config($vmid, sub {
5824
5825         my $conf = PVE::QemuConfig->load_config($vmid);
5826
5827         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5828
5829         mon_cmd($vmid, "system_reset");
5830     });
5831 }
5832
5833 sub get_vm_volumes {
5834     my ($conf) = @_;
5835
5836     my $vollist = [];
5837     foreach_volid($conf, sub {
5838         my ($volid, $attr) = @_;
5839
5840         return if $volid =~ m|^/|;
5841
5842         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5843         return if !$sid;
5844
5845         push @$vollist, $volid;
5846     });
5847
5848     return $vollist;
5849 }
5850
5851 sub vm_stop_cleanup {
5852     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5853
5854     eval {
5855
5856         if (!$keepActive) {
5857             my $vollist = get_vm_volumes($conf);
5858             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5859
5860             if (my $tpmdrive = $conf->{tpmstate0}) {
5861                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5862                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5863                 if ($storeid) {
5864                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5865                 }
5866             }
5867         }
5868
5869         foreach my $ext (qw(mon qmp pid vnc qga)) {
5870             unlink "/var/run/qemu-server/${vmid}.$ext";
5871         }
5872
5873         if ($conf->{ivshmem}) {
5874             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5875             # just delete it for now, VMs which have this already open do not
5876             # are affected, but new VMs will get a separated one. If this
5877             # becomes an issue we either add some sort of ref-counting or just
5878             # add a "don't delete on stop" flag to the ivshmem format.
5879             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5880         }
5881
5882         my $ids = [];
5883         foreach my $key (keys %$conf) {
5884             next if $key !~ m/^hostpci(\d+)$/;
5885             my $hostpciindex = $1;
5886             my $d = parse_hostpci($conf->{$key});
5887             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5888
5889             foreach my $pci (@{$d->{pciid}}) {
5890                 my $pciid = $pci->{id};
5891                 push @$ids, $pci->{id};
5892                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5893             }
5894         }
5895         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5896
5897         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5898     };
5899     warn $@ if $@; # avoid errors - just warn
5900 }
5901
5902 # call only in locked context
5903 sub _do_vm_stop {
5904     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5905
5906     my $pid = check_running($vmid, $nocheck);
5907     return if !$pid;
5908
5909     my $conf;
5910     if (!$nocheck) {
5911         $conf = PVE::QemuConfig->load_config($vmid);
5912         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5913         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5914             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5915             $timeout = $opts->{down} if $opts->{down};
5916         }
5917         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5918     }
5919
5920     eval {
5921         if ($shutdown) {
5922             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5923                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5924             } else {
5925                 mon_cmd($vmid, "system_powerdown");
5926             }
5927         } else {
5928             mon_cmd($vmid, "quit");
5929         }
5930     };
5931     my $err = $@;
5932
5933     if (!$err) {
5934         $timeout = 60 if !defined($timeout);
5935
5936         my $count = 0;
5937         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5938             $count++;
5939             sleep 1;
5940         }
5941
5942         if ($count >= $timeout) {
5943             if ($force) {
5944                 warn "VM still running - terminating now with SIGTERM\n";
5945                 kill 15, $pid;
5946             } else {
5947                 die "VM quit/powerdown failed - got timeout\n";
5948             }
5949         } else {
5950             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5951             return;
5952         }
5953     } else {
5954        if (!check_running($vmid, $nocheck)) {
5955             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5956             return;
5957        }
5958        if ($force) {
5959             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5960             kill 15, $pid;
5961         } else {
5962             die "VM quit/powerdown failed\n";
5963         }
5964     }
5965
5966     # wait again
5967     $timeout = 10;
5968
5969     my $count = 0;
5970     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5971         $count++;
5972         sleep 1;
5973     }
5974
5975     if ($count >= $timeout) {
5976         warn "VM still running - terminating now with SIGKILL\n";
5977         kill 9, $pid;
5978         sleep 1;
5979     }
5980
5981     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5982 }
5983
5984 # Note: use $nocheck to skip tests if VM configuration file exists.
5985 # We need that when migration VMs to other nodes (files already moved)
5986 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5987 sub vm_stop {
5988     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5989
5990     $force = 1 if !defined($force) && !$shutdown;
5991
5992     if ($migratedfrom){
5993         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5994         kill 15, $pid if $pid;
5995         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5996         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5997         return;
5998     }
5999
6000     PVE::QemuConfig->lock_config($vmid, sub {
6001         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6002    });
6003 }
6004
6005 sub vm_reboot {
6006     my ($vmid, $timeout) = @_;
6007
6008     PVE::QemuConfig->lock_config($vmid, sub {
6009         eval {
6010
6011             # only reboot if running, as qmeventd starts it again on a stop event
6012             return if !check_running($vmid);
6013
6014             create_reboot_request($vmid);
6015
6016             my $storecfg = PVE::Storage::config();
6017             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6018
6019         };
6020         if (my $err = $@) {
6021             # avoid that the next normal shutdown will be confused for a reboot
6022             clear_reboot_request($vmid);
6023             die $err;
6024         }
6025    });
6026 }
6027
6028 # note: if using the statestorage parameter, the caller has to check privileges
6029 sub vm_suspend {
6030     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6031
6032     my $conf;
6033     my $path;
6034     my $storecfg;
6035     my $vmstate;
6036
6037     PVE::QemuConfig->lock_config($vmid, sub {
6038
6039         $conf = PVE::QemuConfig->load_config($vmid);
6040
6041         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6042         PVE::QemuConfig->check_lock($conf)
6043             if !($skiplock || $is_backing_up);
6044
6045         die "cannot suspend to disk during backup\n"
6046             if $is_backing_up && $includestate;
6047
6048         if ($includestate) {
6049             $conf->{lock} = 'suspending';
6050             my $date = strftime("%Y-%m-%d", localtime(time()));
6051             $storecfg = PVE::Storage::config();
6052             if (!$statestorage) {
6053                 $statestorage = find_vmstate_storage($conf, $storecfg);
6054                 # check permissions for the storage
6055                 my $rpcenv = PVE::RPCEnvironment::get();
6056                 if ($rpcenv->{type} ne 'cli') {
6057                     my $authuser = $rpcenv->get_user();
6058                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6059                 }
6060             }
6061
6062
6063             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6064                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6065             $path = PVE::Storage::path($storecfg, $vmstate);
6066             PVE::QemuConfig->write_config($vmid, $conf);
6067         } else {
6068             mon_cmd($vmid, "stop");
6069         }
6070     });
6071
6072     if ($includestate) {
6073         # save vm state
6074         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6075
6076         eval {
6077             set_migration_caps($vmid, 1);
6078             mon_cmd($vmid, "savevm-start", statefile => $path);
6079             for(;;) {
6080                 my $state = mon_cmd($vmid, "query-savevm");
6081                 if (!$state->{status}) {
6082                     die "savevm not active\n";
6083                 } elsif ($state->{status} eq 'active') {
6084                     sleep(1);
6085                     next;
6086                 } elsif ($state->{status} eq 'completed') {
6087                     print "State saved, quitting\n";
6088                     last;
6089                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6090                     die "query-savevm failed with error '$state->{error}'\n"
6091                 } else {
6092                     die "query-savevm returned status '$state->{status}'\n";
6093                 }
6094             }
6095         };
6096         my $err = $@;
6097
6098         PVE::QemuConfig->lock_config($vmid, sub {
6099             $conf = PVE::QemuConfig->load_config($vmid);
6100             if ($err) {
6101                 # cleanup, but leave suspending lock, to indicate something went wrong
6102                 eval {
6103                     mon_cmd($vmid, "savevm-end");
6104                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6105                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6106                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6107                     PVE::QemuConfig->write_config($vmid, $conf);
6108                 };
6109                 warn $@ if $@;
6110                 die $err;
6111             }
6112
6113             die "lock changed unexpectedly\n"
6114                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6115
6116             mon_cmd($vmid, "quit");
6117             $conf->{lock} = 'suspended';
6118             PVE::QemuConfig->write_config($vmid, $conf);
6119         });
6120     }
6121 }
6122
6123 sub vm_resume {
6124     my ($vmid, $skiplock, $nocheck) = @_;
6125
6126     PVE::QemuConfig->lock_config($vmid, sub {
6127         my $res = mon_cmd($vmid, 'query-status');
6128         my $resume_cmd = 'cont';
6129         my $reset = 0;
6130
6131         if ($res->{status}) {
6132             return if $res->{status} eq 'running'; # job done, go home
6133             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6134             $reset = 1 if $res->{status} eq 'shutdown';
6135         }
6136
6137         if (!$nocheck) {
6138
6139             my $conf = PVE::QemuConfig->load_config($vmid);
6140
6141             PVE::QemuConfig->check_lock($conf)
6142                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6143         }
6144
6145         if ($reset) {
6146             # required if a VM shuts down during a backup and we get a resume
6147             # request before the backup finishes for example
6148             mon_cmd($vmid, "system_reset");
6149         }
6150         mon_cmd($vmid, $resume_cmd);
6151     });
6152 }
6153
6154 sub vm_sendkey {
6155     my ($vmid, $skiplock, $key) = @_;
6156
6157     PVE::QemuConfig->lock_config($vmid, sub {
6158
6159         my $conf = PVE::QemuConfig->load_config($vmid);
6160
6161         # there is no qmp command, so we use the human monitor command
6162         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6163         die $res if $res ne '';
6164     });
6165 }
6166
6167 # vzdump restore implementaion
6168
6169 sub tar_archive_read_firstfile {
6170     my $archive = shift;
6171
6172     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6173
6174     # try to detect archive type first
6175     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6176         die "unable to open file '$archive'\n";
6177     my $firstfile = <$fh>;
6178     kill 15, $pid;
6179     close $fh;
6180
6181     die "ERROR: archive contaions no data\n" if !$firstfile;
6182     chomp $firstfile;
6183
6184     return $firstfile;
6185 }
6186
6187 sub tar_restore_cleanup {
6188     my ($storecfg, $statfile) = @_;
6189
6190     print STDERR "starting cleanup\n";
6191
6192     if (my $fd = IO::File->new($statfile, "r")) {
6193         while (defined(my $line = <$fd>)) {
6194             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6195                 my $volid = $2;
6196                 eval {
6197                     if ($volid =~ m|^/|) {
6198                         unlink $volid || die 'unlink failed\n';
6199                     } else {
6200                         PVE::Storage::vdisk_free($storecfg, $volid);
6201                     }
6202                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6203                 };
6204                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6205             } else {
6206                 print STDERR "unable to parse line in statfile - $line";
6207             }
6208         }
6209         $fd->close();
6210     }
6211 }
6212
6213 sub restore_file_archive {
6214     my ($archive, $vmid, $user, $opts) = @_;
6215
6216     return restore_vma_archive($archive, $vmid, $user, $opts)
6217         if $archive eq '-';
6218
6219     my $info = PVE::Storage::archive_info($archive);
6220     my $format = $opts->{format} // $info->{format};
6221     my $comp = $info->{compression};
6222
6223     # try to detect archive format
6224     if ($format eq 'tar') {
6225         return restore_tar_archive($archive, $vmid, $user, $opts);
6226     } else {
6227         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6228     }
6229 }
6230
6231 # hepler to remove disks that will not be used after restore
6232 my $restore_cleanup_oldconf = sub {
6233     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6234
6235     my $kept_disks = {};
6236
6237     PVE::QemuConfig->foreach_volume($oldconf, sub {
6238         my ($ds, $drive) = @_;
6239
6240         return if drive_is_cdrom($drive, 1);
6241
6242         my $volid = $drive->{file};
6243         return if !$volid || $volid =~ m|^/|;
6244
6245         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6246         return if !$path || !$owner || ($owner != $vmid);
6247
6248         # Note: only delete disk we want to restore
6249         # other volumes will become unused
6250         if ($virtdev_hash->{$ds}) {
6251             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6252             if (my $err = $@) {
6253                 warn $err;
6254             }
6255         } else {
6256             $kept_disks->{$volid} = 1;
6257         }
6258     });
6259
6260     # after the restore we have no snapshots anymore
6261     for my $snapname (keys $oldconf->{snapshots}->%*) {
6262         my $snap = $oldconf->{snapshots}->{$snapname};
6263         if ($snap->{vmstate}) {
6264             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6265             if (my $err = $@) {
6266                 warn $err;
6267             }
6268         }
6269
6270         for my $volid (keys $kept_disks->%*) {
6271             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6272             warn $@ if $@;
6273         }
6274     }
6275 };
6276
6277 # Helper to parse vzdump backup device hints
6278 #
6279 # $rpcenv: Environment, used to ckeck storage permissions
6280 # $user: User ID, to check storage permissions
6281 # $storecfg: Storage configuration
6282 # $fh: the file handle for reading the configuration
6283 # $devinfo: should contain device sizes for all backu-up'ed devices
6284 # $options: backup options (pool, default storage)
6285 #
6286 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6287 my $parse_backup_hints = sub {
6288     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6289
6290     my $check_storage = sub { # assert if an image can be allocate
6291         my ($storeid, $scfg) = @_;
6292         die "Content type 'images' is not available on storage '$storeid'\n"
6293             if !$scfg->{content}->{images};
6294         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6295             if $user ne 'root@pam';
6296     };
6297
6298     my $virtdev_hash = {};
6299     while (defined(my $line = <$fh>)) {
6300         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6301             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6302             die "archive does not contain data for drive '$virtdev'\n"
6303                 if !$devinfo->{$devname};
6304
6305             if (defined($options->{storage})) {
6306                 $storeid = $options->{storage} || 'local';
6307             } elsif (!$storeid) {
6308                 $storeid = 'local';
6309             }
6310             $format = 'raw' if !$format;
6311             $devinfo->{$devname}->{devname} = $devname;
6312             $devinfo->{$devname}->{virtdev} = $virtdev;
6313             $devinfo->{$devname}->{format} = $format;
6314             $devinfo->{$devname}->{storeid} = $storeid;
6315
6316             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6317             $check_storage->($storeid, $scfg); # permission and content type check
6318
6319             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6320         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6321             my $virtdev = $1;
6322             my $drive = parse_drive($virtdev, $2);
6323
6324             if (drive_is_cloudinit($drive)) {
6325                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6326                 $storeid = $options->{storage} if defined ($options->{storage});
6327                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6328                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6329
6330                 $check_storage->($storeid, $scfg); # permission and content type check
6331
6332                 $virtdev_hash->{$virtdev} = {
6333                     format => $format,
6334                     storeid => $storeid,
6335                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6336                     is_cloudinit => 1,
6337                 };
6338             }
6339         }
6340     }
6341
6342     return $virtdev_hash;
6343 };
6344
6345 # Helper to allocate and activate all volumes required for a restore
6346 #
6347 # $storecfg: Storage configuration
6348 # $virtdev_hash: as returned by parse_backup_hints()
6349 #
6350 # Returns: { $virtdev => $volid }
6351 my $restore_allocate_devices = sub {
6352     my ($storecfg, $virtdev_hash, $vmid) = @_;
6353
6354     my $map = {};
6355     foreach my $virtdev (sort keys %$virtdev_hash) {
6356         my $d = $virtdev_hash->{$virtdev};
6357         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6358         my $storeid = $d->{storeid};
6359         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6360
6361         # test if requested format is supported
6362         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6363         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6364         $d->{format} = $defFormat if !$supported;
6365
6366         my $name;
6367         if ($d->{is_cloudinit}) {
6368             $name = "vm-$vmid-cloudinit";
6369             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6370             if ($scfg->{path}) {
6371                 $name .= ".$d->{format}";
6372             }
6373         }
6374
6375         my $volid = PVE::Storage::vdisk_alloc(
6376             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6377
6378         print STDERR "new volume ID is '$volid'\n";
6379         $d->{volid} = $volid;
6380
6381         PVE::Storage::activate_volumes($storecfg, [$volid]);
6382
6383         $map->{$virtdev} = $volid;
6384     }
6385
6386     return $map;
6387 };
6388
6389 sub restore_update_config_line {
6390     my ($cookie, $map, $line, $unique) = @_;
6391
6392     return '' if $line =~ m/^\#qmdump\#/;
6393     return '' if $line =~ m/^\#vzdump\#/;
6394     return '' if $line =~ m/^lock:/;
6395     return '' if $line =~ m/^unused\d+:/;
6396     return '' if $line =~ m/^parent:/;
6397
6398     my $res = '';
6399
6400     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6401     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6402         # try to convert old 1.X settings
6403         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6404         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6405             my ($model, $macaddr) = split(/\=/, $devconfig);
6406             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6407             my $net = {
6408                 model => $model,
6409                 bridge => "vmbr$ind",
6410                 macaddr => $macaddr,
6411             };
6412             my $netstr = print_net($net);
6413
6414             $res .= "net$cookie->{netcount}: $netstr\n";
6415             $cookie->{netcount}++;
6416         }
6417     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6418         my ($id, $netstr) = ($1, $2);
6419         my $net = parse_net($netstr);
6420         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6421         $netstr = print_net($net);
6422         $res .= "$id: $netstr\n";
6423     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6424         my $virtdev = $1;
6425         my $value = $3;
6426         my $di = parse_drive($virtdev, $value);
6427         if (defined($di->{backup}) && !$di->{backup}) {
6428             $res .= "#$line";
6429         } elsif ($map->{$virtdev}) {
6430             delete $di->{format}; # format can change on restore
6431             $di->{file} = $map->{$virtdev};
6432             $value = print_drive($di);
6433             $res .= "$virtdev: $value\n";
6434         } else {
6435             $res .= $line;
6436         }
6437     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6438         my $vmgenid = $1;
6439         if ($vmgenid ne '0') {
6440             # always generate a new vmgenid if there was a valid one setup
6441             $vmgenid = generate_uuid();
6442         }
6443         $res .= "vmgenid: $vmgenid\n";
6444     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6445         my ($uuid, $uuid_str);
6446         UUID::generate($uuid);
6447         UUID::unparse($uuid, $uuid_str);
6448         my $smbios1 = parse_smbios1($2);
6449         $smbios1->{uuid} = $uuid_str;
6450         $res .= $1.print_smbios1($smbios1)."\n";
6451     } else {
6452         $res .= $line;
6453     }
6454
6455     return $res;
6456 }
6457
6458 my $restore_deactivate_volumes = sub {
6459     my ($storecfg, $virtdev_hash) = @_;
6460
6461     my $vollist = [];
6462     for my $dev (values $virtdev_hash->%*) {
6463         push $vollist->@*, $dev->{volid} if $dev->{volid};
6464     }
6465
6466     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6467     print STDERR $@ if $@;
6468 };
6469
6470 my $restore_destroy_volumes = sub {
6471     my ($storecfg, $virtdev_hash) = @_;
6472
6473     for my $dev (values $virtdev_hash->%*) {
6474         my $volid = $dev->{volid} or next;
6475         eval {
6476             PVE::Storage::vdisk_free($storecfg, $volid);
6477             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6478         };
6479         print STDERR "unable to cleanup '$volid' - $@" if $@;
6480     }
6481 };
6482
6483 sub scan_volids {
6484     my ($cfg, $vmid) = @_;
6485
6486     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6487
6488     my $volid_hash = {};
6489     foreach my $storeid (keys %$info) {
6490         foreach my $item (@{$info->{$storeid}}) {
6491             next if !($item->{volid} && $item->{size});
6492             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6493             $volid_hash->{$item->{volid}} = $item;
6494         }
6495     }
6496
6497     return $volid_hash;
6498 }
6499
6500 sub update_disk_config {
6501     my ($vmid, $conf, $volid_hash) = @_;
6502
6503     my $changes;
6504     my $prefix = "VM $vmid";
6505
6506     # used and unused disks
6507     my $referenced = {};
6508
6509     # Note: it is allowed to define multiple storages with same path (alias), so
6510     # we need to check both 'volid' and real 'path' (two different volid can point
6511     # to the same path).
6512
6513     my $referencedpath = {};
6514
6515     # update size info
6516     PVE::QemuConfig->foreach_volume($conf, sub {
6517         my ($opt, $drive) = @_;
6518
6519         my $volid = $drive->{file};
6520         return if !$volid;
6521         my $volume = $volid_hash->{$volid};
6522
6523         # mark volid as "in-use" for next step
6524         $referenced->{$volid} = 1;
6525         if ($volume && (my $path = $volume->{path})) {
6526             $referencedpath->{$path} = 1;
6527         }
6528
6529         return if drive_is_cdrom($drive);
6530         return if !$volume;
6531
6532         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6533         if (defined($updated)) {
6534             $changes = 1;
6535             $conf->{$opt} = print_drive($updated);
6536             print "$prefix ($opt): $msg\n";
6537         }
6538     });
6539
6540     # remove 'unusedX' entry if volume is used
6541     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6542         my ($opt, $drive) = @_;
6543
6544         my $volid = $drive->{file};
6545         return if !$volid;
6546
6547         my $path;
6548         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6549         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6550             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6551             $changes = 1;
6552             delete $conf->{$opt};
6553         }
6554
6555         $referenced->{$volid} = 1;
6556         $referencedpath->{$path} = 1 if $path;
6557     });
6558
6559     foreach my $volid (sort keys %$volid_hash) {
6560         next if $volid =~ m/vm-$vmid-state-/;
6561         next if $referenced->{$volid};
6562         my $path = $volid_hash->{$volid}->{path};
6563         next if !$path; # just to be sure
6564         next if $referencedpath->{$path};
6565         $changes = 1;
6566         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6567         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6568         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6569     }
6570
6571     return $changes;
6572 }
6573
6574 sub rescan {
6575     my ($vmid, $nolock, $dryrun) = @_;
6576
6577     my $cfg = PVE::Storage::config();
6578
6579     print "rescan volumes...\n";
6580     my $volid_hash = scan_volids($cfg, $vmid);
6581
6582     my $updatefn =  sub {
6583         my ($vmid) = @_;
6584
6585         my $conf = PVE::QemuConfig->load_config($vmid);
6586
6587         PVE::QemuConfig->check_lock($conf);
6588
6589         my $vm_volids = {};
6590         foreach my $volid (keys %$volid_hash) {
6591             my $info = $volid_hash->{$volid};
6592             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6593         }
6594
6595         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6596
6597         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6598     };
6599
6600     if (defined($vmid)) {
6601         if ($nolock) {
6602             &$updatefn($vmid);
6603         } else {
6604             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6605         }
6606     } else {
6607         my $vmlist = config_list();
6608         foreach my $vmid (keys %$vmlist) {
6609             if ($nolock) {
6610                 &$updatefn($vmid);
6611             } else {
6612                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6613             }
6614         }
6615     }
6616 }
6617
6618 sub restore_proxmox_backup_archive {
6619     my ($archive, $vmid, $user, $options) = @_;
6620
6621     my $storecfg = PVE::Storage::config();
6622
6623     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6624     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6625
6626     my $fingerprint = $scfg->{fingerprint};
6627     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6628
6629     my $repo = PVE::PBSClient::get_repository($scfg);
6630
6631     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6632     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6633     local $ENV{PBS_PASSWORD} = $password;
6634     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6635
6636     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6637         PVE::Storage::parse_volname($storecfg, $archive);
6638
6639     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6640
6641     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6642
6643     my $tmpdir = "/var/tmp/vzdumptmp$$";
6644     rmtree $tmpdir;
6645     mkpath $tmpdir;
6646
6647     my $conffile = PVE::QemuConfig->config_file($vmid);
6648      # disable interrupts (always do cleanups)
6649     local $SIG{INT} =
6650         local $SIG{TERM} =
6651         local $SIG{QUIT} =
6652         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6653
6654     # Note: $oldconf is undef if VM does not exists
6655     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6656     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6657     my $new_conf_raw = '';
6658
6659     my $rpcenv = PVE::RPCEnvironment::get();
6660     my $devinfo = {}; # info about drives included in backup
6661     my $virtdev_hash = {}; # info about allocated drives
6662
6663     eval {
6664         # enable interrupts
6665         local $SIG{INT} =
6666             local $SIG{TERM} =
6667             local $SIG{QUIT} =
6668             local $SIG{HUP} =
6669             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6670
6671         my $cfgfn = "$tmpdir/qemu-server.conf";
6672         my $firewall_config_fn = "$tmpdir/fw.conf";
6673         my $index_fn = "$tmpdir/index.json";
6674
6675         my $cmd = "restore";
6676
6677         my $param = [$pbs_backup_name, "index.json", $index_fn];
6678         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6679         my $index = PVE::Tools::file_get_contents($index_fn);
6680         $index = decode_json($index);
6681
6682         foreach my $info (@{$index->{files}}) {
6683             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6684                 my $devname = $1;
6685                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6686                     $devinfo->{$devname}->{size} = $1;
6687                 } else {
6688                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6689                 }
6690             }
6691         }
6692
6693         my $is_qemu_server_backup = scalar(
6694             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6695         );
6696         if (!$is_qemu_server_backup) {
6697             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6698         }
6699         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6700
6701         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6702         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6703
6704         if ($has_firewall_config) {
6705             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6706             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6707
6708             my $pve_firewall_dir = '/etc/pve/firewall';
6709             mkdir $pve_firewall_dir; # make sure the dir exists
6710             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6711         }
6712
6713         my $fh = IO::File->new($cfgfn, "r") ||
6714             die "unable to read qemu-server.conf - $!\n";
6715
6716         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6717
6718         # fixme: rate limit?
6719
6720         # create empty/temp config
6721         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6722
6723         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6724
6725         # allocate volumes
6726         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6727
6728         foreach my $virtdev (sort keys %$virtdev_hash) {
6729             my $d = $virtdev_hash->{$virtdev};
6730             next if $d->{is_cloudinit}; # no need to restore cloudinit
6731
6732             # this fails if storage is unavailable
6733             my $volid = $d->{volid};
6734             my $path = PVE::Storage::path($storecfg, $volid);
6735
6736             # for live-restore we only want to preload the efidisk and TPM state
6737             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6738
6739             my $pbs_restore_cmd = [
6740                 '/usr/bin/pbs-restore',
6741                 '--repository', $repo,
6742                 $pbs_backup_name,
6743                 "$d->{devname}.img.fidx",
6744                 $path,
6745                 '--verbose',
6746                 ];
6747
6748             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6749             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6750
6751             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6752                 push @$pbs_restore_cmd, '--skip-zero';
6753             }
6754
6755             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6756             print "restore proxmox backup image: $dbg_cmdstring\n";
6757             run_command($pbs_restore_cmd);
6758         }
6759
6760         $fh->seek(0, 0) || die "seek failed - $!\n";
6761
6762         my $cookie = { netcount => 0 };
6763         while (defined(my $line = <$fh>)) {
6764             $new_conf_raw .= restore_update_config_line(
6765                 $cookie,
6766                 $map,
6767                 $line,
6768                 $options->{unique},
6769             );
6770         }
6771
6772         $fh->close();
6773     };
6774     my $err = $@;
6775
6776     if ($err || !$options->{live}) {
6777         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6778     }
6779
6780     rmtree $tmpdir;
6781
6782     if ($err) {
6783         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6784         die $err;
6785     }
6786
6787     if ($options->{live}) {
6788         # keep lock during live-restore
6789         $new_conf_raw .= "\nlock: create";
6790     }
6791
6792     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6793
6794     PVE::Cluster::cfs_update(); # make sure we read new file
6795
6796     eval { rescan($vmid, 1); };
6797     warn $@ if $@;
6798
6799     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6800
6801     if ($options->{live}) {
6802         # enable interrupts
6803         local $SIG{INT} =
6804             local $SIG{TERM} =
6805             local $SIG{QUIT} =
6806             local $SIG{HUP} =
6807             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6808
6809         my $conf = PVE::QemuConfig->load_config($vmid);
6810         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6811
6812         # these special drives are already restored before start
6813         delete $devinfo->{'drive-efidisk0'};
6814         delete $devinfo->{'drive-tpmstate0-backup'};
6815         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6816
6817         PVE::QemuConfig->remove_lock($vmid, "create");
6818     }
6819 }
6820
6821 sub pbs_live_restore {
6822     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6823
6824     print "starting VM for live-restore\n";
6825     print "repository: '$repo', snapshot: '$snap'\n";
6826
6827     my $pbs_backing = {};
6828     for my $ds (keys %$restored_disks) {
6829         $ds =~ m/^drive-(.*)$/;
6830         my $confname = $1;
6831         $pbs_backing->{$confname} = {
6832             repository => $repo,
6833             snapshot => $snap,
6834             archive => "$ds.img.fidx",
6835         };
6836         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6837
6838         my $drive = parse_drive($confname, $conf->{$confname});
6839         print "restoring '$ds' to '$drive->{file}'\n";
6840     }
6841
6842     my $drives_streamed = 0;
6843     eval {
6844         # make sure HA doesn't interrupt our restore by stopping the VM
6845         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6846             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6847         }
6848
6849         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6850         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6851         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6852
6853         my $qmeventd_fd = register_qmeventd_handle($vmid);
6854
6855         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6856         # this will effectively collapse the backing image chain consisting of
6857         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6858         # removes itself once all backing images vanish with 'auto-remove=on')
6859         my $jobs = {};
6860         for my $ds (sort keys %$restored_disks) {
6861             my $job_id = "restore-$ds";
6862             mon_cmd($vmid, 'block-stream',
6863                 'job-id' => $job_id,
6864                 device => "$ds",
6865             );
6866             $jobs->{$job_id} = {};
6867         }
6868
6869         mon_cmd($vmid, 'cont');
6870         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6871
6872         print "restore-drive jobs finished successfully, removing all tracking block devices"
6873             ." to disconnect from Proxmox Backup Server\n";
6874
6875         for my $ds (sort keys %$restored_disks) {
6876             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6877         }
6878
6879         close($qmeventd_fd);
6880     };
6881
6882     my $err = $@;
6883
6884     if ($err) {
6885         warn "An error occurred during live-restore: $err\n";
6886         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6887         die "live-restore failed\n";
6888     }
6889 }
6890
6891 sub restore_vma_archive {
6892     my ($archive, $vmid, $user, $opts, $comp) = @_;
6893
6894     my $readfrom = $archive;
6895
6896     my $cfg = PVE::Storage::config();
6897     my $commands = [];
6898     my $bwlimit = $opts->{bwlimit};
6899
6900     my $dbg_cmdstring = '';
6901     my $add_pipe = sub {
6902         my ($cmd) = @_;
6903         push @$commands, $cmd;
6904         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6905         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6906         $readfrom = '-';
6907     };
6908
6909     my $input = undef;
6910     if ($archive eq '-') {
6911         $input = '<&STDIN';
6912     } else {
6913         # If we use a backup from a PVE defined storage we also consider that
6914         # storage's rate limit:
6915         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6916         if (defined($volid)) {
6917             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6918             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6919             if ($readlimit) {
6920                 print STDERR "applying read rate limit: $readlimit\n";
6921                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6922                 $add_pipe->($cstream);
6923             }
6924         }
6925     }
6926
6927     if ($comp) {
6928         my $info = PVE::Storage::decompressor_info('vma', $comp);
6929         my $cmd = $info->{decompressor};
6930         push @$cmd, $readfrom;
6931         $add_pipe->($cmd);
6932     }
6933
6934     my $tmpdir = "/var/tmp/vzdumptmp$$";
6935     rmtree $tmpdir;
6936
6937     # disable interrupts (always do cleanups)
6938     local $SIG{INT} =
6939         local $SIG{TERM} =
6940         local $SIG{QUIT} =
6941         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6942
6943     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6944     POSIX::mkfifo($mapfifo, 0600);
6945     my $fifofh;
6946     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6947
6948     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6949
6950     my $oldtimeout;
6951     my $timeout = 5;
6952
6953     my $devinfo = {}; # info about drives included in backup
6954     my $virtdev_hash = {}; # info about allocated drives
6955
6956     my $rpcenv = PVE::RPCEnvironment::get();
6957
6958     my $conffile = PVE::QemuConfig->config_file($vmid);
6959
6960     # Note: $oldconf is undef if VM does not exist
6961     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6962     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6963     my $new_conf_raw = '';
6964
6965     my %storage_limits;
6966
6967     my $print_devmap = sub {
6968         my $cfgfn = "$tmpdir/qemu-server.conf";
6969
6970         # we can read the config - that is already extracted
6971         my $fh = IO::File->new($cfgfn, "r") ||
6972             die "unable to read qemu-server.conf - $!\n";
6973
6974         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6975         if (-f $fwcfgfn) {
6976             my $pve_firewall_dir = '/etc/pve/firewall';
6977             mkdir $pve_firewall_dir; # make sure the dir exists
6978             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6979         }
6980
6981         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6982
6983         foreach my $info (values %{$virtdev_hash}) {
6984             my $storeid = $info->{storeid};
6985             next if defined($storage_limits{$storeid});
6986
6987             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6988             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6989             $storage_limits{$storeid} = $limit * 1024;
6990         }
6991
6992         foreach my $devname (keys %$devinfo) {
6993             die "found no device mapping information for device '$devname'\n"
6994                 if !$devinfo->{$devname}->{virtdev};
6995         }
6996
6997         # create empty/temp config
6998         if ($oldconf) {
6999             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7000             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7001         }
7002
7003         # allocate volumes
7004         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7005
7006         # print restore information to $fifofh
7007         foreach my $virtdev (sort keys %$virtdev_hash) {
7008             my $d = $virtdev_hash->{$virtdev};
7009             next if $d->{is_cloudinit}; # no need to restore cloudinit
7010
7011             my $storeid = $d->{storeid};
7012             my $volid = $d->{volid};
7013
7014             my $map_opts = '';
7015             if (my $limit = $storage_limits{$storeid}) {
7016                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7017             }
7018
7019             my $write_zeros = 1;
7020             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7021                 $write_zeros = 0;
7022             }
7023
7024             my $path = PVE::Storage::path($cfg, $volid);
7025
7026             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7027
7028             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7029         }
7030
7031         $fh->seek(0, 0) || die "seek failed - $!\n";
7032
7033         my $cookie = { netcount => 0 };
7034         while (defined(my $line = <$fh>)) {
7035             $new_conf_raw .= restore_update_config_line(
7036                 $cookie,
7037                 $map,
7038                 $line,
7039                 $opts->{unique},
7040             );
7041         }
7042
7043         $fh->close();
7044     };
7045
7046     eval {
7047         # enable interrupts
7048         local $SIG{INT} =
7049             local $SIG{TERM} =
7050             local $SIG{QUIT} =
7051             local $SIG{HUP} =
7052             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7053         local $SIG{ALRM} = sub { die "got timeout\n"; };
7054
7055         $oldtimeout = alarm($timeout);
7056
7057         my $parser = sub {
7058             my $line = shift;
7059
7060             print "$line\n";
7061
7062             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7063                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7064                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7065             } elsif ($line =~ m/^CTIME: /) {
7066                 # we correctly received the vma config, so we can disable
7067                 # the timeout now for disk allocation (set to 10 minutes, so
7068                 # that we always timeout if something goes wrong)
7069                 alarm(600);
7070                 &$print_devmap();
7071                 print $fifofh "done\n";
7072                 my $tmp = $oldtimeout || 0;
7073                 $oldtimeout = undef;
7074                 alarm($tmp);
7075                 close($fifofh);
7076                 $fifofh = undef;
7077             }
7078         };
7079
7080         print "restore vma archive: $dbg_cmdstring\n";
7081         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7082     };
7083     my $err = $@;
7084
7085     alarm($oldtimeout) if $oldtimeout;
7086
7087     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7088
7089     close($fifofh) if $fifofh;
7090     unlink $mapfifo;
7091     rmtree $tmpdir;
7092
7093     if ($err) {
7094         $restore_destroy_volumes->($cfg, $virtdev_hash);
7095         die $err;
7096     }
7097
7098     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7099
7100     PVE::Cluster::cfs_update(); # make sure we read new file
7101
7102     eval { rescan($vmid, 1); };
7103     warn $@ if $@;
7104
7105     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7106 }
7107
7108 sub restore_tar_archive {
7109     my ($archive, $vmid, $user, $opts) = @_;
7110
7111     if ($archive ne '-') {
7112         my $firstfile = tar_archive_read_firstfile($archive);
7113         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7114             if $firstfile ne 'qemu-server.conf';
7115     }
7116
7117     my $storecfg = PVE::Storage::config();
7118
7119     # avoid zombie disks when restoring over an existing VM -> cleanup first
7120     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7121     # skiplock=1 because qmrestore has set the 'create' lock itself already
7122     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7123     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7124
7125     my $tocmd = "/usr/lib/qemu-server/qmextract";
7126
7127     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7128     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7129     $tocmd .= ' --prealloc' if $opts->{prealloc};
7130     $tocmd .= ' --info' if $opts->{info};
7131
7132     # tar option "xf" does not autodetect compression when read from STDIN,
7133     # so we pipe to zcat
7134     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7135         PVE::Tools::shellquote("--to-command=$tocmd");
7136
7137     my $tmpdir = "/var/tmp/vzdumptmp$$";
7138     mkpath $tmpdir;
7139
7140     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7141     local $ENV{VZDUMP_VMID} = $vmid;
7142     local $ENV{VZDUMP_USER} = $user;
7143
7144     my $conffile = PVE::QemuConfig->config_file($vmid);
7145     my $new_conf_raw = '';
7146
7147     # disable interrupts (always do cleanups)
7148     local $SIG{INT} =
7149         local $SIG{TERM} =
7150         local $SIG{QUIT} =
7151         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7152
7153     eval {
7154         # enable interrupts
7155         local $SIG{INT} =
7156             local $SIG{TERM} =
7157             local $SIG{QUIT} =
7158             local $SIG{HUP} =
7159             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7160
7161         if ($archive eq '-') {
7162             print "extracting archive from STDIN\n";
7163             run_command($cmd, input => "<&STDIN");
7164         } else {
7165             print "extracting archive '$archive'\n";
7166             run_command($cmd);
7167         }
7168
7169         return if $opts->{info};
7170
7171         # read new mapping
7172         my $map = {};
7173         my $statfile = "$tmpdir/qmrestore.stat";
7174         if (my $fd = IO::File->new($statfile, "r")) {
7175             while (defined (my $line = <$fd>)) {
7176                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7177                     $map->{$1} = $2 if $1;
7178                 } else {
7179                     print STDERR "unable to parse line in statfile - $line\n";
7180                 }
7181             }
7182             $fd->close();
7183         }
7184
7185         my $confsrc = "$tmpdir/qemu-server.conf";
7186
7187         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7188
7189         my $cookie = { netcount => 0 };
7190         while (defined (my $line = <$srcfd>)) {
7191             $new_conf_raw .= restore_update_config_line(
7192                 $cookie,
7193                 $map,
7194                 $line,
7195                 $opts->{unique},
7196             );
7197         }
7198
7199         $srcfd->close();
7200     };
7201     if (my $err = $@) {
7202         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7203         die $err;
7204     }
7205
7206     rmtree $tmpdir;
7207
7208     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7209
7210     PVE::Cluster::cfs_update(); # make sure we read new file
7211
7212     eval { rescan($vmid, 1); };
7213     warn $@ if $@;
7214 };
7215
7216 sub foreach_storage_used_by_vm {
7217     my ($conf, $func) = @_;
7218
7219     my $sidhash = {};
7220
7221     PVE::QemuConfig->foreach_volume($conf, sub {
7222         my ($ds, $drive) = @_;
7223         return if drive_is_cdrom($drive);
7224
7225         my $volid = $drive->{file};
7226
7227         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7228         $sidhash->{$sid} = $sid if $sid;
7229     });
7230
7231     foreach my $sid (sort keys %$sidhash) {
7232         &$func($sid);
7233     }
7234 }
7235
7236 my $qemu_snap_storage = {
7237     rbd => 1,
7238 };
7239 sub do_snapshots_with_qemu {
7240     my ($storecfg, $volid, $deviceid) = @_;
7241
7242     return if $deviceid =~ m/tpmstate0/;
7243
7244     my $storage_name = PVE::Storage::parse_volume_id($volid);
7245     my $scfg = $storecfg->{ids}->{$storage_name};
7246     die "could not find storage '$storage_name'\n" if !defined($scfg);
7247
7248     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7249         return 1;
7250     }
7251
7252     if ($volid =~ m/\.(qcow2|qed)$/){
7253         return 1;
7254     }
7255
7256     return;
7257 }
7258
7259 sub qga_check_running {
7260     my ($vmid, $nowarn) = @_;
7261
7262     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7263     if ($@) {
7264         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7265         return 0;
7266     }
7267     return 1;
7268 }
7269
7270 sub template_create {
7271     my ($vmid, $conf, $disk) = @_;
7272
7273     my $storecfg = PVE::Storage::config();
7274
7275     PVE::QemuConfig->foreach_volume($conf, sub {
7276         my ($ds, $drive) = @_;
7277
7278         return if drive_is_cdrom($drive);
7279         return if $disk && $ds ne $disk;
7280
7281         my $volid = $drive->{file};
7282         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7283
7284         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7285         $drive->{file} = $voliddst;
7286         $conf->{$ds} = print_drive($drive);
7287         PVE::QemuConfig->write_config($vmid, $conf);
7288     });
7289 }
7290
7291 sub convert_iscsi_path {
7292     my ($path) = @_;
7293
7294     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7295         my $portal = $1;
7296         my $target = $2;
7297         my $lun = $3;
7298
7299         my $initiator_name = get_initiator_name();
7300
7301         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7302                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7303     }
7304
7305     die "cannot convert iscsi path '$path', unkown format\n";
7306 }
7307
7308 sub qemu_img_convert {
7309     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7310
7311     my $storecfg = PVE::Storage::config();
7312     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7313     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7314
7315     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7316
7317     my $cachemode;
7318     my $src_path;
7319     my $src_is_iscsi = 0;
7320     my $src_format;
7321
7322     if ($src_storeid) {
7323         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7324         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7325         $src_format = qemu_img_format($src_scfg, $src_volname);
7326         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7327         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7328         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7329     } elsif (-f $src_volid || -b $src_volid) {
7330         $src_path = $src_volid;
7331         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7332             $src_format = $1;
7333         }
7334     }
7335
7336     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7337
7338     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7339     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7340     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7341     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7342
7343     my $cmd = [];
7344     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7345     push @$cmd, '-l', "snapshot.name=$snapname"
7346         if $snapname && $src_format && $src_format eq "qcow2";
7347     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7348     push @$cmd, '-T', $cachemode if defined($cachemode);
7349
7350     if ($src_is_iscsi) {
7351         push @$cmd, '--image-opts';
7352         $src_path = convert_iscsi_path($src_path);
7353     } elsif ($src_format) {
7354         push @$cmd, '-f', $src_format;
7355     }
7356
7357     if ($dst_is_iscsi) {
7358         push @$cmd, '--target-image-opts';
7359         $dst_path = convert_iscsi_path($dst_path);
7360     } else {
7361         push @$cmd, '-O', $dst_format;
7362     }
7363
7364     push @$cmd, $src_path;
7365
7366     if (!$dst_is_iscsi && $is_zero_initialized) {
7367         push @$cmd, "zeroinit:$dst_path";
7368     } else {
7369         push @$cmd, $dst_path;
7370     }
7371
7372     my $parser = sub {
7373         my $line = shift;
7374         if($line =~ m/\((\S+)\/100\%\)/){
7375             my $percent = $1;
7376             my $transferred = int($size * $percent / 100);
7377             my $total_h = render_bytes($size, 1);
7378             my $transferred_h = render_bytes($transferred, 1);
7379
7380             print "transferred $transferred_h of $total_h ($percent%)\n";
7381         }
7382
7383     };
7384
7385     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7386     my $err = $@;
7387     die "copy failed: $err" if $err;
7388 }
7389
7390 sub qemu_img_format {
7391     my ($scfg, $volname) = @_;
7392
7393     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7394         return $1;
7395     } else {
7396         return "raw";
7397     }
7398 }
7399
7400 sub qemu_drive_mirror {
7401     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7402
7403     $jobs = {} if !$jobs;
7404
7405     my $qemu_target;
7406     my $format;
7407     $jobs->{"drive-$drive"} = {};
7408
7409     if ($dst_volid =~ /^nbd:/) {
7410         $qemu_target = $dst_volid;
7411         $format = "nbd";
7412     } else {
7413         my $storecfg = PVE::Storage::config();
7414         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7415
7416         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7417
7418         $format = qemu_img_format($dst_scfg, $dst_volname);
7419
7420         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7421
7422         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7423     }
7424
7425     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7426     $opts->{format} = $format if $format;
7427
7428     if (defined($src_bitmap)) {
7429         $opts->{sync} = 'incremental';
7430         $opts->{bitmap} = $src_bitmap;
7431         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7432     }
7433
7434     if (defined($bwlimit)) {
7435         $opts->{speed} = $bwlimit * 1024;
7436         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7437     } else {
7438         print "drive mirror is starting for drive-$drive\n";
7439     }
7440
7441     # if a job already runs for this device we get an error, catch it for cleanup
7442     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7443     if (my $err = $@) {
7444         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7445         warn "$@\n" if $@;
7446         die "mirroring error: $err\n";
7447     }
7448
7449     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7450 }
7451
7452 # $completion can be either
7453 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7454 # 'cancel': wait until all jobs are ready, block-job-cancel them
7455 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7456 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7457 sub qemu_drive_mirror_monitor {
7458     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7459
7460     $completion //= 'complete';
7461     $op //= "mirror";
7462
7463     eval {
7464         my $err_complete = 0;
7465
7466         my $starttime = time ();
7467         while (1) {
7468             die "block job ('$op') timed out\n" if $err_complete > 300;
7469
7470             my $stats = mon_cmd($vmid, "query-block-jobs");
7471             my $ctime = time();
7472
7473             my $running_jobs = {};
7474             for my $stat (@$stats) {
7475                 next if $stat->{type} ne $op;
7476                 $running_jobs->{$stat->{device}} = $stat;
7477             }
7478
7479             my $readycounter = 0;
7480
7481             for my $job_id (sort keys %$jobs) {
7482                 my $job = $running_jobs->{$job_id};
7483
7484                 my $vanished = !defined($job);
7485                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7486                 if($complete || ($vanished && $completion eq 'auto')) {
7487                     print "$job_id: $op-job finished\n";
7488                     delete $jobs->{$job_id};
7489                     next;
7490                 }
7491
7492                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7493
7494                 my $busy = $job->{busy};
7495                 my $ready = $job->{ready};
7496                 if (my $total = $job->{len}) {
7497                     my $transferred = $job->{offset} || 0;
7498                     my $remaining = $total - $transferred;
7499                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7500
7501                     my $duration = $ctime - $starttime;
7502                     my $total_h = render_bytes($total, 1);
7503                     my $transferred_h = render_bytes($transferred, 1);
7504
7505                     my $status = sprintf(
7506                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7507                         render_duration($duration),
7508                     );
7509
7510                     if ($ready) {
7511                         if ($busy) {
7512                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7513                         } else {
7514                             $status .= ", ready";
7515                         }
7516                     }
7517                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7518                     $jobs->{$job_id}->{ready} = $ready;
7519                 }
7520
7521                 $readycounter++ if $job->{ready};
7522             }
7523
7524             last if scalar(keys %$jobs) == 0;
7525
7526             if ($readycounter == scalar(keys %$jobs)) {
7527                 print "all '$op' jobs are ready\n";
7528
7529                 # do the complete later (or has already been done)
7530                 last if $completion eq 'skip' || $completion eq 'auto';
7531
7532                 if ($vmiddst && $vmiddst != $vmid) {
7533                     my $agent_running = $qga && qga_check_running($vmid);
7534                     if ($agent_running) {
7535                         print "freeze filesystem\n";
7536                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7537                         warn $@ if $@;
7538                     } else {
7539                         print "suspend vm\n";
7540                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7541                         warn $@ if $@;
7542                     }
7543
7544                     # if we clone a disk for a new target vm, we don't switch the disk
7545                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7546
7547                     if ($agent_running) {
7548                         print "unfreeze filesystem\n";
7549                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7550                         warn $@ if $@;
7551                     } else {
7552                         print "resume vm\n";
7553                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7554                         warn $@ if $@;
7555                     }
7556
7557                     last;
7558                 } else {
7559
7560                     for my $job_id (sort keys %$jobs) {
7561                         # try to switch the disk if source and destination are on the same guest
7562                         print "$job_id: Completing block job_id...\n";
7563
7564                         my $op;
7565                         if ($completion eq 'complete') {
7566                             $op = 'block-job-complete';
7567                         } elsif ($completion eq 'cancel') {
7568                             $op = 'block-job-cancel';
7569                         } else {
7570                             die "invalid completion value: $completion\n";
7571                         }
7572                         eval { mon_cmd($vmid, $op, device => $job_id) };
7573                         if ($@ =~ m/cannot be completed/) {
7574                             print "$job_id: block job cannot be completed, trying again.\n";
7575                             $err_complete++;
7576                         }else {
7577                             print "$job_id: Completed successfully.\n";
7578                             $jobs->{$job_id}->{complete} = 1;
7579                         }
7580                     }
7581                 }
7582             }
7583             sleep 1;
7584         }
7585     };
7586     my $err = $@;
7587
7588     if ($err) {
7589         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7590         die "block job ($op) error: $err";
7591     }
7592 }
7593
7594 sub qemu_blockjobs_cancel {
7595     my ($vmid, $jobs) = @_;
7596
7597     foreach my $job (keys %$jobs) {
7598         print "$job: Cancelling block job\n";
7599         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7600         $jobs->{$job}->{cancel} = 1;
7601     }
7602
7603     while (1) {
7604         my $stats = mon_cmd($vmid, "query-block-jobs");
7605
7606         my $running_jobs = {};
7607         foreach my $stat (@$stats) {
7608             $running_jobs->{$stat->{device}} = $stat;
7609         }
7610
7611         foreach my $job (keys %$jobs) {
7612
7613             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7614                 print "$job: Done.\n";
7615                 delete $jobs->{$job};
7616             }
7617         }
7618
7619         last if scalar(keys %$jobs) == 0;
7620
7621         sleep 1;
7622     }
7623 }
7624
7625 sub clone_disk {
7626     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7627
7628     my ($vmid, $running) = $source->@{qw(vmid running)};
7629     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7630
7631     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7632     my ($storage, $format) = $dest->@{qw(storage format)};
7633
7634     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7635
7636     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7637         die "cloning from/to EFI disk requires EFI disk\n"
7638             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7639         die "cloning from/to TPM state requires TPM state\n"
7640             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7641
7642         # This would lead to two device nodes in QEMU pointing to the same backing image!
7643         die "cannot change drive name when cloning disk from/to the same VM\n"
7644             if $use_drive_mirror && $vmid == $newvmid;
7645     }
7646
7647     die "cannot move TPM state while VM is running\n"
7648         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7649
7650     my $newvolid;
7651
7652     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7653     print "$src_drivename " if $src_drivename;
7654     print "($drive->{file})\n";
7655
7656     if (!$full) {
7657         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7658         push @$newvollist, $newvolid;
7659     } else {
7660
7661         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7662         $storeid = $storage if $storage;
7663
7664         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7665
7666         my $name = undef;
7667         my $size = undef;
7668         if (drive_is_cloudinit($drive)) {
7669             $name = "vm-$newvmid-cloudinit";
7670             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7671             if ($scfg->{path}) {
7672                 $name .= ".$dst_format";
7673             }
7674             $snapname = undef;
7675             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7676         } elsif ($dst_drivename eq 'efidisk0') {
7677             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7678         } elsif ($dst_drivename eq 'tpmstate0') {
7679             $dst_format = 'raw';
7680             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7681         } else {
7682             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7683         }
7684         $newvolid = PVE::Storage::vdisk_alloc(
7685             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7686         );
7687         push @$newvollist, $newvolid;
7688
7689         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7690
7691         if (drive_is_cloudinit($drive)) {
7692             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7693             # if this is the case, we have to complete any block-jobs still there from
7694             # previous drive-mirrors
7695             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7696                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7697             }
7698             goto no_data_clone;
7699         }
7700
7701         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7702         if ($use_drive_mirror) {
7703             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7704                 $completion, $qga, $bwlimit);
7705         } else {
7706             # TODO: handle bwlimits
7707             if ($dst_drivename eq 'efidisk0') {
7708                 # the relevant data on the efidisk may be smaller than the source
7709                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7710                 # that is given by the OVMF_VARS.fd
7711                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7712                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7713
7714                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7715
7716                 # better for Ceph if block size is not too small, see bug #3324
7717                 my $bs = 1024*1024;
7718
7719                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7720
7721                 if ($src_format eq 'qcow2' && $snapname) {
7722                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7723                         if !min_version(kvm_user_version(), 6, 2);
7724                     push $cmd->@*, '-l', $snapname;
7725                 }
7726                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7727                 run_command($cmd);
7728             } else {
7729                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7730             }
7731         }
7732     }
7733
7734 no_data_clone:
7735     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7736
7737     my $disk = dclone($drive);
7738     delete $disk->{format};
7739     $disk->{file} = $newvolid;
7740     $disk->{size} = $size if defined($size);
7741
7742     return $disk;
7743 }
7744
7745 sub get_running_qemu_version {
7746     my ($vmid) = @_;
7747     my $res = mon_cmd($vmid, "query-version");
7748     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7749 }
7750
7751 sub qemu_use_old_bios_files {
7752     my ($machine_type) = @_;
7753
7754     return if !$machine_type;
7755
7756     my $use_old_bios_files = undef;
7757
7758     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7759         $machine_type = $1;
7760         $use_old_bios_files = 1;
7761     } else {
7762         my $version = extract_version($machine_type, kvm_user_version());
7763         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7764         # load new efi bios files on migration. So this hack is required to allow
7765         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7766         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7767         $use_old_bios_files = !min_version($version, 2, 4);
7768     }
7769
7770     return ($use_old_bios_files, $machine_type);
7771 }
7772
7773 sub get_efivars_size {
7774     my ($conf, $efidisk) = @_;
7775
7776     my $arch = get_vm_arch($conf);
7777     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7778     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7779     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7780     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7781     return -s $ovmf_vars;
7782 }
7783
7784 sub update_efidisk_size {
7785     my ($conf) = @_;
7786
7787     return if !defined($conf->{efidisk0});
7788
7789     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7790     $disk->{size} = get_efivars_size($conf);
7791     $conf->{efidisk0} = print_drive($disk);
7792
7793     return;
7794 }
7795
7796 sub update_tpmstate_size {
7797     my ($conf) = @_;
7798
7799     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7800     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7801     $conf->{tpmstate0} = print_drive($disk);
7802 }
7803
7804 sub create_efidisk($$$$$$$) {
7805     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7806
7807     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7808     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7809
7810     my $vars_size_b = -s $ovmf_vars;
7811     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7812     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7813     PVE::Storage::activate_volumes($storecfg, [$volid]);
7814
7815     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7816     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7817
7818     return ($volid, $size/1024);
7819 }
7820
7821 sub vm_iothreads_list {
7822     my ($vmid) = @_;
7823
7824     my $res = mon_cmd($vmid, 'query-iothreads');
7825
7826     my $iothreads = {};
7827     foreach my $iothread (@$res) {
7828         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7829     }
7830
7831     return $iothreads;
7832 }
7833
7834 sub scsihw_infos {
7835     my ($conf, $drive) = @_;
7836
7837     my $maxdev = 0;
7838
7839     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7840         $maxdev = 7;
7841     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7842         $maxdev = 1;
7843     } else {
7844         $maxdev = 256;
7845     }
7846
7847     my $controller = int($drive->{index} / $maxdev);
7848     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7849         ? "virtioscsi"
7850         : "scsihw";
7851
7852     return ($maxdev, $controller, $controller_prefix);
7853 }
7854
7855 sub windows_version {
7856     my ($ostype) = @_;
7857
7858     return 0 if !$ostype;
7859
7860     my $winversion = 0;
7861
7862     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7863         $winversion = 5;
7864     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7865         $winversion = 6;
7866     } elsif ($ostype =~ m/^win(\d+)$/) {
7867         $winversion = $1;
7868     }
7869
7870     return $winversion;
7871 }
7872
7873 sub resolve_dst_disk_format {
7874         my ($storecfg, $storeid, $src_volname, $format) = @_;
7875         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7876
7877         if (!$format) {
7878             # if no target format is specified, use the source disk format as hint
7879             if ($src_volname) {
7880                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7881                 $format = qemu_img_format($scfg, $src_volname);
7882             } else {
7883                 return $defFormat;
7884             }
7885         }
7886
7887         # test if requested format is supported - else use default
7888         my $supported = grep { $_ eq $format } @$validFormats;
7889         $format = $defFormat if !$supported;
7890         return $format;
7891 }
7892
7893 # NOTE: if this logic changes, please update docs & possibly gui logic
7894 sub find_vmstate_storage {
7895     my ($conf, $storecfg) = @_;
7896
7897     # first, return storage from conf if set
7898     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7899
7900     my ($target, $shared, $local);
7901
7902     foreach_storage_used_by_vm($conf, sub {
7903         my ($sid) = @_;
7904         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7905         my $dst = $scfg->{shared} ? \$shared : \$local;
7906         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7907     });
7908
7909     # second, use shared storage where VM has at least one disk
7910     # third, use local storage where VM has at least one disk
7911     # fall back to local storage
7912     $target = $shared // $local // 'local';
7913
7914     return $target;
7915 }
7916
7917 sub generate_uuid {
7918     my ($uuid, $uuid_str);
7919     UUID::generate($uuid);
7920     UUID::unparse($uuid, $uuid_str);
7921     return $uuid_str;
7922 }
7923
7924 sub generate_smbios1_uuid {
7925     return "uuid=".generate_uuid();
7926 }
7927
7928 sub nbd_stop {
7929     my ($vmid) = @_;
7930
7931     mon_cmd($vmid, 'nbd-server-stop');
7932 }
7933
7934 sub create_reboot_request {
7935     my ($vmid) = @_;
7936     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7937         or die "failed to create reboot trigger file: $!\n";
7938     close($fh);
7939 }
7940
7941 sub clear_reboot_request {
7942     my ($vmid) = @_;
7943     my $path = "/run/qemu-server/$vmid.reboot";
7944     my $res = 0;
7945
7946     $res = unlink($path);
7947     die "could not remove reboot request for $vmid: $!"
7948         if !$res && $! != POSIX::ENOENT;
7949
7950     return $res;
7951 }
7952
7953 sub bootorder_from_legacy {
7954     my ($conf, $bootcfg) = @_;
7955
7956     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7957     my $bootindex_hash = {};
7958     my $i = 1;
7959     foreach my $o (split(//, $boot)) {
7960         $bootindex_hash->{$o} = $i*100;
7961         $i++;
7962     }
7963
7964     my $bootorder = {};
7965
7966     PVE::QemuConfig->foreach_volume($conf, sub {
7967         my ($ds, $drive) = @_;
7968
7969         if (drive_is_cdrom ($drive, 1)) {
7970             if ($bootindex_hash->{d}) {
7971                 $bootorder->{$ds} = $bootindex_hash->{d};
7972                 $bootindex_hash->{d} += 1;
7973             }
7974         } elsif ($bootindex_hash->{c}) {
7975             $bootorder->{$ds} = $bootindex_hash->{c}
7976                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7977             $bootindex_hash->{c} += 1;
7978         }
7979     });
7980
7981     if ($bootindex_hash->{n}) {
7982         for (my $i = 0; $i < $MAX_NETS; $i++) {
7983             my $netname = "net$i";
7984             next if !$conf->{$netname};
7985             $bootorder->{$netname} = $bootindex_hash->{n};
7986             $bootindex_hash->{n} += 1;
7987         }
7988     }
7989
7990     return $bootorder;
7991 }
7992
7993 # Generate default device list for 'boot: order=' property. Matches legacy
7994 # default boot order, but with explicit device names. This is important, since
7995 # the fallback for when neither 'order' nor the old format is specified relies
7996 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7997 sub get_default_bootdevices {
7998     my ($conf) = @_;
7999
8000     my @ret = ();
8001
8002     # harddisk
8003     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8004     push @ret, $first if $first;
8005
8006     # cdrom
8007     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8008     push @ret, $first if $first;
8009
8010     # network
8011     for (my $i = 0; $i < $MAX_NETS; $i++) {
8012         my $netname = "net$i";
8013         next if !$conf->{$netname};
8014         push @ret, $netname;
8015         last;
8016     }
8017
8018     return \@ret;
8019 }
8020
8021 sub device_bootorder {
8022     my ($conf) = @_;
8023
8024     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8025
8026     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8027
8028     my $bootorder = {};
8029     if (!defined($boot) || $boot->{legacy}) {
8030         $bootorder = bootorder_from_legacy($conf, $boot);
8031     } elsif ($boot->{order}) {
8032         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8033         for my $dev (PVE::Tools::split_list($boot->{order})) {
8034             $bootorder->{$dev} = $i++;
8035         }
8036     }
8037
8038     return $bootorder;
8039 }
8040
8041 sub register_qmeventd_handle {
8042     my ($vmid) = @_;
8043
8044     my $fh;
8045     my $peer = "/var/run/qmeventd.sock";
8046     my $count = 0;
8047
8048     for (;;) {
8049         $count++;
8050         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8051         last if $fh;
8052         if ($! != EINTR && $! != EAGAIN) {
8053             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8054         }
8055         if ($count > 4) {
8056             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8057               . "after $count retries\n";
8058         }
8059         usleep(25000);
8060     }
8061
8062     # send handshake to mark VM as backing up
8063     print $fh to_json({vzdump => {vmid => "$vmid"}});
8064
8065     # return handle to be closed later when inhibit is no longer required
8066     return $fh;
8067 }
8068
8069 # bash completion helper
8070
8071 sub complete_backup_archives {
8072     my ($cmdname, $pname, $cvalue) = @_;
8073
8074     my $cfg = PVE::Storage::config();
8075
8076     my $storeid;
8077
8078     if ($cvalue =~ m/^([^:]+):/) {
8079         $storeid = $1;
8080     }
8081
8082     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8083
8084     my $res = [];
8085     foreach my $id (keys %$data) {
8086         foreach my $item (@{$data->{$id}}) {
8087             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8088             push @$res, $item->{volid} if defined($item->{volid});
8089         }
8090     }
8091
8092     return $res;
8093 }
8094
8095 my $complete_vmid_full = sub {
8096     my ($running) = @_;
8097
8098     my $idlist = vmstatus();
8099
8100     my $res = [];
8101
8102     foreach my $id (keys %$idlist) {
8103         my $d = $idlist->{$id};
8104         if (defined($running)) {
8105             next if $d->{template};
8106             next if $running && $d->{status} ne 'running';
8107             next if !$running && $d->{status} eq 'running';
8108         }
8109         push @$res, $id;
8110
8111     }
8112     return $res;
8113 };
8114
8115 sub complete_vmid {
8116     return &$complete_vmid_full();
8117 }
8118
8119 sub complete_vmid_stopped {
8120     return &$complete_vmid_full(0);
8121 }
8122
8123 sub complete_vmid_running {
8124     return &$complete_vmid_full(1);
8125 }
8126
8127 sub complete_storage {
8128
8129     my $cfg = PVE::Storage::config();
8130     my $ids = $cfg->{ids};
8131
8132     my $res = [];
8133     foreach my $sid (keys %$ids) {
8134         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8135         next if !$ids->{$sid}->{content}->{images};
8136         push @$res, $sid;
8137     }
8138
8139     return $res;
8140 }
8141
8142 sub complete_migration_storage {
8143     my ($cmd, $param, $current_value, $all_args) = @_;
8144
8145     my $targetnode = @$all_args[1];
8146
8147     my $cfg = PVE::Storage::config();
8148     my $ids = $cfg->{ids};
8149
8150     my $res = [];
8151     foreach my $sid (keys %$ids) {
8152         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8153         next if !$ids->{$sid}->{content}->{images};
8154         push @$res, $sid;
8155     }
8156
8157     return $res;
8158 }
8159
8160 sub vm_is_paused {
8161     my ($vmid) = @_;
8162     my $qmpstatus = eval {
8163         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8164         mon_cmd($vmid, "query-status");
8165     };
8166     warn "$@\n" if $@;
8167     return $qmpstatus && $qmpstatus->{status} eq "paused";
8168 }
8169
8170 sub check_volume_storage_type {
8171     my ($storecfg, $vol) = @_;
8172
8173     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8174     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8175     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8176
8177     die "storage '$storeid' does not support content-type '$vtype'\n"
8178         if !$scfg->{content}->{$vtype};
8179
8180     return 1;
8181 }
8182
8183 1;