PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. The default is read from the"
 371             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.",
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.",
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1040
1041     return verify_volume_id_or_absolute_path($volid, $noerr);
1042 }
1043
1044 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1045 sub verify_volume_id_or_absolute_path {
1046     my ($volid, $noerr) = @_;
1047
1048     return $volid if $volid =~ m|^/|;
1049
1050     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1051     if ($@) {
1052         return if $noerr;
1053         die $@;
1054     }
1055     return $volid;
1056 }
1057
1058 my $usb_fmt = {
1059     host => {
1060         default_key => 1,
1061         type => 'string', format => 'pve-qm-usb-device',
1062         format_description => 'HOSTUSBDEVICE|spice',
1063         description => <<EODESCR,
1064 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1065
1066  'bus-port(.port)*' (decimal numbers) or
1067  'vendor_id:product_id' (hexadeciaml numbers) or
1068  'spice'
1069
1070 You can use the 'lsusb -t' command to list existing usb devices.
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1073 machines - use with special care.
1074
1075 The value 'spice' can be used to add a usb redirection devices for spice.
1076 EODESCR
1077     },
1078     usb3 => {
1079         optional => 1,
1080         type => 'boolean',
1081         description => "Specifies whether if given host option is a USB3 device or port.",
1082         default => 0,
1083     },
1084 };
1085
1086 my $usbdesc = {
1087     optional => 1,
1088     type => 'string', format => $usb_fmt,
1089     description => "Configure an USB device (n is 0 to 4).",
1090 };
1091 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1092
1093 my $serialdesc = {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '(/dev/.+|socket)',
1097         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1098         verbose_description =>  <<EODESCR,
1099 Create a serial device inside the VM (n is 0 to 3), and pass through a
1100 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1101 host side (use 'qm terminal' to open a terminal connection).
1102
1103 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1104 use with special care.
1105
1106 CAUTION: Experimental! User reported problems with this option.
1107 EODESCR
1108 };
1109
1110 my $paralleldesc= {
1111         optional => 1,
1112         type => 'string',
1113         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1114         description =>  "Map host parallel devices (n is 0 to 2).",
1115         verbose_description =>  <<EODESCR,
1116 Map host parallel devices (n is 0 to 2).
1117
1118 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1119 machines - use with special care.
1120
1121 CAUTION: Experimental! User reported problems with this option.
1122 EODESCR
1123 };
1124
1125 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1126     $confdesc->{"parallel$i"} = $paralleldesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1130     $confdesc->{"serial$i"} = $serialdesc;
1131 }
1132
1133 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1134     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1135 }
1136
1137 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1138     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1139 }
1140
1141 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1142     $confdesc->{"usb$i"} = $usbdesc;
1143 }
1144
1145 my $boot_fmt = {
1146     legacy => {
1147         optional => 1,
1148         default_key => 1,
1149         type => 'string',
1150         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1151                      . " Deprecated, use 'order=' instead.",
1152         pattern => '[acdn]{1,4}',
1153         format_description => "[acdn]{1,4}",
1154
1155         # note: this is also the fallback if boot: is not given at all
1156         default => 'cdn',
1157     },
1158     order => {
1159         optional => 1,
1160         type => 'string',
1161         format => 'pve-qm-bootdev-list',
1162         format_description => "device[;device...]",
1163         description => <<EODESC,
1164 The guest will attempt to boot from devices in the order they appear here.
1165
1166 Disks, optical drives and passed-through storage USB devices will be directly
1167 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1168 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1169
1170 Note that only devices in this list will be marked as bootable and thus loaded
1171 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1172 (e.g. software-raid), you need to specify all of them here.
1173
1174 Overrides the deprecated 'legacy=[acdn]*' value when given.
1175 EODESC
1176     },
1177 };
1178 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1179
1180 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1181 sub verify_bootdev {
1182     my ($dev, $noerr) = @_;
1183
1184     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1185     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1186
1187     my $check = sub {
1188         my ($base) = @_;
1189         return 0 if $dev !~ m/^$base\d+$/;
1190         return 0 if !$confdesc->{$dev};
1191         return 1;
1192     };
1193
1194     return $dev if $check->("net");
1195     return $dev if $check->("usb");
1196     return $dev if $check->("hostpci");
1197
1198     return if $noerr;
1199     die "invalid boot device '$dev'\n";
1200 }
1201
1202 sub print_bootorder {
1203     my ($devs) = @_;
1204     return "" if !@$devs;
1205     my $data = { order => join(';', @$devs) };
1206     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1207 }
1208
1209 my $kvm_api_version = 0;
1210
1211 sub kvm_version {
1212     return $kvm_api_version if $kvm_api_version;
1213
1214     open my $fh, '<', '/dev/kvm' or return;
1215
1216     # 0xae00 => KVM_GET_API_VERSION
1217     $kvm_api_version = ioctl($fh, 0xae00, 0);
1218     close($fh);
1219
1220     return $kvm_api_version;
1221 }
1222
1223 my $kvm_user_version = {};
1224 my $kvm_mtime = {};
1225
1226 sub kvm_user_version {
1227     my ($binary) = @_;
1228
1229     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1230     my $st = stat($binary);
1231
1232     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1233     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1234         $cachedmtime == $st->mtime;
1235
1236     $kvm_user_version->{$binary} = 'unknown';
1237     $kvm_mtime->{$binary} = $st->mtime;
1238
1239     my $code = sub {
1240         my $line = shift;
1241         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1242             $kvm_user_version->{$binary} = $2;
1243         }
1244     };
1245
1246     eval { run_command([$binary, '--version'], outfunc => $code); };
1247     warn $@ if $@;
1248
1249     return $kvm_user_version->{$binary};
1250
1251 }
1252 my sub extract_version {
1253     my ($machine_type, $version) = @_;
1254     $version = kvm_user_version() if !defined($version);
1255     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1256 }
1257
1258 sub kernel_has_vhost_net {
1259     return -c '/dev/vhost-net';
1260 }
1261
1262 sub option_exists {
1263     my $key = shift;
1264     return defined($confdesc->{$key});
1265 }
1266
1267 my $cdrom_path;
1268 sub get_cdrom_path {
1269
1270     return  $cdrom_path if $cdrom_path;
1271
1272     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1273     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1274     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1275 }
1276
1277 sub get_iso_path {
1278     my ($storecfg, $vmid, $cdrom) = @_;
1279
1280     if ($cdrom eq 'cdrom') {
1281         return get_cdrom_path();
1282     } elsif ($cdrom eq 'none') {
1283         return '';
1284     } elsif ($cdrom =~ m|^/|) {
1285         return $cdrom;
1286     } else {
1287         return PVE::Storage::path($storecfg, $cdrom);
1288     }
1289 }
1290
1291 # try to convert old style file names to volume IDs
1292 sub filename_to_volume_id {
1293     my ($vmid, $file, $media) = @_;
1294
1295      if (!($file eq 'none' || $file eq 'cdrom' ||
1296           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1297
1298         return if $file =~ m|/|;
1299
1300         if ($media && $media eq 'cdrom') {
1301             $file = "local:iso/$file";
1302         } else {
1303             $file = "local:$vmid/$file";
1304         }
1305     }
1306
1307     return $file;
1308 }
1309
1310 sub verify_media_type {
1311     my ($opt, $vtype, $media) = @_;
1312
1313     return if !$media;
1314
1315     my $etype;
1316     if ($media eq 'disk') {
1317         $etype = 'images';
1318     } elsif ($media eq 'cdrom') {
1319         $etype = 'iso';
1320     } else {
1321         die "internal error";
1322     }
1323
1324     return if ($vtype eq $etype);
1325
1326     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1327 }
1328
1329 sub cleanup_drive_path {
1330     my ($opt, $storecfg, $drive) = @_;
1331
1332     # try to convert filesystem paths to volume IDs
1333
1334     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1335         ($drive->{file} !~ m|^/dev/.+|) &&
1336         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1337         ($drive->{file} !~ m/^\d+$/)) {
1338         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1339         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1340             if !$vtype;
1341         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1342         verify_media_type($opt, $vtype, $drive->{media});
1343         $drive->{file} = $volid;
1344     }
1345
1346     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1347 }
1348
1349 sub parse_hotplug_features {
1350     my ($data) = @_;
1351
1352     my $res = {};
1353
1354     return $res if $data eq '0';
1355
1356     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1357
1358     foreach my $feature (PVE::Tools::split_list($data)) {
1359         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1360             $res->{$1} = 1;
1361         } else {
1362             die "invalid hotplug feature '$feature'\n";
1363         }
1364     }
1365     return $res;
1366 }
1367
1368 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1369 sub pve_verify_hotplug_features {
1370     my ($value, $noerr) = @_;
1371
1372     return $value if parse_hotplug_features($value);
1373
1374     return if $noerr;
1375
1376     die "unable to parse hotplug option\n";
1377 }
1378
1379 sub scsi_inquiry {
1380     my($fh, $noerr) = @_;
1381
1382     my $SG_IO = 0x2285;
1383     my $SG_GET_VERSION_NUM = 0x2282;
1384
1385     my $versionbuf = "\x00" x 8;
1386     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1387     if (!$ret) {
1388         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1389         return;
1390     }
1391     my $version = unpack("I", $versionbuf);
1392     if ($version < 30000) {
1393         die "scsi generic interface too old\n"  if !$noerr;
1394         return;
1395     }
1396
1397     my $buf = "\x00" x 36;
1398     my $sensebuf = "\x00" x 8;
1399     my $cmd = pack("C x3 C x1", 0x12, 36);
1400
1401     # see /usr/include/scsi/sg.h
1402     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1403
1404     my $packet = pack(
1405         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1406     );
1407
1408     $ret = ioctl($fh, $SG_IO, $packet);
1409     if (!$ret) {
1410         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my @res = unpack($sg_io_hdr_t, $packet);
1415     if ($res[17] || $res[18]) {
1416         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1417         return;
1418     }
1419
1420     my $res = {};
1421     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1422
1423     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1424     $res->{type} &= 0x1F;
1425
1426     return $res;
1427 }
1428
1429 sub path_is_scsi {
1430     my ($path) = @_;
1431
1432     my $fh = IO::File->new("+<$path") || return;
1433     my $res = scsi_inquiry($fh, 1);
1434     close($fh);
1435
1436     return $res;
1437 }
1438
1439 sub print_tabletdevice_full {
1440     my ($conf, $arch) = @_;
1441
1442     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1443
1444     # we use uhci for old VMs because tablet driver was buggy in older qemu
1445     my $usbbus;
1446     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1447         $usbbus = 'ehci';
1448     } else {
1449         $usbbus = 'uhci';
1450     }
1451
1452     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1453 }
1454
1455 sub print_keyboarddevice_full {
1456     my ($conf, $arch) = @_;
1457
1458     return if $arch ne 'aarch64';
1459
1460     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1461 }
1462
1463 my sub get_drive_id {
1464     my ($drive) = @_;
1465     return "$drive->{interface}$drive->{index}";
1466 }
1467
1468 sub print_drivedevice_full {
1469     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1470
1471     my $device = '';
1472     my $maxdev = 0;
1473
1474     my $drive_id = get_drive_id($drive);
1475     if ($drive->{interface} eq 'virtio') {
1476         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1477         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1478         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1479     } elsif ($drive->{interface} eq 'scsi') {
1480
1481         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1482         my $unit = $drive->{index} % $maxdev;
1483         my $devicetype = 'hd';
1484         my $path = '';
1485         if (drive_is_cdrom($drive)) {
1486             $devicetype = 'cd';
1487         } else {
1488             if ($drive->{file} =~ m|^/|) {
1489                 $path = $drive->{file};
1490                 if (my $info = path_is_scsi($path)) {
1491                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1492                         $devicetype = 'block';
1493                     } elsif ($info->{type} == 1) { # tape
1494                         $devicetype = 'generic';
1495                     }
1496                 }
1497             } else {
1498                  $path = PVE::Storage::path($storecfg, $drive->{file});
1499             }
1500
1501             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1502             my $version = extract_version($machine_type, kvm_user_version());
1503             if ($path =~ m/^iscsi\:\/\// &&
1504                !min_version($version, 4, 1)) {
1505                 $devicetype = 'generic';
1506             }
1507         }
1508
1509         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1510             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1511         } else {
1512             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1513                 .",lun=$drive->{index}";
1514         }
1515         $device .= ",drive=drive-$drive_id,id=$drive_id";
1516
1517         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1518             $device .= ",rotation_rate=1";
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521
1522     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1523         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1524         my $controller = int($drive->{index} / $maxdev);
1525         my $unit = $drive->{index} % $maxdev;
1526         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1527
1528         $device = "ide-$devicetype";
1529         if ($drive->{interface} eq 'ide') {
1530             $device .= ",bus=ide.$controller,unit=$unit";
1531         } else {
1532             $device .= ",bus=ahci$controller.$unit";
1533         }
1534         $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536         if ($devicetype eq 'hd') {
1537             if (my $model = $drive->{model}) {
1538                 $model = URI::Escape::uri_unescape($model);
1539                 $device .= ",model=$model";
1540             }
1541             if ($drive->{ssd}) {
1542                 $device .= ",rotation_rate=1";
1543             }
1544         }
1545         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1546     } elsif ($drive->{interface} eq 'usb') {
1547         die "implement me";
1548         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1549     } else {
1550         die "unsupported interface type";
1551     }
1552
1553     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1554
1555     if (my $serial = $drive->{serial}) {
1556         $serial = URI::Escape::uri_unescape($serial);
1557         $device .= ",serial=$serial";
1558     }
1559
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_commandline_full {
1579     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format = $drive->{format};
1584     my $drive_id = get_drive_id($drive);
1585
1586     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1587     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1588
1589     if (drive_is_cdrom($drive)) {
1590         $path = get_iso_path($storecfg, $vmid, $volid);
1591         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1592     } else {
1593         if ($storeid) {
1594             $path = PVE::Storage::path($storecfg, $volid);
1595             $format //= qemu_img_format($scfg, $volname);
1596         } else {
1597             $path = $volid;
1598             $format //= "raw";
1599         }
1600    }
1601
1602    my $is_rbd = $path =~ m/^rbd:/;
1603
1604     my $opts = '';
1605     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1606     foreach my $o (@qemu_drive_options) {
1607         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1608     }
1609
1610     # snapshot only accepts on|off
1611     if (defined($drive->{snapshot})) {
1612         my $v = $drive->{snapshot} ? 'on' : 'off';
1613         $opts .= ",snapshot=$v";
1614     }
1615
1616     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1617         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1618     }
1619
1620     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1621         my ($dir, $qmpname) = @$type;
1622         if (my $v = $drive->{"mbps$dir"}) {
1623             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1624         }
1625         if (my $v = $drive->{"mbps${dir}_max"}) {
1626             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1627         }
1628         if (my $v = $drive->{"bps${dir}_max_length"}) {
1629             $opts .= ",throttling.bps$qmpname-max-length=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}"}) {
1632             $opts .= ",throttling.iops$qmpname=$v";
1633         }
1634         if (my $v = $drive->{"iops${dir}_max"}) {
1635             $opts .= ",throttling.iops$qmpname-max=$v";
1636         }
1637         if (my $v = $drive->{"iops${dir}_max_length"}) {
1638             $opts .= ",throttling.iops$qmpname-max-length=$v";
1639         }
1640     }
1641
1642     if ($pbs_name) {
1643         $format = "rbd" if $is_rbd;
1644         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1645             if !$format;
1646         $opts .= ",format=alloc-track,file.driver=$format";
1647     } elsif ($format) {
1648         $opts .= ",format=$format";
1649     }
1650
1651     my $cache_direct = 0;
1652
1653     if (my $cache = $drive->{cache}) {
1654         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1655     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1656         $opts .= ",cache=none";
1657         $cache_direct = 1;
1658     }
1659
1660     # io_uring with cache mode writeback or writethrough on krbd will hang...
1661     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1662
1663     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1664     # sometimes, just plain disable...
1665     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1666
1667     if (!$drive->{aio}) {
1668         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1669             # io_uring supports all cache modes
1670             $opts .= ",aio=io_uring";
1671         } else {
1672             # aio native works only with O_DIRECT
1673             if($cache_direct) {
1674                 $opts .= ",aio=native";
1675             } else {
1676                 $opts .= ",aio=threads";
1677             }
1678         }
1679     }
1680
1681     if (!drive_is_cdrom($drive)) {
1682         my $detectzeroes;
1683         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1684             $detectzeroes = 'off';
1685         } elsif ($drive->{discard}) {
1686             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1687         } else {
1688             # This used to be our default with discard not being specified:
1689             $detectzeroes = 'on';
1690         }
1691
1692         # note: 'detect-zeroes' works per blockdev and we want it to persist
1693         # after the alloc-track is removed, so put it on 'file' directly
1694         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1695         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1696     }
1697
1698     if ($pbs_name) {
1699         $opts .= ",backing=$pbs_name";
1700         $opts .= ",auto-remove=on";
1701     }
1702
1703     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1704     my $file_param = "file";
1705     if ($pbs_name) {
1706         # non-rbd drivers require the underlying file to be a seperate block
1707         # node, so add a second .file indirection
1708         $file_param .= ".file" if !$is_rbd;
1709         $file_param .= ".filename";
1710     }
1711     my $pathinfo = $path ? "$file_param=$path," : '';
1712
1713     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1714 }
1715
1716 sub print_pbs_blockdev {
1717     my ($pbs_conf, $pbs_name) = @_;
1718     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1719     $blockdev .= ",repository=$pbs_conf->{repository}";
1720     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1721     $blockdev .= ",archive=$pbs_conf->{archive}";
1722     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1723     return $blockdev;
1724 }
1725
1726 sub print_netdevice_full {
1727     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1728
1729     my $device = $net->{model};
1730     if ($net->{model} eq 'virtio') {
1731          $device = 'virtio-net-pci';
1732      };
1733
1734     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1735     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1736     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1737         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1738         # and out of each queue plus one config interrupt and control vector queue
1739         my $vectors = $net->{queues} * 2 + 2;
1740         $tmpstr .= ",vectors=$vectors,mq=on";
1741     }
1742     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1743
1744     if (my $mtu = $net->{mtu}) {
1745         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1746             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1747             if ($mtu == 1) {
1748                  $mtu = $bridge_mtu;
1749             } elsif ($mtu < 576) {
1750                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1751             } elsif ($mtu > $bridge_mtu) {
1752                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1753             }
1754             $tmpstr .= ",host_mtu=$mtu";
1755         } else {
1756             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1757         }
1758     }
1759
1760     if ($use_old_bios_files) {
1761         my $romfile;
1762         if ($device eq 'virtio-net-pci') {
1763             $romfile = 'pxe-virtio.rom';
1764         } elsif ($device eq 'e1000') {
1765             $romfile = 'pxe-e1000.rom';
1766         } elsif ($device eq 'e1000e') {
1767             $romfile = 'pxe-e1000e.rom';
1768         } elsif ($device eq 'ne2k') {
1769             $romfile = 'pxe-ne2k_pci.rom';
1770         } elsif ($device eq 'pcnet') {
1771             $romfile = 'pxe-pcnet.rom';
1772         } elsif ($device eq 'rtl8139') {
1773             $romfile = 'pxe-rtl8139.rom';
1774         }
1775         $tmpstr .= ",romfile=$romfile" if $romfile;
1776     }
1777
1778     return $tmpstr;
1779 }
1780
1781 sub print_netdev_full {
1782     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1783
1784     my $i = '';
1785     if ($netid =~ m/^net(\d+)$/) {
1786         $i = int($1);
1787     }
1788
1789     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1790
1791     my $ifname = "tap${vmid}i$i";
1792
1793     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1794     die "interface name '$ifname' is too long (max 15 character)\n"
1795         if length($ifname) >= 16;
1796
1797     my $vhostparam = '';
1798     if (is_native($arch)) {
1799         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1800     }
1801
1802     my $vmname = $conf->{name} || "vm$vmid";
1803
1804     my $netdev = "";
1805     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1806
1807     if ($net->{bridge}) {
1808         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1809             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1810     } else {
1811         $netdev = "type=user,id=$netid,hostname=$vmname";
1812     }
1813
1814     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1815
1816     return $netdev;
1817 }
1818
1819 my $vga_map = {
1820     'cirrus' => 'cirrus-vga',
1821     'std' => 'VGA',
1822     'vmware' => 'vmware-svga',
1823     'virtio' => 'virtio-vga',
1824 };
1825
1826 sub print_vga_device {
1827     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1828
1829     my $type = $vga_map->{$vga->{type}};
1830     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1831         $type = 'virtio-gpu';
1832     }
1833     my $vgamem_mb = $vga->{memory};
1834
1835     my $max_outputs = '';
1836     if ($qxlnum) {
1837         $type = $id ? 'qxl' : 'qxl-vga';
1838
1839         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1840             # set max outputs so linux can have up to 4 qxl displays with one device
1841             if (min_version($machine_version, 4, 1)) {
1842                 $max_outputs = ",max_outputs=4";
1843             }
1844         }
1845     }
1846
1847     die "no devicetype for $vga->{type}\n" if !$type;
1848
1849     my $memory = "";
1850     if ($vgamem_mb) {
1851         if ($vga->{type} eq 'virtio') {
1852             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1853             $memory = ",max_hostmem=$bytes";
1854         } elsif ($qxlnum) {
1855             # from https://www.spice-space.org/multiple-monitors.html
1856             $memory = ",vgamem_mb=$vga->{memory}";
1857             my $ram = $vgamem_mb * 4;
1858             my $vram = $vgamem_mb * 2;
1859             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1860         } else {
1861             $memory = ",vgamem_mb=$vga->{memory}";
1862         }
1863     } elsif ($qxlnum && $id) {
1864         $memory = ",ram_size=67108864,vram_size=33554432";
1865     }
1866
1867     my $edidoff = "";
1868     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1869         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1870     }
1871
1872     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1873     my $vgaid = "vga" . ($id // '');
1874     my $pciaddr;
1875     if ($q35 && $vgaid eq 'vga') {
1876         # the first display uses pcie.0 bus on q35 machines
1877         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1878     } else {
1879         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1880     }
1881
1882     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1883 }
1884
1885 sub parse_number_sets {
1886     my ($set) = @_;
1887     my $res = [];
1888     foreach my $part (split(/;/, $set)) {
1889         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1890             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1891             push @$res, [ $1, $2 ];
1892         } else {
1893             die "invalid range: $part\n";
1894         }
1895     }
1896     return $res;
1897 }
1898
1899 sub parse_numa {
1900     my ($data) = @_;
1901
1902     my $res = parse_property_string($numa_fmt, $data);
1903     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1904     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1905     return $res;
1906 }
1907
1908 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1909 sub parse_net {
1910     my ($data) = @_;
1911
1912     my $res = eval { parse_property_string($net_fmt, $data) };
1913     if ($@) {
1914         warn $@;
1915         return;
1916     }
1917     if (!defined($res->{macaddr})) {
1918         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1919         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1920     }
1921     return $res;
1922 }
1923
1924 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1925 sub parse_ipconfig {
1926     my ($data) = @_;
1927
1928     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1929     if ($@) {
1930         warn $@;
1931         return;
1932     }
1933
1934     if ($res->{gw} && !$res->{ip}) {
1935         warn 'gateway specified without specifying an IP address';
1936         return;
1937     }
1938     if ($res->{gw6} && !$res->{ip6}) {
1939         warn 'IPv6 gateway specified without specifying an IPv6 address';
1940         return;
1941     }
1942     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1943         warn 'gateway specified together with DHCP';
1944         return;
1945     }
1946     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1947         # gw6 + auto/dhcp
1948         warn "IPv6 gateway specified together with $res->{ip6} address";
1949         return;
1950     }
1951
1952     if (!$res->{ip} && !$res->{ip6}) {
1953         return { ip => 'dhcp', ip6 => 'dhcp' };
1954     }
1955
1956     return $res;
1957 }
1958
1959 sub print_net {
1960     my $net = shift;
1961
1962     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1963 }
1964
1965 sub add_random_macs {
1966     my ($settings) = @_;
1967
1968     foreach my $opt (keys %$settings) {
1969         next if $opt !~ m/^net(\d+)$/;
1970         my $net = parse_net($settings->{$opt});
1971         next if !$net;
1972         $settings->{$opt} = print_net($net);
1973     }
1974 }
1975
1976 sub vm_is_volid_owner {
1977     my ($storecfg, $vmid, $volid) = @_;
1978
1979     if ($volid !~  m|^/|) {
1980         my ($path, $owner);
1981         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1982         if ($owner && ($owner == $vmid)) {
1983             return 1;
1984         }
1985     }
1986
1987     return;
1988 }
1989
1990 sub vmconfig_register_unused_drive {
1991     my ($storecfg, $vmid, $conf, $drive) = @_;
1992
1993     if (drive_is_cloudinit($drive)) {
1994         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1995         warn $@ if $@;
1996     } elsif (!drive_is_cdrom($drive)) {
1997         my $volid = $drive->{file};
1998         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1999             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2000         }
2001     }
2002 }
2003
2004 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2005 my $smbios1_fmt = {
2006     uuid => {
2007         type => 'string',
2008         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2009         format_description => 'UUID',
2010         description => "Set SMBIOS1 UUID.",
2011         optional => 1,
2012     },
2013     version => {
2014         type => 'string',
2015         pattern => '[A-Za-z0-9+\/]+={0,2}',
2016         format_description => 'Base64 encoded string',
2017         description => "Set SMBIOS1 version.",
2018         optional => 1,
2019     },
2020     serial => {
2021         type => 'string',
2022         pattern => '[A-Za-z0-9+\/]+={0,2}',
2023         format_description => 'Base64 encoded string',
2024         description => "Set SMBIOS1 serial number.",
2025         optional => 1,
2026     },
2027     manufacturer => {
2028         type => 'string',
2029         pattern => '[A-Za-z0-9+\/]+={0,2}',
2030         format_description => 'Base64 encoded string',
2031         description => "Set SMBIOS1 manufacturer.",
2032         optional => 1,
2033     },
2034     product => {
2035         type => 'string',
2036         pattern => '[A-Za-z0-9+\/]+={0,2}',
2037         format_description => 'Base64 encoded string',
2038         description => "Set SMBIOS1 product ID.",
2039         optional => 1,
2040     },
2041     sku => {
2042         type => 'string',
2043         pattern => '[A-Za-z0-9+\/]+={0,2}',
2044         format_description => 'Base64 encoded string',
2045         description => "Set SMBIOS1 SKU string.",
2046         optional => 1,
2047     },
2048     family => {
2049         type => 'string',
2050         pattern => '[A-Za-z0-9+\/]+={0,2}',
2051         format_description => 'Base64 encoded string',
2052         description => "Set SMBIOS1 family string.",
2053         optional => 1,
2054     },
2055     base64 => {
2056         type => 'boolean',
2057         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2058         optional => 1,
2059     },
2060 };
2061
2062 sub parse_smbios1 {
2063     my ($data) = @_;
2064
2065     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2066     warn $@ if $@;
2067     return $res;
2068 }
2069
2070 sub print_smbios1 {
2071     my ($smbios1) = @_;
2072     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2073 }
2074
2075 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2076
2077 sub parse_watchdog {
2078     my ($value) = @_;
2079
2080     return if !$value;
2081
2082     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2083     warn $@ if $@;
2084     return $res;
2085 }
2086
2087 sub parse_guest_agent {
2088     my ($conf) = @_;
2089
2090     return {} if !defined($conf->{agent});
2091
2092     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2093     warn $@ if $@;
2094
2095     # if the agent is disabled ignore the other potentially set properties
2096     return {} if !$res->{enabled};
2097     return $res;
2098 }
2099
2100 sub get_qga_key {
2101     my ($conf, $key) = @_;
2102     return undef if !defined($conf->{agent});
2103
2104     my $agent = parse_guest_agent($conf);
2105     return $agent->{$key};
2106 }
2107
2108 sub parse_vga {
2109     my ($value) = @_;
2110
2111     return {} if !$value;
2112     my $res = eval { parse_property_string($vga_fmt, $value) };
2113     warn $@ if $@;
2114     return $res;
2115 }
2116
2117 sub parse_rng {
2118     my ($value) = @_;
2119
2120     return if !$value;
2121
2122     my $res = eval { parse_property_string($rng_fmt, $value) };
2123     warn $@ if $@;
2124     return $res;
2125 }
2126
2127 sub parse_meta_info {
2128     my ($value) = @_;
2129
2130     return if !$value;
2131
2132     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2133     warn $@ if $@;
2134     return $res;
2135 }
2136
2137 sub new_meta_info_string {
2138     my () = @_; # for now do not allow to override any value
2139
2140     return PVE::JSONSchema::print_property_string(
2141         {
2142             'creation-qemu' => kvm_user_version(),
2143             ctime => "". int(time()),
2144         },
2145         $meta_info_fmt
2146     );
2147 }
2148
2149 sub qemu_created_version_fixups {
2150     my ($conf, $forcemachine, $kvmver) = @_;
2151
2152     my $meta = parse_meta_info($conf->{meta}) // {};
2153     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2154
2155     # check if we need to apply some handling for VMs that always use the latest machine version but
2156     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2157     # would be required (we do not want to pin machine version for non-windows OS type)
2158     if (
2159         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2160         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2161         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2162         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2163     ) {
2164         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2165         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2166             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2167             # and thus with the predictable interface naming of systemd
2168             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2169         }
2170     }
2171     return;
2172 }
2173
2174 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2175 sub verify_usb_device {
2176     my ($value, $noerr) = @_;
2177
2178     return $value if parse_usb_device($value);
2179
2180     return if $noerr;
2181
2182     die "unable to parse usb device\n";
2183 }
2184
2185 # add JSON properties for create and set function
2186 sub json_config_properties {
2187     my ($prop, $with_disk_alloc) = @_;
2188
2189     my $skip_json_config_opts = {
2190         parent => 1,
2191         snaptime => 1,
2192         vmstate => 1,
2193         runningmachine => 1,
2194         runningcpu => 1,
2195         meta => 1,
2196     };
2197
2198     foreach my $opt (keys %$confdesc) {
2199         next if $skip_json_config_opts->{$opt};
2200
2201         if ($with_disk_alloc && is_valid_drivename($opt)) {
2202             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2203         } else {
2204             $prop->{$opt} = $confdesc->{$opt};
2205         }
2206     }
2207
2208     return $prop;
2209 }
2210
2211 # Properties that we can read from an OVF file
2212 sub json_ovf_properties {
2213     my $prop = {};
2214
2215     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2216         $prop->{$device} = {
2217             type => 'string',
2218             format => 'pve-volume-id-or-absolute-path',
2219             description => "Disk image that gets imported to $device",
2220             optional => 1,
2221         };
2222     }
2223
2224     $prop->{cores} = {
2225         type => 'integer',
2226         description => "The number of CPU cores.",
2227         optional => 1,
2228     };
2229     $prop->{memory} = {
2230         type => 'integer',
2231         description => "Amount of RAM for the VM in MB.",
2232         optional => 1,
2233     };
2234     $prop->{name} = {
2235         type => 'string',
2236         description => "Name of the VM.",
2237         optional => 1,
2238     };
2239
2240     return $prop;
2241 }
2242
2243 # return copy of $confdesc_cloudinit to generate documentation
2244 sub cloudinit_config_properties {
2245
2246     return dclone($confdesc_cloudinit);
2247 }
2248
2249 sub check_type {
2250     my ($key, $value) = @_;
2251
2252     die "unknown setting '$key'\n" if !$confdesc->{$key};
2253
2254     my $type = $confdesc->{$key}->{type};
2255
2256     if (!defined($value)) {
2257         die "got undefined value\n";
2258     }
2259
2260     if ($value =~ m/[\n\r]/) {
2261         die "property contains a line feed\n";
2262     }
2263
2264     if ($type eq 'boolean') {
2265         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2266         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2267         die "type check ('boolean') failed - got '$value'\n";
2268     } elsif ($type eq 'integer') {
2269         return int($1) if $value =~ m/^(\d+)$/;
2270         die "type check ('integer') failed - got '$value'\n";
2271     } elsif ($type eq 'number') {
2272         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2273         die "type check ('number') failed - got '$value'\n";
2274     } elsif ($type eq 'string') {
2275         if (my $fmt = $confdesc->{$key}->{format}) {
2276             PVE::JSONSchema::check_format($fmt, $value);
2277             return $value;
2278         }
2279         $value =~ s/^\"(.*)\"$/$1/;
2280         return $value;
2281     } else {
2282         die "internal error"
2283     }
2284 }
2285
2286 sub destroy_vm {
2287     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2288
2289     my $conf = PVE::QemuConfig->load_config($vmid);
2290
2291     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2292
2293     if ($conf->{template}) {
2294         # check if any base image is still used by a linked clone
2295         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2296                 my ($ds, $drive) = @_;
2297                 return if drive_is_cdrom($drive);
2298
2299                 my $volid = $drive->{file};
2300                 return if !$volid || $volid =~ m|^/|;
2301
2302                 die "base volume '$volid' is still in use by linked cloned\n"
2303                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2304
2305         });
2306     }
2307
2308     my $volids = {};
2309     my $remove_owned_drive = sub {
2310         my ($ds, $drive) = @_;
2311         return if drive_is_cdrom($drive, 1);
2312
2313         my $volid = $drive->{file};
2314         return if !$volid || $volid =~ m|^/|;
2315         return if $volids->{$volid};
2316
2317         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2318         return if !$path || !$owner || ($owner != $vmid);
2319
2320         $volids->{$volid} = 1;
2321         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2322         warn "Could not remove disk '$volid', check manually: $@" if $@;
2323     };
2324
2325     # only remove disks owned by this VM (referenced in the config)
2326     my $include_opts = {
2327         include_unused => 1,
2328         extra_keys => ['vmstate'],
2329     };
2330     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2331
2332     for my $snap (values %{$conf->{snapshots}}) {
2333         next if !defined($snap->{vmstate});
2334         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2335         next if !defined($drive);
2336         $remove_owned_drive->('vmstate', $drive);
2337     }
2338
2339     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2340
2341     if ($purge_unreferenced) { # also remove unreferenced disk
2342         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2343         PVE::Storage::foreach_volid($vmdisks, sub {
2344             my ($volid, $sid, $volname, $d) = @_;
2345             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2346             warn $@ if $@;
2347         });
2348     }
2349
2350     if (defined $replacement_conf) {
2351         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2352     } else {
2353         PVE::QemuConfig->destroy_config($vmid);
2354     }
2355 }
2356
2357 sub parse_vm_config {
2358     my ($filename, $raw, $strict) = @_;
2359
2360     return if !defined($raw);
2361
2362     my $res = {
2363         digest => Digest::SHA::sha1_hex($raw),
2364         snapshots => {},
2365         pending => {},
2366     };
2367
2368     my $handle_error = sub {
2369         my ($msg) = @_;
2370
2371         if ($strict) {
2372             die $msg;
2373         } else {
2374             warn $msg;
2375         }
2376     };
2377
2378     $filename =~ m|/qemu-server/(\d+)\.conf$|
2379         || die "got strange filename '$filename'";
2380
2381     my $vmid = $1;
2382
2383     my $conf = $res;
2384     my $descr;
2385     my $section = '';
2386
2387     my @lines = split(/\n/, $raw);
2388     foreach my $line (@lines) {
2389         next if $line =~ m/^\s*$/;
2390
2391         if ($line =~ m/^\[PENDING\]\s*$/i) {
2392             $section = 'pending';
2393             if (defined($descr)) {
2394                 $descr =~ s/\s+$//;
2395                 $conf->{description} = $descr;
2396             }
2397             $descr = undef;
2398             $conf = $res->{$section} = {};
2399             next;
2400
2401         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2402             $section = $1;
2403             if (defined($descr)) {
2404                 $descr =~ s/\s+$//;
2405                 $conf->{description} = $descr;
2406             }
2407             $descr = undef;
2408             $conf = $res->{snapshots}->{$section} = {};
2409             next;
2410         }
2411
2412         if ($line =~ m/^\#(.*)\s*$/) {
2413             $descr = '' if !defined($descr);
2414             $descr .= PVE::Tools::decode_text($1) . "\n";
2415             next;
2416         }
2417
2418         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2419             $descr = '' if !defined($descr);
2420             $descr .= PVE::Tools::decode_text($2);
2421         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2422             $conf->{snapstate} = $1;
2423         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2424             my $key = $1;
2425             my $value = $2;
2426             $conf->{$key} = $value;
2427         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2428             my $value = $1;
2429             if ($section eq 'pending') {
2430                 $conf->{delete} = $value; # we parse this later
2431             } else {
2432                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2433             }
2434         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2435             my $key = $1;
2436             my $value = $2;
2437             eval { $value = check_type($key, $value); };
2438             if ($@) {
2439                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2440             } else {
2441                 $key = 'ide2' if $key eq 'cdrom';
2442                 my $fmt = $confdesc->{$key}->{format};
2443                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2444                     my $v = parse_drive($key, $value);
2445                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2446                         $v->{file} = $volid;
2447                         $value = print_drive($v);
2448                     } else {
2449                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2450                         next;
2451                     }
2452                 }
2453
2454                 $conf->{$key} = $value;
2455             }
2456         } else {
2457             $handle_error->("vm $vmid - unable to parse config: $line\n");
2458         }
2459     }
2460
2461     if (defined($descr)) {
2462         $descr =~ s/\s+$//;
2463         $conf->{description} = $descr;
2464     }
2465     delete $res->{snapstate}; # just to be sure
2466
2467     return $res;
2468 }
2469
2470 sub write_vm_config {
2471     my ($filename, $conf) = @_;
2472
2473     delete $conf->{snapstate}; # just to be sure
2474
2475     if ($conf->{cdrom}) {
2476         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2477         $conf->{ide2} = $conf->{cdrom};
2478         delete $conf->{cdrom};
2479     }
2480
2481     # we do not use 'smp' any longer
2482     if ($conf->{sockets}) {
2483         delete $conf->{smp};
2484     } elsif ($conf->{smp}) {
2485         $conf->{sockets} = $conf->{smp};
2486         delete $conf->{cores};
2487         delete $conf->{smp};
2488     }
2489
2490     my $used_volids = {};
2491
2492     my $cleanup_config = sub {
2493         my ($cref, $pending, $snapname) = @_;
2494
2495         foreach my $key (keys %$cref) {
2496             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2497                 $key eq 'snapstate' || $key eq 'pending';
2498             my $value = $cref->{$key};
2499             if ($key eq 'delete') {
2500                 die "propertry 'delete' is only allowed in [PENDING]\n"
2501                     if !$pending;
2502                 # fixme: check syntax?
2503                 next;
2504             }
2505             eval { $value = check_type($key, $value); };
2506             die "unable to parse value of '$key' - $@" if $@;
2507
2508             $cref->{$key} = $value;
2509
2510             if (!$snapname && is_valid_drivename($key)) {
2511                 my $drive = parse_drive($key, $value);
2512                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2513             }
2514         }
2515     };
2516
2517     &$cleanup_config($conf);
2518
2519     &$cleanup_config($conf->{pending}, 1);
2520
2521     foreach my $snapname (keys %{$conf->{snapshots}}) {
2522         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2523         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2524     }
2525
2526     # remove 'unusedX' settings if we re-add a volume
2527     foreach my $key (keys %$conf) {
2528         my $value = $conf->{$key};
2529         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2530             delete $conf->{$key};
2531         }
2532     }
2533
2534     my $generate_raw_config = sub {
2535         my ($conf, $pending) = @_;
2536
2537         my $raw = '';
2538
2539         # add description as comment to top of file
2540         if (defined(my $descr = $conf->{description})) {
2541             if ($descr) {
2542                 foreach my $cl (split(/\n/, $descr)) {
2543                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2544                 }
2545             } else {
2546                 $raw .= "#\n" if $pending;
2547             }
2548         }
2549
2550         foreach my $key (sort keys %$conf) {
2551             next if $key =~ /^(digest|description|pending|snapshots)$/;
2552             $raw .= "$key: $conf->{$key}\n";
2553         }
2554         return $raw;
2555     };
2556
2557     my $raw = &$generate_raw_config($conf);
2558
2559     if (scalar(keys %{$conf->{pending}})){
2560         $raw .= "\n[PENDING]\n";
2561         $raw .= &$generate_raw_config($conf->{pending}, 1);
2562     }
2563
2564     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2565         $raw .= "\n[$snapname]\n";
2566         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2567     }
2568
2569     return $raw;
2570 }
2571
2572 sub load_defaults {
2573
2574     my $res = {};
2575
2576     # we use static defaults from our JSON schema configuration
2577     foreach my $key (keys %$confdesc) {
2578         if (defined(my $default = $confdesc->{$key}->{default})) {
2579             $res->{$key} = $default;
2580         }
2581     }
2582
2583     return $res;
2584 }
2585
2586 sub config_list {
2587     my $vmlist = PVE::Cluster::get_vmlist();
2588     my $res = {};
2589     return $res if !$vmlist || !$vmlist->{ids};
2590     my $ids = $vmlist->{ids};
2591     my $nodename = nodename();
2592
2593     foreach my $vmid (keys %$ids) {
2594         my $d = $ids->{$vmid};
2595         next if !$d->{node} || $d->{node} ne $nodename;
2596         next if !$d->{type} || $d->{type} ne 'qemu';
2597         $res->{$vmid}->{exists} = 1;
2598     }
2599     return $res;
2600 }
2601
2602 # test if VM uses local resources (to prevent migration)
2603 sub check_local_resources {
2604     my ($conf, $noerr) = @_;
2605
2606     my @loc_res = ();
2607
2608     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2609     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2610
2611     push @loc_res, "ivshmem" if $conf->{ivshmem};
2612
2613     foreach my $k (keys %$conf) {
2614         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2615         # sockets are safe: they will recreated be on the target side post-migrate
2616         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2617         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2618     }
2619
2620     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2621
2622     return \@loc_res;
2623 }
2624
2625 # check if used storages are available on all nodes (use by migrate)
2626 sub check_storage_availability {
2627     my ($storecfg, $conf, $node) = @_;
2628
2629     PVE::QemuConfig->foreach_volume($conf, sub {
2630         my ($ds, $drive) = @_;
2631
2632         my $volid = $drive->{file};
2633         return if !$volid;
2634
2635         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2636         return if !$sid;
2637
2638         # check if storage is available on both nodes
2639         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2640         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2641
2642         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2643
2644         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2645             if !$scfg->{content}->{$vtype};
2646    });
2647 }
2648
2649 # list nodes where all VM images are available (used by has_feature API)
2650 sub shared_nodes {
2651     my ($conf, $storecfg) = @_;
2652
2653     my $nodelist = PVE::Cluster::get_nodelist();
2654     my $nodehash = { map { $_ => 1 } @$nodelist };
2655     my $nodename = nodename();
2656
2657     PVE::QemuConfig->foreach_volume($conf, sub {
2658         my ($ds, $drive) = @_;
2659
2660         my $volid = $drive->{file};
2661         return if !$volid;
2662
2663         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2664         if ($storeid) {
2665             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2666             if ($scfg->{disable}) {
2667                 $nodehash = {};
2668             } elsif (my $avail = $scfg->{nodes}) {
2669                 foreach my $node (keys %$nodehash) {
2670                     delete $nodehash->{$node} if !$avail->{$node};
2671                 }
2672             } elsif (!$scfg->{shared}) {
2673                 foreach my $node (keys %$nodehash) {
2674                     delete $nodehash->{$node} if $node ne $nodename
2675                 }
2676             }
2677         }
2678     });
2679
2680     return $nodehash
2681 }
2682
2683 sub check_local_storage_availability {
2684     my ($conf, $storecfg) = @_;
2685
2686     my $nodelist = PVE::Cluster::get_nodelist();
2687     my $nodehash = { map { $_ => {} } @$nodelist };
2688
2689     PVE::QemuConfig->foreach_volume($conf, sub {
2690         my ($ds, $drive) = @_;
2691
2692         my $volid = $drive->{file};
2693         return if !$volid;
2694
2695         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2696         if ($storeid) {
2697             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2698
2699             if ($scfg->{disable}) {
2700                 foreach my $node (keys %$nodehash) {
2701                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2702                 }
2703             } elsif (my $avail = $scfg->{nodes}) {
2704                 foreach my $node (keys %$nodehash) {
2705                     if (!$avail->{$node}) {
2706                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2707                     }
2708                 }
2709             }
2710         }
2711     });
2712
2713     foreach my $node (values %$nodehash) {
2714         if (my $unavail = $node->{unavailable_storages}) {
2715             $node->{unavailable_storages} = [ sort keys %$unavail ];
2716         }
2717     }
2718
2719     return $nodehash
2720 }
2721
2722 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2723 sub check_running {
2724     my ($vmid, $nocheck, $node) = @_;
2725
2726     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2727     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2728 }
2729
2730 sub vzlist {
2731
2732     my $vzlist = config_list();
2733
2734     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2735
2736     while (defined(my $de = $fd->read)) {
2737         next if $de !~ m/^(\d+)\.pid$/;
2738         my $vmid = $1;
2739         next if !defined($vzlist->{$vmid});
2740         if (my $pid = check_running($vmid)) {
2741             $vzlist->{$vmid}->{pid} = $pid;
2742         }
2743     }
2744
2745     return $vzlist;
2746 }
2747
2748 our $vmstatus_return_properties = {
2749     vmid => get_standard_option('pve-vmid'),
2750     status => {
2751         description => "Qemu process status.",
2752         type => 'string',
2753         enum => ['stopped', 'running'],
2754     },
2755     maxmem => {
2756         description => "Maximum memory in bytes.",
2757         type => 'integer',
2758         optional => 1,
2759         renderer => 'bytes',
2760     },
2761     maxdisk => {
2762         description => "Root disk size in bytes.",
2763         type => 'integer',
2764         optional => 1,
2765         renderer => 'bytes',
2766     },
2767     name => {
2768         description => "VM name.",
2769         type => 'string',
2770         optional => 1,
2771     },
2772     qmpstatus => {
2773         description => "Qemu QMP agent status.",
2774         type => 'string',
2775         optional => 1,
2776     },
2777     pid => {
2778         description => "PID of running qemu process.",
2779         type => 'integer',
2780         optional => 1,
2781     },
2782     uptime => {
2783         description => "Uptime.",
2784         type => 'integer',
2785         optional => 1,
2786         renderer => 'duration',
2787     },
2788     cpus => {
2789         description => "Maximum usable CPUs.",
2790         type => 'number',
2791         optional => 1,
2792     },
2793     lock => {
2794         description => "The current config lock, if any.",
2795         type => 'string',
2796         optional => 1,
2797     },
2798     tags => {
2799         description  => "The current configured tags, if any",
2800         type => 'string',
2801         optional => 1,
2802     },
2803     'running-machine' => {
2804         description => "The currently running machine type (if running).",
2805         type => 'string',
2806         optional => 1,
2807     },
2808     'running-qemu' => {
2809         description => "The currently running QEMU version (if running).",
2810         type => 'string',
2811         optional => 1,
2812     },
2813 };
2814
2815 my $last_proc_pid_stat;
2816
2817 # get VM status information
2818 # This must be fast and should not block ($full == false)
2819 # We only query KVM using QMP if $full == true (this can be slow)
2820 sub vmstatus {
2821     my ($opt_vmid, $full) = @_;
2822
2823     my $res = {};
2824
2825     my $storecfg = PVE::Storage::config();
2826
2827     my $list = vzlist();
2828     my $defaults = load_defaults();
2829
2830     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2831
2832     my $cpucount = $cpuinfo->{cpus} || 1;
2833
2834     foreach my $vmid (keys %$list) {
2835         next if $opt_vmid && ($vmid ne $opt_vmid);
2836
2837         my $conf = PVE::QemuConfig->load_config($vmid);
2838
2839         my $d = { vmid => int($vmid) };
2840         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2841
2842         # fixme: better status?
2843         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2844
2845         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2846         if (defined($size)) {
2847             $d->{disk} = 0; # no info available
2848             $d->{maxdisk} = $size;
2849         } else {
2850             $d->{disk} = 0;
2851             $d->{maxdisk} = 0;
2852         }
2853
2854         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2855             * ($conf->{cores} || $defaults->{cores});
2856         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2857         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2858
2859         $d->{name} = $conf->{name} || "VM $vmid";
2860         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2861             : $defaults->{memory}*(1024*1024);
2862
2863         if ($conf->{balloon}) {
2864             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2865             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2866                 : $defaults->{shares};
2867         }
2868
2869         $d->{uptime} = 0;
2870         $d->{cpu} = 0;
2871         $d->{mem} = 0;
2872
2873         $d->{netout} = 0;
2874         $d->{netin} = 0;
2875
2876         $d->{diskread} = 0;
2877         $d->{diskwrite} = 0;
2878
2879         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2880
2881         $d->{serial} = 1 if conf_has_serial($conf);
2882         $d->{lock} = $conf->{lock} if $conf->{lock};
2883         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2884
2885         $res->{$vmid} = $d;
2886     }
2887
2888     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2889     foreach my $dev (keys %$netdev) {
2890         next if $dev !~ m/^tap([1-9]\d*)i/;
2891         my $vmid = $1;
2892         my $d = $res->{$vmid};
2893         next if !$d;
2894
2895         $d->{netout} += $netdev->{$dev}->{receive};
2896         $d->{netin} += $netdev->{$dev}->{transmit};
2897
2898         if ($full) {
2899             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2900             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2901         }
2902
2903     }
2904
2905     my $ctime = gettimeofday;
2906
2907     foreach my $vmid (keys %$list) {
2908
2909         my $d = $res->{$vmid};
2910         my $pid = $d->{pid};
2911         next if !$pid;
2912
2913         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2914         next if !$pstat; # not running
2915
2916         my $used = $pstat->{utime} + $pstat->{stime};
2917
2918         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2919
2920         if ($pstat->{vsize}) {
2921             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2922         }
2923
2924         my $old = $last_proc_pid_stat->{$pid};
2925         if (!$old) {
2926             $last_proc_pid_stat->{$pid} = {
2927                 time => $ctime,
2928                 used => $used,
2929                 cpu => 0,
2930             };
2931             next;
2932         }
2933
2934         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2935
2936         if ($dtime > 1000) {
2937             my $dutime = $used -  $old->{used};
2938
2939             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2940             $last_proc_pid_stat->{$pid} = {
2941                 time => $ctime,
2942                 used => $used,
2943                 cpu => $d->{cpu},
2944             };
2945         } else {
2946             $d->{cpu} = $old->{cpu};
2947         }
2948     }
2949
2950     return $res if !$full;
2951
2952     my $qmpclient = PVE::QMPClient->new();
2953
2954     my $ballooncb = sub {
2955         my ($vmid, $resp) = @_;
2956
2957         my $info = $resp->{'return'};
2958         return if !$info->{max_mem};
2959
2960         my $d = $res->{$vmid};
2961
2962         # use memory assigned to VM
2963         $d->{maxmem} = $info->{max_mem};
2964         $d->{balloon} = $info->{actual};
2965
2966         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2967             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2968             $d->{freemem} = $info->{free_mem};
2969         }
2970
2971         $d->{ballooninfo} = $info;
2972     };
2973
2974     my $blockstatscb = sub {
2975         my ($vmid, $resp) = @_;
2976         my $data = $resp->{'return'} || [];
2977         my $totalrdbytes = 0;
2978         my $totalwrbytes = 0;
2979
2980         for my $blockstat (@$data) {
2981             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2982             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2983
2984             $blockstat->{device} =~ s/drive-//;
2985             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2986         }
2987         $res->{$vmid}->{diskread} = $totalrdbytes;
2988         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2989     };
2990
2991     my $machinecb = sub {
2992         my ($vmid, $resp) = @_;
2993         my $data = $resp->{'return'} || [];
2994
2995         $res->{$vmid}->{'running-machine'} =
2996             PVE::QemuServer::Machine::current_from_query_machines($data);
2997     };
2998
2999     my $versioncb = sub {
3000         my ($vmid, $resp) = @_;
3001         my $data = $resp->{'return'} // {};
3002         my $version = 'unknown';
3003
3004         if (my $v = $data->{qemu}) {
3005             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3006         }
3007
3008         $res->{$vmid}->{'running-qemu'} = $version;
3009     };
3010
3011     my $statuscb = sub {
3012         my ($vmid, $resp) = @_;
3013
3014         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3015         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3016         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3017         # this fails if ballon driver is not loaded, so this must be
3018         # the last commnand (following command are aborted if this fails).
3019         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3020
3021         my $status = 'unknown';
3022         if (!defined($status = $resp->{'return'}->{status})) {
3023             warn "unable to get VM status\n";
3024             return;
3025         }
3026
3027         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3028     };
3029
3030     foreach my $vmid (keys %$list) {
3031         next if $opt_vmid && ($vmid ne $opt_vmid);
3032         next if !$res->{$vmid}->{pid}; # not running
3033         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3034     }
3035
3036     $qmpclient->queue_execute(undef, 2);
3037
3038     foreach my $vmid (keys %$list) {
3039         next if $opt_vmid && ($vmid ne $opt_vmid);
3040         next if !$res->{$vmid}->{pid}; #not running
3041
3042         # we can't use the $qmpclient since it might have already aborted on
3043         # 'query-balloon', but this might also fail for older versions...
3044         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3045         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3046     }
3047
3048     foreach my $vmid (keys %$list) {
3049         next if $opt_vmid && ($vmid ne $opt_vmid);
3050         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3051     }
3052
3053     return $res;
3054 }
3055
3056 sub conf_has_serial {
3057     my ($conf) = @_;
3058
3059     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3060         if ($conf->{"serial$i"}) {
3061             return 1;
3062         }
3063     }
3064
3065     return 0;
3066 }
3067
3068 sub conf_has_audio {
3069     my ($conf, $id) = @_;
3070
3071     $id //= 0;
3072     my $audio = $conf->{"audio$id"};
3073     return if !defined($audio);
3074
3075     my $audioproperties = parse_property_string($audio_fmt, $audio);
3076     my $audiodriver = $audioproperties->{driver} // 'spice';
3077
3078     return {
3079         dev => $audioproperties->{device},
3080         dev_id => "audiodev$id",
3081         backend => $audiodriver,
3082         backend_id => "$audiodriver-backend${id}",
3083     };
3084 }
3085
3086 sub audio_devs {
3087     my ($audio, $audiopciaddr, $machine_version) = @_;
3088
3089     my $devs = [];
3090
3091     my $id = $audio->{dev_id};
3092     my $audiodev = "";
3093     if (min_version($machine_version, 4, 2)) {
3094         $audiodev = ",audiodev=$audio->{backend_id}";
3095     }
3096
3097     if ($audio->{dev} eq 'AC97') {
3098         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3099     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3100         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3101         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3102         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3103     } else {
3104         die "unkown audio device '$audio->{dev}', implement me!";
3105     }
3106
3107     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3108
3109     return $devs;
3110 }
3111
3112 sub get_tpm_paths {
3113     my ($vmid) = @_;
3114     return {
3115         socket => "/var/run/qemu-server/$vmid.swtpm",
3116         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3117     };
3118 }
3119
3120 sub add_tpm_device {
3121     my ($vmid, $devices, $conf) = @_;
3122
3123     return if !$conf->{tpmstate0};
3124
3125     my $paths = get_tpm_paths($vmid);
3126
3127     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3128     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3129     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3130 }
3131
3132 sub start_swtpm {
3133     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3134
3135     return if !$tpmdrive;
3136
3137     my $state;
3138     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3139     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3140     if ($storeid) {
3141         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3142     } else {
3143         $state = $tpm->{file};
3144     }
3145
3146     my $paths = get_tpm_paths($vmid);
3147
3148     # during migration, we will get state from remote
3149     #
3150     if (!$migration) {
3151         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3152         my $setup_cmd = [
3153             "swtpm_setup",
3154             "--tpmstate",
3155             "file://$state",
3156             "--createek",
3157             "--create-ek-cert",
3158             "--create-platform-cert",
3159             "--lock-nvram",
3160             "--config",
3161             "/etc/swtpm_setup.conf", # do not use XDG configs
3162             "--runas",
3163             "0", # force creation as root, error if not possible
3164             "--not-overwrite", # ignore existing state, do not modify
3165         ];
3166
3167         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3168         # TPM 2.0 supports ECC crypto, use if possible
3169         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3170
3171         run_command($setup_cmd, outfunc => sub {
3172             print "swtpm_setup: $1\n";
3173         });
3174     }
3175
3176     my $emulator_cmd = [
3177         "swtpm",
3178         "socket",
3179         "--tpmstate",
3180         "backend-uri=file://$state,mode=0600",
3181         "--ctrl",
3182         "type=unixio,path=$paths->{socket},mode=0600",
3183         "--pid",
3184         "file=$paths->{pid}",
3185         "--terminate", # terminate on QEMU disconnect
3186         "--daemon",
3187     ];
3188     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3189     run_command($emulator_cmd, outfunc => sub { print $1; });
3190
3191     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3192     while (! -e $paths->{pid}) {
3193         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3194         usleep(50_000);
3195     }
3196
3197     # return untainted PID of swtpm daemon so it can be killed on error
3198     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3199     return $1;
3200 }
3201
3202 sub vga_conf_has_spice {
3203     my ($vga) = @_;
3204
3205     my $vgaconf = parse_vga($vga);
3206     my $vgatype = $vgaconf->{type};
3207     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3208
3209     return $1 || 1;
3210 }
3211
3212 sub is_native($) {
3213     my ($arch) = @_;
3214     return get_host_arch() eq $arch;
3215 }
3216
3217 sub get_vm_arch {
3218     my ($conf) = @_;
3219     return $conf->{arch} // get_host_arch();
3220 }
3221
3222 my $default_machines = {
3223     x86_64 => 'pc',
3224     aarch64 => 'virt',
3225 };
3226
3227 sub get_installed_machine_version {
3228     my ($kvmversion) = @_;
3229     $kvmversion = kvm_user_version() if !defined($kvmversion);
3230     $kvmversion =~ m/^(\d+\.\d+)/;
3231     return $1;
3232 }
3233
3234 sub windows_get_pinned_machine_version {
3235     my ($machine, $base_version, $kvmversion) = @_;
3236
3237     my $pin_version = $base_version;
3238     if (!defined($base_version) ||
3239         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3240     ) {
3241         $pin_version = get_installed_machine_version($kvmversion);
3242     }
3243     if (!$machine || $machine eq 'pc') {
3244         $machine = "pc-i440fx-$pin_version";
3245     } elsif ($machine eq 'q35') {
3246         $machine = "pc-q35-$pin_version";
3247     } elsif ($machine eq 'virt') {
3248         $machine = "virt-$pin_version";
3249     } else {
3250         warn "unknown machine type '$machine', not touching that!\n";
3251     }
3252
3253     return $machine;
3254 }
3255
3256 sub get_vm_machine {
3257     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3258
3259     my $machine = $forcemachine || $conf->{machine};
3260
3261     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3262         $kvmversion //= kvm_user_version();
3263         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3264         # layout which confuses windows quite a bit and may result in various regressions..
3265         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3266         if (windows_version($conf->{ostype})) {
3267             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3268         }
3269         $arch //= 'x86_64';
3270         $machine ||= $default_machines->{$arch};
3271         if ($add_pve_version) {
3272             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3273             $machine .= "+pve$pvever";
3274         }
3275     }
3276
3277     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3278         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3279         $machine = $1 if $is_pxe;
3280
3281         # for version-pinned machines that do not include a pve-version (e.g.
3282         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3283         $machine .= '+pve0';
3284
3285         $machine .= '.pxe' if $is_pxe;
3286     }
3287
3288     return $machine;
3289 }
3290
3291 sub get_ovmf_files($$$) {
3292     my ($arch, $efidisk, $smm) = @_;
3293
3294     my $types = $OVMF->{$arch}
3295         or die "no OVMF images known for architecture '$arch'\n";
3296
3297     my $type = 'default';
3298     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3299         $type = $smm ? "4m" : "4m-no-smm";
3300         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3301     }
3302
3303     return $types->{$type}->@*;
3304 }
3305
3306 my $Arch2Qemu = {
3307     aarch64 => '/usr/bin/qemu-system-aarch64',
3308     x86_64 => '/usr/bin/qemu-system-x86_64',
3309 };
3310 sub get_command_for_arch($) {
3311     my ($arch) = @_;
3312     return '/usr/bin/kvm' if is_native($arch);
3313
3314     my $cmd = $Arch2Qemu->{$arch}
3315         or die "don't know how to emulate architecture '$arch'\n";
3316     return $cmd;
3317 }
3318
3319 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3320 # to use in a QEMU command line (-cpu element), first array_intersect the result
3321 # of query_supported_ with query_understood_. This is necessary because:
3322 #
3323 # a) query_understood_ returns flags the host cannot use and
3324 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3325 #    flags, but CPU settings - with most of them being flags. Those settings
3326 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3327 #
3328 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3329 # expensive. If you need the value returned from this, you can get it much
3330 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3331 # $accel being 'kvm' or 'tcg'.
3332 #
3333 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3334 # changes, automatically populating pmxcfs.
3335 #
3336 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3337 # since kvm and tcg machines support different flags
3338 #
3339 sub query_supported_cpu_flags {
3340     my ($arch) = @_;
3341
3342     $arch //= get_host_arch();
3343     my $default_machine = $default_machines->{$arch};
3344
3345     my $flags = {};
3346
3347     # FIXME: Once this is merged, the code below should work for ARM as well:
3348     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3349     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3350         $arch eq "aarch64";
3351
3352     my $kvm_supported = defined(kvm_version());
3353     my $qemu_cmd = get_command_for_arch($arch);
3354     my $fakevmid = -1;
3355     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3356
3357     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3358     my $query_supported_run_qemu = sub {
3359         my ($kvm) = @_;
3360
3361         my $flags = {};
3362         my $cmd = [
3363             $qemu_cmd,
3364             '-machine', $default_machine,
3365             '-display', 'none',
3366             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3367             '-mon', 'chardev=qmp,mode=control',
3368             '-pidfile', $pidfile,
3369             '-S', '-daemonize'
3370         ];
3371
3372         if (!$kvm) {
3373             push @$cmd, '-accel', 'tcg';
3374         }
3375
3376         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3377         die "QEMU flag querying VM exited with code " . $rc if $rc;
3378
3379         eval {
3380             my $cmd_result = mon_cmd(
3381                 $fakevmid,
3382                 'query-cpu-model-expansion',
3383                 type => 'full',
3384                 model => { name => 'host' }
3385             );
3386
3387             my $props = $cmd_result->{model}->{props};
3388             foreach my $prop (keys %$props) {
3389                 next if $props->{$prop} ne '1';
3390                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3391                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3392                 # We only keep those with underscores, to match /proc/cpuinfo
3393                 $prop =~ s/\.|-/_/g;
3394                 $flags->{$prop} = 1;
3395             }
3396         };
3397         my $err = $@;
3398
3399         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3400         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3401
3402         die $err if $err;
3403
3404         return [ sort keys %$flags ];
3405     };
3406
3407     # We need to query QEMU twice, since KVM and TCG have different supported flags
3408     PVE::QemuConfig->lock_config($fakevmid, sub {
3409         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3410         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3411
3412         if ($kvm_supported) {
3413             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3414             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3415         }
3416     });
3417
3418     return $flags;
3419 }
3420
3421 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3422 my $understood_cpu_flag_dir = "/usr/share/kvm";
3423 sub query_understood_cpu_flags {
3424     my $arch = get_host_arch();
3425     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3426
3427     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3428         if ! -e $filepath;
3429
3430     my $raw = file_get_contents($filepath);
3431     $raw =~ s/^\s+|\s+$//g;
3432     my @flags = split(/\s+/, $raw);
3433
3434     return \@flags;
3435 }
3436
3437 my sub get_cpuunits {
3438     my ($conf) = @_;
3439     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3440
3441     my $cpuunits = $conf->{cpuunits};
3442     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3443
3444     if ($is_cgroupv2) {
3445         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3446     } else {
3447         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3448     }
3449     return $cpuunits;
3450 }
3451
3452 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3453 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3454 my sub should_disable_smm {
3455     my ($conf, $vga) = @_;
3456
3457     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3458         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3459 }
3460
3461 sub config_to_command {
3462     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3463         $pbs_backing) = @_;
3464
3465     my $cmd = [];
3466     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3467     my $devices = [];
3468     my $bridges = {};
3469     my $ostype = $conf->{ostype};
3470     my $winversion = windows_version($ostype);
3471     my $kvm = $conf->{kvm};
3472     my $nodename = nodename();
3473
3474     my $arch = get_vm_arch($conf);
3475     my $kvm_binary = get_command_for_arch($arch);
3476     my $kvmver = kvm_user_version($kvm_binary);
3477
3478     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3479         $kvmver //= "undefined";
3480         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3481     }
3482
3483     my $add_pve_version = min_version($kvmver, 4, 1);
3484
3485     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3486     my $machine_version = extract_version($machine_type, $kvmver);
3487     $kvm //= 1 if is_native($arch);
3488
3489     $machine_version =~ m/(\d+)\.(\d+)/;
3490     my ($machine_major, $machine_minor) = ($1, $2);
3491
3492     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3493         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3494     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3495         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3496             ." please upgrade node '$nodename'\n"
3497     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3498         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3499         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3500             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3501             ." node '$nodename'\n";
3502     }
3503
3504     # if a specific +pve version is required for a feature, use $version_guard
3505     # instead of min_version to allow machines to be run with the minimum
3506     # required version
3507     my $required_pve_version = 0;
3508     my $version_guard = sub {
3509         my ($major, $minor, $pve) = @_;
3510         return 0 if !min_version($machine_version, $major, $minor, $pve);
3511         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3512         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3513         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3514         return 1;
3515     };
3516
3517     if ($kvm && !defined kvm_version()) {
3518         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3519             ." or enable in BIOS.\n";
3520     }
3521
3522     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3523     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3524     my $use_old_bios_files = undef;
3525     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3526
3527     push @$cmd, $kvm_binary;
3528
3529     push @$cmd, '-id', $vmid;
3530
3531     my $vmname = $conf->{name} || "vm$vmid";
3532
3533     push @$cmd, '-name', $vmname;
3534
3535     push @$cmd, '-no-shutdown';
3536
3537     my $use_virtio = 0;
3538
3539     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3540     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3541     push @$cmd, '-mon', "chardev=qmp,mode=control";
3542
3543     if (min_version($machine_version, 2, 12)) {
3544         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3545         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3546     }
3547
3548     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3549
3550     push @$cmd, '-daemonize';
3551
3552     if ($conf->{smbios1}) {
3553         my $smbios_conf = parse_smbios1($conf->{smbios1});
3554         if ($smbios_conf->{base64}) {
3555             # Do not pass base64 flag to qemu
3556             delete $smbios_conf->{base64};
3557             my $smbios_string = "";
3558             foreach my $key (keys %$smbios_conf) {
3559                 my $value;
3560                 if ($key eq "uuid") {
3561                     $value = $smbios_conf->{uuid}
3562                 } else {
3563                     $value = decode_base64($smbios_conf->{$key});
3564                 }
3565                 # qemu accepts any binary data, only commas need escaping by double comma
3566                 $value =~ s/,/,,/g;
3567                 $smbios_string .= "," . $key . "=" . $value if $value;
3568             }
3569             push @$cmd, '-smbios', "type=1" . $smbios_string;
3570         } else {
3571             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3572         }
3573     }
3574
3575     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3576         my $d;
3577         if (my $efidisk = $conf->{efidisk0}) {
3578             $d = parse_drive('efidisk0', $efidisk);
3579         }
3580
3581         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3582         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3583
3584         my ($path, $format);
3585         my $read_only_str = '';
3586         if ($d) {
3587             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3588             $format = $d->{format};
3589             if ($storeid) {
3590                 $path = PVE::Storage::path($storecfg, $d->{file});
3591                 if (!defined($format)) {
3592                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3593                     $format = qemu_img_format($scfg, $volname);
3594                 }
3595             } else {
3596                 $path = $d->{file};
3597                 die "efidisk format must be specified\n"
3598                     if !defined($format);
3599             }
3600
3601             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3602         } else {
3603             warn "no efidisk configured! Using temporary efivars disk.\n";
3604             $path = "/tmp/$vmid-ovmf.fd";
3605             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3606             $format = 'raw';
3607         }
3608
3609         my $size_str = "";
3610
3611         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3612             $size_str = ",size=" . (-s $ovmf_vars);
3613         }
3614
3615         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3616         my $cache = "";
3617         if ($path =~ m/^rbd:/) {
3618                 $cache = ',cache=writeback';
3619                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3620         }
3621
3622         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3623         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3624     }
3625
3626     if ($q35) { # tell QEMU to load q35 config early
3627         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3628         if (min_version($machine_version, 4, 0)) {
3629             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3630         } else {
3631             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3632         }
3633     }
3634
3635     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3636         push @$cmd, $fixups->@*;
3637     }
3638
3639     if ($conf->{vmgenid}) {
3640         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3641     }
3642
3643     # add usb controllers
3644     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3645         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3646     push @$devices, @usbcontrollers if @usbcontrollers;
3647     my $vga = parse_vga($conf->{vga});
3648
3649     my $qxlnum = vga_conf_has_spice($conf->{vga});
3650     $vga->{type} = 'qxl' if $qxlnum;
3651
3652     if (!$vga->{type}) {
3653         if ($arch eq 'aarch64') {
3654             $vga->{type} = 'virtio';
3655         } elsif (min_version($machine_version, 2, 9)) {
3656             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3657         } else {
3658             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3659         }
3660     }
3661
3662     # enable absolute mouse coordinates (needed by vnc)
3663     my $tablet = $conf->{tablet};
3664     if (!defined($tablet)) {
3665         $tablet = $defaults->{tablet};
3666         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3667         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3668     }
3669
3670     if ($tablet) {
3671         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3672         my $kbd = print_keyboarddevice_full($conf, $arch);
3673         push @$devices, '-device', $kbd if defined($kbd);
3674     }
3675
3676     my $bootorder = device_bootorder($conf);
3677
3678     # host pci device passthrough
3679     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3680         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3681
3682     # usb devices
3683     my $usb_dev_features = {};
3684     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3685
3686     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3687         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3688     push @$devices, @usbdevices if @usbdevices;
3689
3690     # serial devices
3691     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3692         my $path = $conf->{"serial$i"} or next;
3693         if ($path eq 'socket') {
3694             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3695             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3696             # On aarch64, serial0 is the UART device. Qemu only allows
3697             # connecting UART devices via the '-serial' command line, as
3698             # the device has a fixed slot on the hardware...
3699             if ($arch eq 'aarch64' && $i == 0) {
3700                 push @$devices, '-serial', "chardev:serial$i";
3701             } else {
3702                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3703             }
3704         } else {
3705             die "no such serial device\n" if ! -c $path;
3706             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3707             push @$devices, '-device', "isa-serial,chardev=serial$i";
3708         }
3709     }
3710
3711     # parallel devices
3712     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3713         if (my $path = $conf->{"parallel$i"}) {
3714             die "no such parallel device\n" if ! -c $path;
3715             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3716             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3717             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3718         }
3719     }
3720
3721     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3722         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3723         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3724         push @$devices, @$audio_devs;
3725     }
3726
3727     add_tpm_device($vmid, $devices, $conf);
3728
3729     my $sockets = 1;
3730     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3731     $sockets = $conf->{sockets} if  $conf->{sockets};
3732
3733     my $cores = $conf->{cores} || 1;
3734
3735     my $maxcpus = $sockets * $cores;
3736
3737     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3738
3739     my $allowed_vcpus = $cpuinfo->{cpus};
3740
3741     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3742
3743     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3744         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3745         for (my $i = 2; $i <= $vcpus; $i++)  {
3746             my $cpustr = print_cpu_device($conf,$i);
3747             push @$cmd, '-device', $cpustr;
3748         }
3749
3750     } else {
3751
3752         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3753     }
3754     push @$cmd, '-nodefaults';
3755
3756     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3757
3758     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3759
3760     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3761
3762     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3763         push @$devices, '-device', print_vga_device(
3764             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3765         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3766         push @$cmd,  '-vnc', "unix:$socket,password=on";
3767     } else {
3768         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3769         push @$cmd, '-nographic';
3770     }
3771
3772     # time drift fix
3773     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3774     my $useLocaltime = $conf->{localtime};
3775
3776     if ($winversion >= 5) { # windows
3777         $useLocaltime = 1 if !defined($conf->{localtime});
3778
3779         # use time drift fix when acpi is enabled
3780         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3781             $tdf = 1 if !defined($conf->{tdf});
3782         }
3783     }
3784
3785     if ($winversion >= 6) {
3786         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3787         push @$cmd, '-no-hpet';
3788     }
3789
3790     push @$rtcFlags, 'driftfix=slew' if $tdf;
3791
3792     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3793         push @$rtcFlags, "base=$conf->{startdate}";
3794     } elsif ($useLocaltime) {
3795         push @$rtcFlags, 'base=localtime';
3796     }
3797
3798     if ($forcecpu) {
3799         push @$cmd, '-cpu', $forcecpu;
3800     } else {
3801         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3802     }
3803
3804     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3805
3806     push @$cmd, '-S' if $conf->{freeze};
3807
3808     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3809
3810     my $guest_agent = parse_guest_agent($conf);
3811
3812     if ($guest_agent->{enabled}) {
3813         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3814         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3815
3816         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3817             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3818             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3819             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3820         } elsif ($guest_agent->{type} eq 'isa') {
3821             push @$devices, '-device', "isa-serial,chardev=qga0";
3822         }
3823     }
3824
3825     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3826     if ($rng && $version_guard->(4, 1, 2)) {
3827         check_rng_source($rng->{source});
3828
3829         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3830         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3831         my $limiter_str = "";
3832         if ($max_bytes) {
3833             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3834         }
3835
3836         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3837         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3838         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3839     }
3840
3841     my $spice_port;
3842
3843     if ($qxlnum) {
3844         if ($qxlnum > 1) {
3845             if ($winversion){
3846                 for (my $i = 1; $i < $qxlnum; $i++){
3847                     push @$devices, '-device', print_vga_device(
3848                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3849                 }
3850             } else {
3851                 # assume other OS works like Linux
3852                 my ($ram, $vram) = ("134217728", "67108864");
3853                 if ($vga->{memory}) {
3854                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3855                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3856                 }
3857                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3858                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3859             }
3860         }
3861
3862         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3863
3864         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3865         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3866         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3867
3868         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3869         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3870         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3871
3872         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3873         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3874
3875         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3876         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3877         if ($spice_enhancement->{foldersharing}) {
3878             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3879             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3880         }
3881
3882         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3883         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3884             if $spice_enhancement->{videostreaming};
3885
3886         push @$devices, '-spice', "$spice_opts";
3887     }
3888
3889     # enable balloon by default, unless explicitly disabled
3890     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3891         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3892         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3893     }
3894
3895     if ($conf->{watchdog}) {
3896         my $wdopts = parse_watchdog($conf->{watchdog});
3897         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3898         my $watchdog = $wdopts->{model} || 'i6300esb';
3899         push @$devices, '-device', "$watchdog$pciaddr";
3900         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3901     }
3902
3903     my $vollist = [];
3904     my $scsicontroller = {};
3905     my $ahcicontroller = {};
3906     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3907
3908     # Add iscsi initiator name if available
3909     if (my $initiator = get_initiator_name()) {
3910         push @$devices, '-iscsi', "initiator-name=$initiator";
3911     }
3912
3913     PVE::QemuConfig->foreach_volume($conf, sub {
3914         my ($ds, $drive) = @_;
3915
3916         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3917             check_volume_storage_type($storecfg, $drive->{file});
3918             push @$vollist, $drive->{file};
3919         }
3920
3921         # ignore efidisk here, already added in bios/fw handling code above
3922         return if $drive->{interface} eq 'efidisk';
3923         # similar for TPM
3924         return if $drive->{interface} eq 'tpmstate';
3925
3926         $use_virtio = 1 if $ds =~ m/^virtio/;
3927
3928         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3929
3930         if ($drive->{interface} eq 'virtio'){
3931            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3932         }
3933
3934         if ($drive->{interface} eq 'scsi') {
3935
3936             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3937
3938             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3939                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3940
3941             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3942             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3943
3944             my $iothread = '';
3945             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3946                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3947                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3948             } elsif ($drive->{iothread}) {
3949                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3950             }
3951
3952             my $queues = '';
3953             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3954                 $queues = ",num_queues=$drive->{queues}";
3955             }
3956
3957             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3958                 if !$scsicontroller->{$controller};
3959             $scsicontroller->{$controller}=1;
3960         }
3961
3962         if ($drive->{interface} eq 'sata') {
3963             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3964             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3965             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3966                 if !$ahcicontroller->{$controller};
3967             $ahcicontroller->{$controller}=1;
3968         }
3969
3970         my $pbs_conf = $pbs_backing->{$ds};
3971         my $pbs_name = undef;
3972         if ($pbs_conf) {
3973             $pbs_name = "drive-$ds-pbs";
3974             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3975         }
3976
3977         my $drive_cmd = print_drive_commandline_full(
3978             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3979
3980         # extra protection for templates, but SATA and IDE don't support it..
3981         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3982
3983         push @$devices, '-drive',$drive_cmd;
3984         push @$devices, '-device', print_drivedevice_full(
3985             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3986     });
3987
3988     for (my $i = 0; $i < $MAX_NETS; $i++) {
3989         my $netname = "net$i";
3990
3991         next if !$conf->{$netname};
3992         my $d = parse_net($conf->{$netname});
3993         next if !$d;
3994
3995         $use_virtio = 1 if $d->{model} eq 'virtio';
3996
3997         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3998
3999         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4000         push @$devices, '-netdev', $netdevfull;
4001
4002         my $netdevicefull = print_netdevice_full(
4003             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4004
4005         push @$devices, '-device', $netdevicefull;
4006     }
4007
4008     if ($conf->{ivshmem}) {
4009         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4010
4011         my $bus;
4012         if ($q35) {
4013             $bus = print_pcie_addr("ivshmem");
4014         } else {
4015             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4016         }
4017
4018         my $ivshmem_name = $ivshmem->{name} // $vmid;
4019         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4020
4021         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4022         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4023             .",size=$ivshmem->{size}M";
4024     }
4025
4026     # pci.4 is nested in pci.1
4027     $bridges->{1} = 1 if $bridges->{4};
4028
4029     if (!$q35) { # add pci bridges
4030         if (min_version($machine_version, 2, 3)) {
4031            $bridges->{1} = 1;
4032            $bridges->{2} = 1;
4033         }
4034         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4035     }
4036
4037     for my $k (sort {$b cmp $a} keys %$bridges) {
4038         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4039
4040         my $k_name = $k;
4041         if ($k == 2 && $legacy_igd) {
4042             $k_name = "$k-igd";
4043         }
4044         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4045         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4046
4047         if ($q35) { # add after -readconfig pve-q35.cfg
4048             splice @$devices, 2, 0, '-device', $devstr;
4049         } else {
4050             unshift @$devices, '-device', $devstr if $k > 0;
4051         }
4052     }
4053
4054     if (!$kvm) {
4055         push @$machineFlags, 'accel=tcg';
4056     }
4057
4058     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4059
4060     my $machine_type_min = $machine_type;
4061     if ($add_pve_version) {
4062         $machine_type_min =~ s/\+pve\d+$//;
4063         $machine_type_min .= "+pve$required_pve_version";
4064     }
4065     push @$machineFlags, "type=${machine_type_min}";
4066
4067     push @$cmd, @$devices;
4068     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4069     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4070     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4071
4072     if (my $vmstate = $conf->{vmstate}) {
4073         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4074         push @$vollist, $vmstate;
4075         push @$cmd, '-loadstate', $statepath;
4076         print "activating and using '$vmstate' as vmstate\n";
4077     }
4078
4079     if (PVE::QemuConfig->is_template($conf)) {
4080         # needed to workaround base volumes being read-only
4081         push @$cmd, '-snapshot';
4082     }
4083
4084     # add custom args
4085     if ($conf->{args}) {
4086         my $aa = PVE::Tools::split_args($conf->{args});
4087         push @$cmd, @$aa;
4088     }
4089
4090     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4091 }
4092
4093 sub check_rng_source {
4094     my ($source) = @_;
4095
4096     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4097     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4098         if ! -e $source;
4099
4100     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4101     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4102         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4103         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4104         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4105             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4106             ." to the host.\n";
4107     }
4108 }
4109
4110 sub spice_port {
4111     my ($vmid) = @_;
4112
4113     my $res = mon_cmd($vmid, 'query-spice');
4114
4115     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4116 }
4117
4118 sub vm_devices_list {
4119     my ($vmid) = @_;
4120
4121     my $res = mon_cmd($vmid, 'query-pci');
4122     my $devices_to_check = [];
4123     my $devices = {};
4124     foreach my $pcibus (@$res) {
4125         push @$devices_to_check, @{$pcibus->{devices}},
4126     }
4127
4128     while (@$devices_to_check) {
4129         my $to_check = [];
4130         for my $d (@$devices_to_check) {
4131             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4132             next if !$d->{'pci_bridge'};
4133
4134             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4135             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4136         }
4137         $devices_to_check = $to_check;
4138     }
4139
4140     my $resblock = mon_cmd($vmid, 'query-block');
4141     foreach my $block (@$resblock) {
4142         if($block->{device} =~ m/^drive-(\S+)/){
4143                 $devices->{$1} = 1;
4144         }
4145     }
4146
4147     my $resmice = mon_cmd($vmid, 'query-mice');
4148     foreach my $mice (@$resmice) {
4149         if ($mice->{name} eq 'QEMU HID Tablet') {
4150             $devices->{tablet} = 1;
4151             last;
4152         }
4153     }
4154
4155     # for usb devices there is no query-usb
4156     # but we can iterate over the entries in
4157     # qom-list path=/machine/peripheral
4158     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4159     foreach my $per (@$resperipheral) {
4160         if ($per->{name} =~ m/^usb\d+$/) {
4161             $devices->{$per->{name}} = 1;
4162         }
4163     }
4164
4165     return $devices;
4166 }
4167
4168 sub vm_deviceplug {
4169     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4170
4171     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4172
4173     my $devices_list = vm_devices_list($vmid);
4174     return 1 if defined($devices_list->{$deviceid});
4175
4176     # add PCI bridge if we need it for the device
4177     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4178
4179     if ($deviceid eq 'tablet') {
4180         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4181     } elsif ($deviceid eq 'keyboard') {
4182         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4183     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4184         die "usb hotplug currently not reliable\n";
4185         # since we can't reliably hot unplug all added usb devices and usb
4186         # passthrough breaks live migration we disable usb hotplugging for now
4187         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4188     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4189         qemu_iothread_add($vmid, $deviceid, $device);
4190
4191         qemu_driveadd($storecfg, $vmid, $device);
4192         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4193
4194         qemu_deviceadd($vmid, $devicefull);
4195         eval { qemu_deviceaddverify($vmid, $deviceid); };
4196         if (my $err = $@) {
4197             eval { qemu_drivedel($vmid, $deviceid); };
4198             warn $@ if $@;
4199             die $err;
4200         }
4201     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4202         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4203         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4204         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4205
4206         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4207
4208         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4209             qemu_iothread_add($vmid, $deviceid, $device);
4210             $devicefull .= ",iothread=iothread-$deviceid";
4211         }
4212
4213         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4214             $devicefull .= ",num_queues=$device->{queues}";
4215         }
4216
4217         qemu_deviceadd($vmid, $devicefull);
4218         qemu_deviceaddverify($vmid, $deviceid);
4219     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4220         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4221         qemu_driveadd($storecfg, $vmid, $device);
4222
4223         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4224         eval { qemu_deviceadd($vmid, $devicefull); };
4225         if (my $err = $@) {
4226             eval { qemu_drivedel($vmid, $deviceid); };
4227             warn $@ if $@;
4228             die $err;
4229         }
4230     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4231         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4232
4233         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4234         my $use_old_bios_files = undef;
4235         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4236
4237         my $netdevicefull = print_netdevice_full(
4238             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4239         qemu_deviceadd($vmid, $netdevicefull);
4240         eval {
4241             qemu_deviceaddverify($vmid, $deviceid);
4242             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4243         };
4244         if (my $err = $@) {
4245             eval { qemu_netdevdel($vmid, $deviceid); };
4246             warn $@ if $@;
4247             die $err;
4248         }
4249     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4250         my $bridgeid = $2;
4251         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4252         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4253
4254         qemu_deviceadd($vmid, $devicefull);
4255         qemu_deviceaddverify($vmid, $deviceid);
4256     } else {
4257         die "can't hotplug device '$deviceid'\n";
4258     }
4259
4260     return 1;
4261 }
4262
4263 # fixme: this should raise exceptions on error!
4264 sub vm_deviceunplug {
4265     my ($vmid, $conf, $deviceid) = @_;
4266
4267     my $devices_list = vm_devices_list($vmid);
4268     return 1 if !defined($devices_list->{$deviceid});
4269
4270     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4271     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4272
4273     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4274         qemu_devicedel($vmid, $deviceid);
4275     } elsif ($deviceid =~ m/^usb\d+$/) {
4276         die "usb hotplug currently not reliable\n";
4277         # when unplugging usb devices this way, there may be remaining usb
4278         # controllers/hubs so we disable it for now
4279         #qemu_devicedel($vmid, $deviceid);
4280         #qemu_devicedelverify($vmid, $deviceid);
4281     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4282         my $device = parse_drive($deviceid, $conf->{$deviceid});
4283
4284         qemu_devicedel($vmid, $deviceid);
4285         qemu_devicedelverify($vmid, $deviceid);
4286         qemu_drivedel($vmid, $deviceid);
4287         qemu_iothread_del($vmid, $deviceid, $device);
4288     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4289         qemu_devicedel($vmid, $deviceid);
4290         qemu_devicedelverify($vmid, $deviceid);
4291     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4292         my $device = parse_drive($deviceid, $conf->{$deviceid});
4293
4294         qemu_devicedel($vmid, $deviceid);
4295         qemu_devicedelverify($vmid, $deviceid);
4296         qemu_drivedel($vmid, $deviceid);
4297         qemu_deletescsihw($conf, $vmid, $deviceid);
4298
4299         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4300             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4301     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4302         qemu_devicedel($vmid, $deviceid);
4303         qemu_devicedelverify($vmid, $deviceid);
4304         qemu_netdevdel($vmid, $deviceid);
4305     } else {
4306         die "can't unplug device '$deviceid'\n";
4307     }
4308
4309     return 1;
4310 }
4311
4312 sub qemu_deviceadd {
4313     my ($vmid, $devicefull) = @_;
4314
4315     $devicefull = "driver=".$devicefull;
4316     my %options =  split(/[=,]/, $devicefull);
4317
4318     mon_cmd($vmid, "device_add" , %options);
4319 }
4320
4321 sub qemu_devicedel {
4322     my ($vmid, $deviceid) = @_;
4323
4324     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4325 }
4326
4327 sub qemu_iothread_add {
4328     my ($vmid, $deviceid, $device) = @_;
4329
4330     if ($device->{iothread}) {
4331         my $iothreads = vm_iothreads_list($vmid);
4332         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4333     }
4334 }
4335
4336 sub qemu_iothread_del {
4337     my ($vmid, $deviceid, $device) = @_;
4338
4339     if ($device->{iothread}) {
4340         my $iothreads = vm_iothreads_list($vmid);
4341         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4342     }
4343 }
4344
4345 sub qemu_objectadd {
4346     my ($vmid, $objectid, $qomtype) = @_;
4347
4348     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4349
4350     return 1;
4351 }
4352
4353 sub qemu_objectdel {
4354     my ($vmid, $objectid) = @_;
4355
4356     mon_cmd($vmid, "object-del", id => $objectid);
4357
4358     return 1;
4359 }
4360
4361 sub qemu_driveadd {
4362     my ($storecfg, $vmid, $device) = @_;
4363
4364     my $kvmver = get_running_qemu_version($vmid);
4365     my $io_uring = min_version($kvmver, 6, 0);
4366     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4367     $drive =~ s/\\/\\\\/g;
4368     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4369
4370     # If the command succeeds qemu prints: "OK"
4371     return 1 if $ret =~ m/OK/s;
4372
4373     die "adding drive failed: $ret\n";
4374 }
4375
4376 sub qemu_drivedel {
4377     my ($vmid, $deviceid) = @_;
4378
4379     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4380     $ret =~ s/^\s+//;
4381
4382     return 1 if $ret eq "";
4383
4384     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4385     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4386
4387     die "deleting drive $deviceid failed : $ret\n";
4388 }
4389
4390 sub qemu_deviceaddverify {
4391     my ($vmid, $deviceid) = @_;
4392
4393     for (my $i = 0; $i <= 5; $i++) {
4394          my $devices_list = vm_devices_list($vmid);
4395          return 1 if defined($devices_list->{$deviceid});
4396          sleep 1;
4397     }
4398
4399     die "error on hotplug device '$deviceid'\n";
4400 }
4401
4402
4403 sub qemu_devicedelverify {
4404     my ($vmid, $deviceid) = @_;
4405
4406     # need to verify that the device is correctly removed as device_del
4407     # is async and empty return is not reliable
4408
4409     for (my $i = 0; $i <= 5; $i++) {
4410          my $devices_list = vm_devices_list($vmid);
4411          return 1 if !defined($devices_list->{$deviceid});
4412          sleep 1;
4413     }
4414
4415     die "error on hot-unplugging device '$deviceid'\n";
4416 }
4417
4418 sub qemu_findorcreatescsihw {
4419     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4420
4421     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4422
4423     my $scsihwid="$controller_prefix$controller";
4424     my $devices_list = vm_devices_list($vmid);
4425
4426     if (!defined($devices_list->{$scsihwid})) {
4427         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4428     }
4429
4430     return 1;
4431 }
4432
4433 sub qemu_deletescsihw {
4434     my ($conf, $vmid, $opt) = @_;
4435
4436     my $device = parse_drive($opt, $conf->{$opt});
4437
4438     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4439         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4440         return 1;
4441     }
4442
4443     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4444
4445     my $devices_list = vm_devices_list($vmid);
4446     foreach my $opt (keys %{$devices_list}) {
4447         if (is_valid_drivename($opt)) {
4448             my $drive = parse_drive($opt, $conf->{$opt});
4449             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4450                 return 1;
4451             }
4452         }
4453     }
4454
4455     my $scsihwid="scsihw$controller";
4456
4457     vm_deviceunplug($vmid, $conf, $scsihwid);
4458
4459     return 1;
4460 }
4461
4462 sub qemu_add_pci_bridge {
4463     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4464
4465     my $bridges = {};
4466
4467     my $bridgeid;
4468
4469     print_pci_addr($device, $bridges, $arch, $machine_type);
4470
4471     while (my ($k, $v) = each %$bridges) {
4472         $bridgeid = $k;
4473     }
4474     return 1 if !defined($bridgeid) || $bridgeid < 1;
4475
4476     my $bridge = "pci.$bridgeid";
4477     my $devices_list = vm_devices_list($vmid);
4478
4479     if (!defined($devices_list->{$bridge})) {
4480         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4481     }
4482
4483     return 1;
4484 }
4485
4486 sub qemu_set_link_status {
4487     my ($vmid, $device, $up) = @_;
4488
4489     mon_cmd($vmid, "set_link", name => $device,
4490                up => $up ? JSON::true : JSON::false);
4491 }
4492
4493 sub qemu_netdevadd {
4494     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4495
4496     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4497     my %options =  split(/[=,]/, $netdev);
4498
4499     if (defined(my $vhost = $options{vhost})) {
4500         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4501     }
4502
4503     if (defined(my $queues = $options{queues})) {
4504         $options{queues} = $queues + 0;
4505     }
4506
4507     mon_cmd($vmid, "netdev_add",  %options);
4508     return 1;
4509 }
4510
4511 sub qemu_netdevdel {
4512     my ($vmid, $deviceid) = @_;
4513
4514     mon_cmd($vmid, "netdev_del", id => $deviceid);
4515 }
4516
4517 sub qemu_usb_hotplug {
4518     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4519
4520     return if !$device;
4521
4522     # remove the old one first
4523     vm_deviceunplug($vmid, $conf, $deviceid);
4524
4525     # check if xhci controller is necessary and available
4526     if ($device->{usb3}) {
4527
4528         my $devicelist = vm_devices_list($vmid);
4529
4530         if (!$devicelist->{xhci}) {
4531             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4532             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4533         }
4534     }
4535     my $d = parse_usb_device($device->{host});
4536     $d->{usb3} = $device->{usb3};
4537
4538     # add the new one
4539     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4540 }
4541
4542 sub qemu_cpu_hotplug {
4543     my ($vmid, $conf, $vcpus) = @_;
4544
4545     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4546
4547     my $sockets = 1;
4548     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4549     $sockets = $conf->{sockets} if  $conf->{sockets};
4550     my $cores = $conf->{cores} || 1;
4551     my $maxcpus = $sockets * $cores;
4552
4553     $vcpus = $maxcpus if !$vcpus;
4554
4555     die "you can't add more vcpus than maxcpus\n"
4556         if $vcpus > $maxcpus;
4557
4558     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4559
4560     if ($vcpus < $currentvcpus) {
4561
4562         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4563
4564             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4565                 qemu_devicedel($vmid, "cpu$i");
4566                 my $retry = 0;
4567                 my $currentrunningvcpus = undef;
4568                 while (1) {
4569                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4570                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4571                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4572                     $retry++;
4573                     sleep 1;
4574                 }
4575                 #update conf after each succesfull cpu unplug
4576                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4577                 PVE::QemuConfig->write_config($vmid, $conf);
4578             }
4579         } else {
4580             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4581         }
4582
4583         return;
4584     }
4585
4586     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4587     die "vcpus in running vm does not match its configuration\n"
4588         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4589
4590     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4591
4592         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4593             my $cpustr = print_cpu_device($conf, $i);
4594             qemu_deviceadd($vmid, $cpustr);
4595
4596             my $retry = 0;
4597             my $currentrunningvcpus = undef;
4598             while (1) {
4599                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4600                 last if scalar(@{$currentrunningvcpus}) == $i;
4601                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4602                 sleep 1;
4603                 $retry++;
4604             }
4605             #update conf after each succesfull cpu hotplug
4606             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4607             PVE::QemuConfig->write_config($vmid, $conf);
4608         }
4609     } else {
4610
4611         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4612             mon_cmd($vmid, "cpu-add", id => int($i));
4613         }
4614     }
4615 }
4616
4617 sub qemu_block_set_io_throttle {
4618     my ($vmid, $deviceid,
4619         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4620         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4621         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4622         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4623
4624     return if !check_running($vmid) ;
4625
4626     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4627         bps => int($bps),
4628         bps_rd => int($bps_rd),
4629         bps_wr => int($bps_wr),
4630         iops => int($iops),
4631         iops_rd => int($iops_rd),
4632         iops_wr => int($iops_wr),
4633         bps_max => int($bps_max),
4634         bps_rd_max => int($bps_rd_max),
4635         bps_wr_max => int($bps_wr_max),
4636         iops_max => int($iops_max),
4637         iops_rd_max => int($iops_rd_max),
4638         iops_wr_max => int($iops_wr_max),
4639         bps_max_length => int($bps_max_length),
4640         bps_rd_max_length => int($bps_rd_max_length),
4641         bps_wr_max_length => int($bps_wr_max_length),
4642         iops_max_length => int($iops_max_length),
4643         iops_rd_max_length => int($iops_rd_max_length),
4644         iops_wr_max_length => int($iops_wr_max_length),
4645     );
4646
4647 }
4648
4649 sub qemu_block_resize {
4650     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4651
4652     my $running = check_running($vmid);
4653
4654     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4655
4656     return if !$running;
4657
4658     my $padding = (1024 - $size % 1024) % 1024;
4659     $size = $size + $padding;
4660
4661     mon_cmd(
4662         $vmid,
4663         "block_resize",
4664         device => $deviceid,
4665         size => int($size),
4666         timeout => 60,
4667     );
4668 }
4669
4670 sub qemu_volume_snapshot {
4671     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4672
4673     my $running = check_running($vmid);
4674
4675     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4676         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4677     } else {
4678         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4679     }
4680 }
4681
4682 sub qemu_volume_snapshot_delete {
4683     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4684
4685     my $running = check_running($vmid);
4686
4687     if($running) {
4688
4689         $running = undef;
4690         my $conf = PVE::QemuConfig->load_config($vmid);
4691         PVE::QemuConfig->foreach_volume($conf, sub {
4692             my ($ds, $drive) = @_;
4693             $running = 1 if $drive->{file} eq $volid;
4694         });
4695     }
4696
4697     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4698         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4699     } else {
4700         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4701     }
4702 }
4703
4704 sub set_migration_caps {
4705     my ($vmid, $savevm) = @_;
4706
4707     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4708
4709     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4710     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4711
4712     my $cap_ref = [];
4713
4714     my $enabled_cap = {
4715         "auto-converge" => 1,
4716         "xbzrle" => 1,
4717         "x-rdma-pin-all" => 0,
4718         "zero-blocks" => 0,
4719         "compress" => 0,
4720         "dirty-bitmaps" => $dirty_bitmaps,
4721     };
4722
4723     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4724
4725     for my $supported_capability (@$supported_capabilities) {
4726         push @$cap_ref, {
4727             capability => $supported_capability->{capability},
4728             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4729         };
4730     }
4731
4732     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4733 }
4734
4735 sub foreach_volid {
4736     my ($conf, $func, @param) = @_;
4737
4738     my $volhash = {};
4739
4740     my $test_volid = sub {
4741         my ($key, $drive, $snapname) = @_;
4742
4743         my $volid = $drive->{file};
4744         return if !$volid;
4745
4746         $volhash->{$volid}->{cdrom} //= 1;
4747         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4748
4749         my $replicate = $drive->{replicate} // 1;
4750         $volhash->{$volid}->{replicate} //= 0;
4751         $volhash->{$volid}->{replicate} = 1 if $replicate;
4752
4753         $volhash->{$volid}->{shared} //= 0;
4754         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4755
4756         $volhash->{$volid}->{referenced_in_config} //= 0;
4757         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4758
4759         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4760             if defined($snapname);
4761
4762         my $size = $drive->{size};
4763         $volhash->{$volid}->{size} //= $size if $size;
4764
4765         $volhash->{$volid}->{is_vmstate} //= 0;
4766         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4767
4768         $volhash->{$volid}->{is_tpmstate} //= 0;
4769         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4770
4771         $volhash->{$volid}->{is_unused} //= 0;
4772         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4773
4774         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4775     };
4776
4777     my $include_opts = {
4778         extra_keys => ['vmstate'],
4779         include_unused => 1,
4780     };
4781
4782     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4783     foreach my $snapname (keys %{$conf->{snapshots}}) {
4784         my $snap = $conf->{snapshots}->{$snapname};
4785         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4786     }
4787
4788     foreach my $volid (keys %$volhash) {
4789         &$func($volid, $volhash->{$volid}, @param);
4790     }
4791 }
4792
4793 my $fast_plug_option = {
4794     'lock' => 1,
4795     'name' => 1,
4796     'onboot' => 1,
4797     'shares' => 1,
4798     'startup' => 1,
4799     'description' => 1,
4800     'protection' => 1,
4801     'vmstatestorage' => 1,
4802     'hookscript' => 1,
4803     'tags' => 1,
4804 };
4805
4806 # hotplug changes in [PENDING]
4807 # $selection hash can be used to only apply specified options, for
4808 # example: { cores => 1 } (only apply changed 'cores')
4809 # $errors ref is used to return error messages
4810 sub vmconfig_hotplug_pending {
4811     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4812
4813     my $defaults = load_defaults();
4814     my $arch = get_vm_arch($conf);
4815     my $machine_type = get_vm_machine($conf, undef, $arch);
4816
4817     # commit values which do not have any impact on running VM first
4818     # Note: those option cannot raise errors, we we do not care about
4819     # $selection and always apply them.
4820
4821     my $add_error = sub {
4822         my ($opt, $msg) = @_;
4823         $errors->{$opt} = "hotplug problem - $msg";
4824     };
4825
4826     my $changes = 0;
4827     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4828         if ($fast_plug_option->{$opt}) {
4829             $conf->{$opt} = $conf->{pending}->{$opt};
4830             delete $conf->{pending}->{$opt};
4831             $changes = 1;
4832         }
4833     }
4834
4835     if ($changes) {
4836         PVE::QemuConfig->write_config($vmid, $conf);
4837     }
4838
4839     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4840
4841     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4842     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4843     foreach my $opt (sort keys %$pending_delete_hash) {
4844         next if $selection && !$selection->{$opt};
4845         my $force = $pending_delete_hash->{$opt}->{force};
4846         eval {
4847             if ($opt eq 'hotplug') {
4848                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4849             } elsif ($opt eq 'tablet') {
4850                 die "skip\n" if !$hotplug_features->{usb};
4851                 if ($defaults->{tablet}) {
4852                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4853                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4854                         if $arch eq 'aarch64';
4855                 } else {
4856                     vm_deviceunplug($vmid, $conf, 'tablet');
4857                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4858                 }
4859             } elsif ($opt =~ m/^usb\d+/) {
4860                 die "skip\n";
4861                 # since we cannot reliably hot unplug usb devices we are disabling it
4862                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4863                 #vm_deviceunplug($vmid, $conf, $opt);
4864             } elsif ($opt eq 'vcpus') {
4865                 die "skip\n" if !$hotplug_features->{cpu};
4866                 qemu_cpu_hotplug($vmid, $conf, undef);
4867             } elsif ($opt eq 'balloon') {
4868                 # enable balloon device is not hotpluggable
4869                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4870                 # here we reset the ballooning value to memory
4871                 my $balloon = $conf->{memory} || $defaults->{memory};
4872                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4873             } elsif ($fast_plug_option->{$opt}) {
4874                 # do nothing
4875             } elsif ($opt =~ m/^net(\d+)$/) {
4876                 die "skip\n" if !$hotplug_features->{network};
4877                 vm_deviceunplug($vmid, $conf, $opt);
4878             } elsif (is_valid_drivename($opt)) {
4879                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4880                 vm_deviceunplug($vmid, $conf, $opt);
4881                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4882             } elsif ($opt =~ m/^memory$/) {
4883                 die "skip\n" if !$hotplug_features->{memory};
4884                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4885             } elsif ($opt eq 'cpuunits') {
4886                 $cgroup->change_cpu_shares(undef, 1024);
4887             } elsif ($opt eq 'cpulimit') {
4888                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4889             } else {
4890                 die "skip\n";
4891             }
4892         };
4893         if (my $err = $@) {
4894             &$add_error($opt, $err) if $err ne "skip\n";
4895         } else {
4896             delete $conf->{$opt};
4897             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4898         }
4899     }
4900
4901     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4902     $apply_pending_cloudinit = sub {
4903         return if $apply_pending_cloudinit_done; # once is enough
4904         $apply_pending_cloudinit_done = 1; # once is enough
4905
4906         my ($key, $value) = @_;
4907
4908         my @cloudinit_opts = keys %$confdesc_cloudinit;
4909         foreach my $opt (keys %{$conf->{pending}}) {
4910             next if !grep { $_ eq $opt } @cloudinit_opts;
4911             $conf->{$opt} = delete $conf->{pending}->{$opt};
4912         }
4913
4914         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4915         foreach my $opt (sort keys %$pending_delete_hash) {
4916             next if !grep { $_ eq $opt } @cloudinit_opts;
4917             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4918             delete $conf->{$opt};
4919         }
4920
4921         my $new_conf = { %$conf };
4922         $new_conf->{$key} = $value;
4923         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4924     };
4925
4926     foreach my $opt (keys %{$conf->{pending}}) {
4927         next if $selection && !$selection->{$opt};
4928         my $value = $conf->{pending}->{$opt};
4929         eval {
4930             if ($opt eq 'hotplug') {
4931                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4932             } elsif ($opt eq 'tablet') {
4933                 die "skip\n" if !$hotplug_features->{usb};
4934                 if ($value == 1) {
4935                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4936                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4937                         if $arch eq 'aarch64';
4938                 } elsif ($value == 0) {
4939                     vm_deviceunplug($vmid, $conf, 'tablet');
4940                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4941                 }
4942             } elsif ($opt =~ m/^usb\d+$/) {
4943                 die "skip\n";
4944                 # since we cannot reliably hot unplug usb devices we disable it for now
4945                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4946                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4947                 #die "skip\n" if !$d;
4948                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4949             } elsif ($opt eq 'vcpus') {
4950                 die "skip\n" if !$hotplug_features->{cpu};
4951                 qemu_cpu_hotplug($vmid, $conf, $value);
4952             } elsif ($opt eq 'balloon') {
4953                 # enable/disable balloning device is not hotpluggable
4954                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4955                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4956                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4957
4958                 # allow manual ballooning if shares is set to zero
4959                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4960                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4961                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4962                 }
4963             } elsif ($opt =~ m/^net(\d+)$/) {
4964                 # some changes can be done without hotplug
4965                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4966                                     $vmid, $opt, $value, $arch, $machine_type);
4967             } elsif (is_valid_drivename($opt)) {
4968                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4969                 # some changes can be done without hotplug
4970                 my $drive = parse_drive($opt, $value);
4971                 if (drive_is_cloudinit($drive)) {
4972                     &$apply_pending_cloudinit($opt, $value);
4973                 }
4974                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4975                                      $vmid, $opt, $value, $arch, $machine_type);
4976             } elsif ($opt =~ m/^memory$/) { #dimms
4977                 die "skip\n" if !$hotplug_features->{memory};
4978                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4979             } elsif ($opt eq 'cpuunits') {
4980                 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4981                 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4982             } elsif ($opt eq 'cpulimit') {
4983                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4984                 $cgroup->change_cpu_quota($cpulimit, 100000);
4985             } elsif ($opt eq 'agent') {
4986                 vmconfig_update_agent($conf, $opt, $value);
4987             } else {
4988                 die "skip\n";  # skip non-hot-pluggable options
4989             }
4990         };
4991         if (my $err = $@) {
4992             &$add_error($opt, $err) if $err ne "skip\n";
4993         } else {
4994             $conf->{$opt} = $value;
4995             delete $conf->{pending}->{$opt};
4996         }
4997     }
4998
4999     PVE::QemuConfig->write_config($vmid, $conf);
5000 }
5001
5002 sub try_deallocate_drive {
5003     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5004
5005     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5006         my $volid = $drive->{file};
5007         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5008             my $sid = PVE::Storage::parse_volume_id($volid);
5009             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5010
5011             # check if the disk is really unused
5012             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5013                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5014             PVE::Storage::vdisk_free($storecfg, $volid);
5015             return 1;
5016         } else {
5017             # If vm is not owner of this disk remove from config
5018             return 1;
5019         }
5020     }
5021
5022     return;
5023 }
5024
5025 sub vmconfig_delete_or_detach_drive {
5026     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5027
5028     my $drive = parse_drive($opt, $conf->{$opt});
5029
5030     my $rpcenv = PVE::RPCEnvironment::get();
5031     my $authuser = $rpcenv->get_user();
5032
5033     if ($force) {
5034         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5035         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5036     } else {
5037         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5038     }
5039 }
5040
5041
5042
5043 sub vmconfig_apply_pending {
5044     my ($vmid, $conf, $storecfg, $errors) = @_;
5045
5046     return if !scalar(keys %{$conf->{pending}});
5047
5048     my $add_apply_error = sub {
5049         my ($opt, $msg) = @_;
5050         my $err_msg = "unable to apply pending change $opt : $msg";
5051         $errors->{$opt} = $err_msg;
5052         warn $err_msg;
5053     };
5054
5055     # cold plug
5056
5057     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5058     foreach my $opt (sort keys %$pending_delete_hash) {
5059         my $force = $pending_delete_hash->{$opt}->{force};
5060         eval {
5061             if ($opt =~ m/^unused/) {
5062                 die "internal error";
5063             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5064                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5065             }
5066         };
5067         if (my $err = $@) {
5068             $add_apply_error->($opt, $err);
5069         } else {
5070             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5071             delete $conf->{$opt};
5072         }
5073     }
5074
5075     PVE::QemuConfig->cleanup_pending($conf);
5076
5077     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5078         next if $opt eq 'delete'; # just to be sure
5079         eval {
5080             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5081                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5082             }
5083         };
5084         if (my $err = $@) {
5085             $add_apply_error->($opt, $err);
5086         } else {
5087             $conf->{$opt} = delete $conf->{pending}->{$opt};
5088         }
5089     }
5090
5091     # write all changes at once to avoid unnecessary i/o
5092     PVE::QemuConfig->write_config($vmid, $conf);
5093 }
5094
5095 sub vmconfig_update_net {
5096     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5097
5098     my $newnet = parse_net($value);
5099
5100     if ($conf->{$opt}) {
5101         my $oldnet = parse_net($conf->{$opt});
5102
5103         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5104             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5105             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5106             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5107
5108             # for non online change, we try to hot-unplug
5109             die "skip\n" if !$hotplug;
5110             vm_deviceunplug($vmid, $conf, $opt);
5111         } else {
5112
5113             die "internal error" if $opt !~ m/net(\d+)/;
5114             my $iface = "tap${vmid}i$1";
5115
5116             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5117                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5118                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5119                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5120                 PVE::Network::tap_unplug($iface);
5121
5122                 if ($have_sdn) {
5123                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5124                 } else {
5125                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5126                 }
5127             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5128                 # Rate can be applied on its own but any change above needs to
5129                 # include the rate in tap_plug since OVS resets everything.
5130                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5131             }
5132
5133             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5134                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5135             }
5136
5137             return 1;
5138         }
5139     }
5140
5141     if ($hotplug) {
5142         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5143     } else {
5144         die "skip\n";
5145     }
5146 }
5147
5148 sub vmconfig_update_agent {
5149     my ($conf, $opt, $value) = @_;
5150
5151     die "skip\n" if !$conf->{$opt};
5152
5153     my $hotplug_options = { fstrim_cloned_disks => 1 };
5154
5155     my $old_agent = parse_guest_agent($conf);
5156     my $agent = parse_guest_agent({$opt => $value});
5157
5158     for my $option (keys %$agent) { # added/changed options
5159         next if defined($hotplug_options->{$option});
5160         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5161     }
5162
5163     for my $option (keys %$old_agent) { # removed options
5164         next if defined($hotplug_options->{$option});
5165         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5166     }
5167
5168     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5169 }
5170
5171 sub vmconfig_update_disk {
5172     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5173
5174     my $drive = parse_drive($opt, $value);
5175
5176     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5177         my $media = $drive->{media} || 'disk';
5178         my $oldmedia = $old_drive->{media} || 'disk';
5179         die "unable to change media type\n" if $media ne $oldmedia;
5180
5181         if (!drive_is_cdrom($old_drive)) {
5182
5183             if ($drive->{file} ne $old_drive->{file}) {
5184
5185                 die "skip\n" if !$hotplug;
5186
5187                 # unplug and register as unused
5188                 vm_deviceunplug($vmid, $conf, $opt);
5189                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5190
5191             } else {
5192                 # update existing disk
5193
5194                 # skip non hotpluggable value
5195                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5196                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5197                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5198                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5199                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5200                     die "skip\n";
5201                 }
5202
5203                 # apply throttle
5204                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5205                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5206                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5207                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5208                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5209                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5210                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5211                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5212                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5213                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5214                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5215                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5216                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5217                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5218                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5219                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5220                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5221                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5222
5223                     qemu_block_set_io_throttle(
5224                         $vmid,"drive-$opt",
5225                         ($drive->{mbps} || 0)*1024*1024,
5226                         ($drive->{mbps_rd} || 0)*1024*1024,
5227                         ($drive->{mbps_wr} || 0)*1024*1024,
5228                          $drive->{iops} || 0,
5229                          $drive->{iops_rd} || 0,
5230                          $drive->{iops_wr} || 0,
5231                         ($drive->{mbps_max} || 0)*1024*1024,
5232                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5233                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5234                          $drive->{iops_max} || 0,
5235                          $drive->{iops_rd_max} || 0,
5236                          $drive->{iops_wr_max} || 0,
5237                          $drive->{bps_max_length} || 1,
5238                          $drive->{bps_rd_max_length} || 1,
5239                          $drive->{bps_wr_max_length} || 1,
5240                          $drive->{iops_max_length} || 1,
5241                          $drive->{iops_rd_max_length} || 1,
5242                          $drive->{iops_wr_max_length} || 1,
5243                     );
5244
5245                 }
5246
5247                 return 1;
5248             }
5249
5250         } else { # cdrom
5251
5252             if ($drive->{file} eq 'none') {
5253                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5254                 if (drive_is_cloudinit($old_drive)) {
5255                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5256                 }
5257             } else {
5258                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5259
5260                 # force eject if locked
5261                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5262
5263                 if ($path) {
5264                     mon_cmd($vmid, "blockdev-change-medium",
5265                         id => "$opt", filename => "$path");
5266                 }
5267             }
5268
5269             return 1;
5270         }
5271     }
5272
5273     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5274     # hotplug new disks
5275     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5276     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5277 }
5278
5279 # called in locked context by incoming migration
5280 sub vm_migrate_get_nbd_disks {
5281     my ($storecfg, $conf, $replicated_volumes) = @_;
5282
5283     my $local_volumes = {};
5284     PVE::QemuConfig->foreach_volume($conf, sub {
5285         my ($ds, $drive) = @_;
5286
5287         return if drive_is_cdrom($drive);
5288         return if $ds eq 'tpmstate0';
5289
5290         my $volid = $drive->{file};
5291
5292         return if !$volid;
5293
5294         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5295
5296         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5297         return if $scfg->{shared};
5298
5299         # replicated disks re-use existing state via bitmap
5300         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5301         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5302     });
5303     return $local_volumes;
5304 }
5305
5306 # called in locked context by incoming migration
5307 sub vm_migrate_alloc_nbd_disks {
5308     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5309
5310     my $nbd = {};
5311     foreach my $opt (sort keys %$source_volumes) {
5312         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5313
5314         if ($use_existing) {
5315             $nbd->{$opt}->{drivestr} = print_drive($drive);
5316             $nbd->{$opt}->{volid} = $volid;
5317             $nbd->{$opt}->{replicated} = 1;
5318             next;
5319         }
5320
5321         # storage mapping + volname = regular migration
5322         # storage mapping + format = remote migration
5323         # order of precedence, filtered by whether storage supports it:
5324         # 1. explicit requested format
5325         # 2. format of current volume
5326         # 3. default format of storage
5327         if (!$storagemap->{identity}) {
5328             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5329             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5330             if (!$format || !grep { $format eq $_ } @$validFormats) {
5331                 if ($volname) {
5332                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5333                     my $fileFormat = qemu_img_format($scfg, $volname);
5334                     $format = $fileFormat
5335                         if grep { $fileFormat eq $_ } @$validFormats;
5336                 }
5337                 $format //= $defFormat;
5338             }
5339         } else {
5340             # can't happen for remote migration, so $volname is always defined
5341             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5342             $format = qemu_img_format($scfg, $volname);
5343         }
5344
5345         my $size = $drive->{size} / 1024;
5346         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5347         my $newdrive = $drive;
5348         $newdrive->{format} = $format;
5349         $newdrive->{file} = $newvolid;
5350         my $drivestr = print_drive($newdrive);
5351         $nbd->{$opt}->{drivestr} = $drivestr;
5352         $nbd->{$opt}->{volid} = $newvolid;
5353     }
5354
5355     return $nbd;
5356 }
5357
5358 # see vm_start_nolock for parameters, additionally:
5359 # migrate_opts:
5360 #   storagemap = parsed storage map for allocating NBD disks
5361 sub vm_start {
5362     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5363
5364     return PVE::QemuConfig->lock_config($vmid, sub {
5365         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5366
5367         die "you can't start a vm if it's a template\n"
5368             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5369
5370         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5371         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5372
5373         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5374
5375         if ($has_backup_lock && $running) {
5376             # a backup is currently running, attempt to start the guest in the
5377             # existing QEMU instance
5378             return vm_resume($vmid);
5379         }
5380
5381         PVE::QemuConfig->check_lock($conf)
5382             if !($params->{skiplock} || $has_suspended_lock);
5383
5384         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5385
5386         die "VM $vmid already running\n" if $running;
5387
5388         if (my $storagemap = $migrate_opts->{storagemap}) {
5389             my $replicated = $migrate_opts->{replicated_volumes};
5390             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5391             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5392
5393             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5394                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5395             }
5396         }
5397
5398         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5399     });
5400 }
5401
5402
5403 # params:
5404 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5405 #   skiplock => 0/1, skip checking for config lock
5406 #   skiptemplate => 0/1, skip checking whether VM is template
5407 #   forcemachine => to force Qemu machine (rollback/migration)
5408 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5409 #   timeout => in seconds
5410 #   paused => start VM in paused state (backup)
5411 #   resume => resume from hibernation
5412 #   pbs-backing => {
5413 #      sata0 => {
5414 #         repository
5415 #         snapshot
5416 #         keyfile
5417 #         archive
5418 #      },
5419 #      virtio2 => ...
5420 #   }
5421 # migrate_opts:
5422 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5423 #   migratedfrom => source node
5424 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5425 #   network => CIDR of migration network
5426 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5427 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5428 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5429 #   tpmstate_vol => new volid of tpmstate0, not yet contained in config
5430 sub vm_start_nolock {
5431     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5432
5433     my $statefile = $params->{statefile};
5434     my $resume = $params->{resume};
5435
5436     my $migratedfrom = $migrate_opts->{migratedfrom};
5437     my $migration_type = $migrate_opts->{type};
5438
5439     my $res = {};
5440
5441     # clean up leftover reboot request files
5442     eval { clear_reboot_request($vmid); };
5443     warn $@ if $@;
5444
5445     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5446         vmconfig_apply_pending($vmid, $conf, $storecfg);
5447         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5448     }
5449
5450     # don't regenerate the ISO if the VM is started as part of a live migration
5451     # this way we can reuse the old ISO with the correct config
5452     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5453
5454     # override TPM state vol if migrated, conf is out of date still
5455     if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
5456         my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
5457         $parsed->{file} = $tpmvol;
5458         $conf->{tpmstate0} = print_drive($parsed);
5459     }
5460
5461     my $defaults = load_defaults();
5462
5463     # set environment variable useful inside network script
5464     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5465
5466     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5467
5468     my $forcemachine = $params->{forcemachine};
5469     my $forcecpu = $params->{forcecpu};
5470     if ($resume) {
5471         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5472         $forcemachine = $conf->{runningmachine};
5473         $forcecpu = $conf->{runningcpu};
5474         print "Resuming suspended VM\n";
5475     }
5476
5477     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5478         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5479
5480     my $migration_ip;
5481     my $get_migration_ip = sub {
5482         my ($nodename) = @_;
5483
5484         return $migration_ip if defined($migration_ip);
5485
5486         my $cidr = $migrate_opts->{network};
5487
5488         if (!defined($cidr)) {
5489             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5490             $cidr = $dc_conf->{migration}->{network};
5491         }
5492
5493         if (defined($cidr)) {
5494             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5495
5496             die "could not get IP: no address configured on local " .
5497                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5498
5499             die "could not get IP: multiple addresses configured on local " .
5500                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5501
5502             $migration_ip = @$ips[0];
5503         }
5504
5505         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5506             if !defined($migration_ip);
5507
5508         return $migration_ip;
5509     };
5510
5511     my $migrate_uri;
5512     if ($statefile) {
5513         if ($statefile eq 'tcp') {
5514             my $localip = "localhost";
5515             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5516             my $nodename = nodename();
5517
5518             if (!defined($migration_type)) {
5519                 if (defined($datacenterconf->{migration}->{type})) {
5520                     $migration_type = $datacenterconf->{migration}->{type};
5521                 } else {
5522                     $migration_type = 'secure';
5523                 }
5524             }
5525
5526             if ($migration_type eq 'insecure') {
5527                 $localip = $get_migration_ip->($nodename);
5528                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5529             }
5530
5531             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5532             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5533             $migrate_uri = "tcp:${localip}:${migrate_port}";
5534             push @$cmd, '-incoming', $migrate_uri;
5535             push @$cmd, '-S';
5536
5537         } elsif ($statefile eq 'unix') {
5538             # should be default for secure migrations as a ssh TCP forward
5539             # tunnel is not deterministic reliable ready and fails regurarly
5540             # to set up in time, so use UNIX socket forwards
5541             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5542             unlink $socket_addr;
5543
5544             $migrate_uri = "unix:$socket_addr";
5545
5546             push @$cmd, '-incoming', $migrate_uri;
5547             push @$cmd, '-S';
5548
5549         } elsif (-e $statefile) {
5550             push @$cmd, '-loadstate', $statefile;
5551         } else {
5552             my $statepath = PVE::Storage::path($storecfg, $statefile);
5553             push @$vollist, $statefile;
5554             push @$cmd, '-loadstate', $statepath;
5555         }
5556     } elsif ($params->{paused}) {
5557         push @$cmd, '-S';
5558     }
5559
5560     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5561
5562     my $pci_devices = {}; # host pci devices
5563     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5564         my $dev = $conf->{"hostpci$i"} or next;
5565         $pci_devices->{$i} = parse_hostpci($dev);
5566     }
5567
5568     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5569     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5570
5571     # map to a flat list of pci ids
5572     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5573
5574     # reserve all PCI IDs before actually doing anything with them
5575     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5576
5577     eval {
5578         for my $id (sort keys %$pci_devices) {
5579             my $d = $pci_devices->{$id};
5580             for my $dev ($d->{pciid}->@*) {
5581                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5582             }
5583         }
5584     };
5585     if (my $err = $@) {
5586         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5587         warn $@ if $@;
5588         die $err;
5589     }
5590
5591     PVE::Storage::activate_volumes($storecfg, $vollist);
5592
5593     eval {
5594         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5595     };
5596     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5597     # timeout should be more than enough here...
5598     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5599
5600     my $cpuunits = get_cpuunits($conf);
5601
5602     my %run_params = (
5603         timeout => $statefile ? undef : $start_timeout,
5604         umask => 0077,
5605         noerr => 1,
5606     );
5607
5608     # when migrating, prefix QEMU output so other side can pick up any
5609     # errors that might occur and show the user
5610     if ($migratedfrom) {
5611         $run_params{quiet} = 1;
5612         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5613     }
5614
5615     my %systemd_properties = (
5616         Slice => 'qemu.slice',
5617         KillMode => 'process',
5618         SendSIGKILL => 0,
5619         TimeoutStopUSec => ULONG_MAX, # infinity
5620     );
5621
5622     if (PVE::CGroup::cgroup_mode() == 2) {
5623         $systemd_properties{CPUWeight} = $cpuunits;
5624     } else {
5625         $systemd_properties{CPUShares} = $cpuunits;
5626     }
5627
5628     if (my $cpulimit = $conf->{cpulimit}) {
5629         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5630     }
5631     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5632
5633     my $run_qemu = sub {
5634         PVE::Tools::run_fork sub {
5635             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5636
5637             my $tpmpid;
5638             if (my $tpm = $conf->{tpmstate0}) {
5639                 # start the TPM emulator so QEMU can connect on start
5640                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5641             }
5642
5643             my $exitcode = run_command($cmd, %run_params);
5644             if ($exitcode) {
5645                 if ($tpmpid) {
5646                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5647                     kill 'TERM', $tpmpid;
5648                 }
5649                 die "QEMU exited with code $exitcode\n";
5650             }
5651         };
5652     };
5653
5654     if ($conf->{hugepages}) {
5655
5656         my $code = sub {
5657             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5658             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5659
5660             PVE::QemuServer::Memory::hugepages_mount();
5661             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5662
5663             eval { $run_qemu->() };
5664             if (my $err = $@) {
5665                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5666                     if !$conf->{keephugepages};
5667                 die $err;
5668             }
5669
5670             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5671                 if !$conf->{keephugepages};
5672         };
5673         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5674
5675     } else {
5676         eval { $run_qemu->() };
5677     }
5678
5679     if (my $err = $@) {
5680         # deactivate volumes if start fails
5681         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5682         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5683
5684         die "start failed: $err";
5685     }
5686
5687     # re-reserve all PCI IDs now that we can know the actual VM PID
5688     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5689     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5690     warn $@ if $@;
5691
5692     print "migration listens on $migrate_uri\n" if $migrate_uri;
5693     $res->{migrate_uri} = $migrate_uri;
5694
5695     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5696         eval { mon_cmd($vmid, "cont"); };
5697         warn $@ if $@;
5698     }
5699
5700     #start nbd server for storage migration
5701     if (my $nbd = $migrate_opts->{nbd}) {
5702         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5703
5704         my $migrate_storage_uri;
5705         # nbd_protocol_version > 0 for unix socket support
5706         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5707             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5708             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5709             $migrate_storage_uri = "nbd:unix:$socket_path";
5710         } else {
5711             my $nodename = nodename();
5712             my $localip = $get_migration_ip->($nodename);
5713             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5714             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5715
5716             mon_cmd($vmid, "nbd-server-start", addr => {
5717                 type => 'inet',
5718                 data => {
5719                     host => "${localip}",
5720                     port => "${storage_migrate_port}",
5721                 },
5722             });
5723             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5724             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5725         }
5726
5727         $res->{migrate_storage_uri} = $migrate_storage_uri;
5728
5729         foreach my $opt (sort keys %$nbd) {
5730             my $drivestr = $nbd->{$opt}->{drivestr};
5731             my $volid = $nbd->{$opt}->{volid};
5732             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5733             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5734             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5735             print "re-using replicated volume: $opt - $volid\n"
5736                 if $nbd->{$opt}->{replicated};
5737
5738             $res->{drives}->{$opt} = $nbd->{$opt};
5739             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5740         }
5741     }
5742
5743     if ($migratedfrom) {
5744         eval {
5745             set_migration_caps($vmid);
5746         };
5747         warn $@ if $@;
5748
5749         if ($spice_port) {
5750             print "spice listens on port $spice_port\n";
5751             $res->{spice_port} = $spice_port;
5752             if ($migrate_opts->{spice_ticket}) {
5753                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5754                     $migrate_opts->{spice_ticket});
5755                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5756             }
5757         }
5758
5759     } else {
5760         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5761             if !$statefile && $conf->{balloon};
5762
5763         foreach my $opt (keys %$conf) {
5764             next if $opt !~  m/^net\d+$/;
5765             my $nicconf = parse_net($conf->{$opt});
5766             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5767         }
5768     }
5769
5770     mon_cmd($vmid, 'qom-set',
5771                 path => "machine/peripheral/balloon0",
5772                 property => "guest-stats-polling-interval",
5773                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5774
5775     if ($resume) {
5776         print "Resumed VM, removing state\n";
5777         if (my $vmstate = $conf->{vmstate}) {
5778             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5779             PVE::Storage::vdisk_free($storecfg, $vmstate);
5780         }
5781         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5782         PVE::QemuConfig->write_config($vmid, $conf);
5783     }
5784
5785     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5786
5787     return $res;
5788 }
5789
5790 sub vm_commandline {
5791     my ($storecfg, $vmid, $snapname) = @_;
5792
5793     my $conf = PVE::QemuConfig->load_config($vmid);
5794
5795     my ($forcemachine, $forcecpu);
5796     if ($snapname) {
5797         my $snapshot = $conf->{snapshots}->{$snapname};
5798         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5799
5800         # check for machine or CPU overrides in snapshot
5801         $forcemachine = $snapshot->{runningmachine};
5802         $forcecpu = $snapshot->{runningcpu};
5803
5804         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5805
5806         $conf = $snapshot;
5807     }
5808
5809     my $defaults = load_defaults();
5810
5811     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5812
5813     return PVE::Tools::cmd2string($cmd);
5814 }
5815
5816 sub vm_reset {
5817     my ($vmid, $skiplock) = @_;
5818
5819     PVE::QemuConfig->lock_config($vmid, sub {
5820
5821         my $conf = PVE::QemuConfig->load_config($vmid);
5822
5823         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5824
5825         mon_cmd($vmid, "system_reset");
5826     });
5827 }
5828
5829 sub get_vm_volumes {
5830     my ($conf) = @_;
5831
5832     my $vollist = [];
5833     foreach_volid($conf, sub {
5834         my ($volid, $attr) = @_;
5835
5836         return if $volid =~ m|^/|;
5837
5838         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5839         return if !$sid;
5840
5841         push @$vollist, $volid;
5842     });
5843
5844     return $vollist;
5845 }
5846
5847 sub vm_stop_cleanup {
5848     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5849
5850     eval {
5851
5852         if (!$keepActive) {
5853             my $vollist = get_vm_volumes($conf);
5854             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5855
5856             if (my $tpmdrive = $conf->{tpmstate0}) {
5857                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5858                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5859                 if ($storeid) {
5860                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5861                 }
5862             }
5863         }
5864
5865         foreach my $ext (qw(mon qmp pid vnc qga)) {
5866             unlink "/var/run/qemu-server/${vmid}.$ext";
5867         }
5868
5869         if ($conf->{ivshmem}) {
5870             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5871             # just delete it for now, VMs which have this already open do not
5872             # are affected, but new VMs will get a separated one. If this
5873             # becomes an issue we either add some sort of ref-counting or just
5874             # add a "don't delete on stop" flag to the ivshmem format.
5875             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5876         }
5877
5878         my $ids = [];
5879         foreach my $key (keys %$conf) {
5880             next if $key !~ m/^hostpci(\d+)$/;
5881             my $hostpciindex = $1;
5882             my $d = parse_hostpci($conf->{$key});
5883             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5884
5885             foreach my $pci (@{$d->{pciid}}) {
5886                 my $pciid = $pci->{id};
5887                 push @$ids, $pci->{id};
5888                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5889             }
5890         }
5891         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5892
5893         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5894     };
5895     warn $@ if $@; # avoid errors - just warn
5896 }
5897
5898 # call only in locked context
5899 sub _do_vm_stop {
5900     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5901
5902     my $pid = check_running($vmid, $nocheck);
5903     return if !$pid;
5904
5905     my $conf;
5906     if (!$nocheck) {
5907         $conf = PVE::QemuConfig->load_config($vmid);
5908         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5909         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5910             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5911             $timeout = $opts->{down} if $opts->{down};
5912         }
5913         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5914     }
5915
5916     eval {
5917         if ($shutdown) {
5918             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5919                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5920             } else {
5921                 mon_cmd($vmid, "system_powerdown");
5922             }
5923         } else {
5924             mon_cmd($vmid, "quit");
5925         }
5926     };
5927     my $err = $@;
5928
5929     if (!$err) {
5930         $timeout = 60 if !defined($timeout);
5931
5932         my $count = 0;
5933         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5934             $count++;
5935             sleep 1;
5936         }
5937
5938         if ($count >= $timeout) {
5939             if ($force) {
5940                 warn "VM still running - terminating now with SIGTERM\n";
5941                 kill 15, $pid;
5942             } else {
5943                 die "VM quit/powerdown failed - got timeout\n";
5944             }
5945         } else {
5946             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5947             return;
5948         }
5949     } else {
5950        if (!check_running($vmid, $nocheck)) {
5951             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5952             return;
5953        }
5954        if ($force) {
5955             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5956             kill 15, $pid;
5957         } else {
5958             die "VM quit/powerdown failed\n";
5959         }
5960     }
5961
5962     # wait again
5963     $timeout = 10;
5964
5965     my $count = 0;
5966     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5967         $count++;
5968         sleep 1;
5969     }
5970
5971     if ($count >= $timeout) {
5972         warn "VM still running - terminating now with SIGKILL\n";
5973         kill 9, $pid;
5974         sleep 1;
5975     }
5976
5977     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5978 }
5979
5980 # Note: use $nocheck to skip tests if VM configuration file exists.
5981 # We need that when migration VMs to other nodes (files already moved)
5982 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5983 sub vm_stop {
5984     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5985
5986     $force = 1 if !defined($force) && !$shutdown;
5987
5988     if ($migratedfrom){
5989         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5990         kill 15, $pid if $pid;
5991         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5992         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5993         return;
5994     }
5995
5996     PVE::QemuConfig->lock_config($vmid, sub {
5997         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5998    });
5999 }
6000
6001 sub vm_reboot {
6002     my ($vmid, $timeout) = @_;
6003
6004     PVE::QemuConfig->lock_config($vmid, sub {
6005         eval {
6006
6007             # only reboot if running, as qmeventd starts it again on a stop event
6008             return if !check_running($vmid);
6009
6010             create_reboot_request($vmid);
6011
6012             my $storecfg = PVE::Storage::config();
6013             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6014
6015         };
6016         if (my $err = $@) {
6017             # avoid that the next normal shutdown will be confused for a reboot
6018             clear_reboot_request($vmid);
6019             die $err;
6020         }
6021    });
6022 }
6023
6024 # note: if using the statestorage parameter, the caller has to check privileges
6025 sub vm_suspend {
6026     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6027
6028     my $conf;
6029     my $path;
6030     my $storecfg;
6031     my $vmstate;
6032
6033     PVE::QemuConfig->lock_config($vmid, sub {
6034
6035         $conf = PVE::QemuConfig->load_config($vmid);
6036
6037         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6038         PVE::QemuConfig->check_lock($conf)
6039             if !($skiplock || $is_backing_up);
6040
6041         die "cannot suspend to disk during backup\n"
6042             if $is_backing_up && $includestate;
6043
6044         if ($includestate) {
6045             $conf->{lock} = 'suspending';
6046             my $date = strftime("%Y-%m-%d", localtime(time()));
6047             $storecfg = PVE::Storage::config();
6048             if (!$statestorage) {
6049                 $statestorage = find_vmstate_storage($conf, $storecfg);
6050                 # check permissions for the storage
6051                 my $rpcenv = PVE::RPCEnvironment::get();
6052                 if ($rpcenv->{type} ne 'cli') {
6053                     my $authuser = $rpcenv->get_user();
6054                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6055                 }
6056             }
6057
6058
6059             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6060                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6061             $path = PVE::Storage::path($storecfg, $vmstate);
6062             PVE::QemuConfig->write_config($vmid, $conf);
6063         } else {
6064             mon_cmd($vmid, "stop");
6065         }
6066     });
6067
6068     if ($includestate) {
6069         # save vm state
6070         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6071
6072         eval {
6073             set_migration_caps($vmid, 1);
6074             mon_cmd($vmid, "savevm-start", statefile => $path);
6075             for(;;) {
6076                 my $state = mon_cmd($vmid, "query-savevm");
6077                 if (!$state->{status}) {
6078                     die "savevm not active\n";
6079                 } elsif ($state->{status} eq 'active') {
6080                     sleep(1);
6081                     next;
6082                 } elsif ($state->{status} eq 'completed') {
6083                     print "State saved, quitting\n";
6084                     last;
6085                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6086                     die "query-savevm failed with error '$state->{error}'\n"
6087                 } else {
6088                     die "query-savevm returned status '$state->{status}'\n";
6089                 }
6090             }
6091         };
6092         my $err = $@;
6093
6094         PVE::QemuConfig->lock_config($vmid, sub {
6095             $conf = PVE::QemuConfig->load_config($vmid);
6096             if ($err) {
6097                 # cleanup, but leave suspending lock, to indicate something went wrong
6098                 eval {
6099                     mon_cmd($vmid, "savevm-end");
6100                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6101                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6102                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6103                     PVE::QemuConfig->write_config($vmid, $conf);
6104                 };
6105                 warn $@ if $@;
6106                 die $err;
6107             }
6108
6109             die "lock changed unexpectedly\n"
6110                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6111
6112             mon_cmd($vmid, "quit");
6113             $conf->{lock} = 'suspended';
6114             PVE::QemuConfig->write_config($vmid, $conf);
6115         });
6116     }
6117 }
6118
6119 sub vm_resume {
6120     my ($vmid, $skiplock, $nocheck) = @_;
6121
6122     PVE::QemuConfig->lock_config($vmid, sub {
6123         my $res = mon_cmd($vmid, 'query-status');
6124         my $resume_cmd = 'cont';
6125         my $reset = 0;
6126
6127         if ($res->{status}) {
6128             return if $res->{status} eq 'running'; # job done, go home
6129             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6130             $reset = 1 if $res->{status} eq 'shutdown';
6131         }
6132
6133         if (!$nocheck) {
6134
6135             my $conf = PVE::QemuConfig->load_config($vmid);
6136
6137             PVE::QemuConfig->check_lock($conf)
6138                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6139         }
6140
6141         if ($reset) {
6142             # required if a VM shuts down during a backup and we get a resume
6143             # request before the backup finishes for example
6144             mon_cmd($vmid, "system_reset");
6145         }
6146         mon_cmd($vmid, $resume_cmd);
6147     });
6148 }
6149
6150 sub vm_sendkey {
6151     my ($vmid, $skiplock, $key) = @_;
6152
6153     PVE::QemuConfig->lock_config($vmid, sub {
6154
6155         my $conf = PVE::QemuConfig->load_config($vmid);
6156
6157         # there is no qmp command, so we use the human monitor command
6158         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6159         die $res if $res ne '';
6160     });
6161 }
6162
6163 # vzdump restore implementaion
6164
6165 sub tar_archive_read_firstfile {
6166     my $archive = shift;
6167
6168     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6169
6170     # try to detect archive type first
6171     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6172         die "unable to open file '$archive'\n";
6173     my $firstfile = <$fh>;
6174     kill 15, $pid;
6175     close $fh;
6176
6177     die "ERROR: archive contaions no data\n" if !$firstfile;
6178     chomp $firstfile;
6179
6180     return $firstfile;
6181 }
6182
6183 sub tar_restore_cleanup {
6184     my ($storecfg, $statfile) = @_;
6185
6186     print STDERR "starting cleanup\n";
6187
6188     if (my $fd = IO::File->new($statfile, "r")) {
6189         while (defined(my $line = <$fd>)) {
6190             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6191                 my $volid = $2;
6192                 eval {
6193                     if ($volid =~ m|^/|) {
6194                         unlink $volid || die 'unlink failed\n';
6195                     } else {
6196                         PVE::Storage::vdisk_free($storecfg, $volid);
6197                     }
6198                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6199                 };
6200                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6201             } else {
6202                 print STDERR "unable to parse line in statfile - $line";
6203             }
6204         }
6205         $fd->close();
6206     }
6207 }
6208
6209 sub restore_file_archive {
6210     my ($archive, $vmid, $user, $opts) = @_;
6211
6212     return restore_vma_archive($archive, $vmid, $user, $opts)
6213         if $archive eq '-';
6214
6215     my $info = PVE::Storage::archive_info($archive);
6216     my $format = $opts->{format} // $info->{format};
6217     my $comp = $info->{compression};
6218
6219     # try to detect archive format
6220     if ($format eq 'tar') {
6221         return restore_tar_archive($archive, $vmid, $user, $opts);
6222     } else {
6223         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6224     }
6225 }
6226
6227 # hepler to remove disks that will not be used after restore
6228 my $restore_cleanup_oldconf = sub {
6229     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6230
6231     PVE::QemuConfig->foreach_volume($oldconf, sub {
6232         my ($ds, $drive) = @_;
6233
6234         return if drive_is_cdrom($drive, 1);
6235
6236         my $volid = $drive->{file};
6237         return if !$volid || $volid =~ m|^/|;
6238
6239         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6240         return if !$path || !$owner || ($owner != $vmid);
6241
6242         # Note: only delete disk we want to restore
6243         # other volumes will become unused
6244         if ($virtdev_hash->{$ds}) {
6245             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6246             if (my $err = $@) {
6247                 warn $err;
6248             }
6249         }
6250     });
6251
6252     # delete vmstate files, after the restore we have no snapshots anymore
6253     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6254         my $snap = $oldconf->{snapshots}->{$snapname};
6255         if ($snap->{vmstate}) {
6256             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6257             if (my $err = $@) {
6258                 warn $err;
6259             }
6260         }
6261     }
6262 };
6263
6264 # Helper to parse vzdump backup device hints
6265 #
6266 # $rpcenv: Environment, used to ckeck storage permissions
6267 # $user: User ID, to check storage permissions
6268 # $storecfg: Storage configuration
6269 # $fh: the file handle for reading the configuration
6270 # $devinfo: should contain device sizes for all backu-up'ed devices
6271 # $options: backup options (pool, default storage)
6272 #
6273 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6274 my $parse_backup_hints = sub {
6275     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6276
6277     my $check_storage = sub { # assert if an image can be allocate
6278         my ($storeid, $scfg) = @_;
6279         die "Content type 'images' is not available on storage '$storeid'\n"
6280             if !$scfg->{content}->{images};
6281         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6282             if $user ne 'root@pam';
6283     };
6284
6285     my $virtdev_hash = {};
6286     while (defined(my $line = <$fh>)) {
6287         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6288             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6289             die "archive does not contain data for drive '$virtdev'\n"
6290                 if !$devinfo->{$devname};
6291
6292             if (defined($options->{storage})) {
6293                 $storeid = $options->{storage} || 'local';
6294             } elsif (!$storeid) {
6295                 $storeid = 'local';
6296             }
6297             $format = 'raw' if !$format;
6298             $devinfo->{$devname}->{devname} = $devname;
6299             $devinfo->{$devname}->{virtdev} = $virtdev;
6300             $devinfo->{$devname}->{format} = $format;
6301             $devinfo->{$devname}->{storeid} = $storeid;
6302
6303             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6304             $check_storage->($storeid, $scfg); # permission and content type check
6305
6306             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6307         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6308             my $virtdev = $1;
6309             my $drive = parse_drive($virtdev, $2);
6310
6311             if (drive_is_cloudinit($drive)) {
6312                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6313                 $storeid = $options->{storage} if defined ($options->{storage});
6314                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6315                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6316
6317                 $check_storage->($storeid, $scfg); # permission and content type check
6318
6319                 $virtdev_hash->{$virtdev} = {
6320                     format => $format,
6321                     storeid => $storeid,
6322                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6323                     is_cloudinit => 1,
6324                 };
6325             }
6326         }
6327     }
6328
6329     return $virtdev_hash;
6330 };
6331
6332 # Helper to allocate and activate all volumes required for a restore
6333 #
6334 # $storecfg: Storage configuration
6335 # $virtdev_hash: as returned by parse_backup_hints()
6336 #
6337 # Returns: { $virtdev => $volid }
6338 my $restore_allocate_devices = sub {
6339     my ($storecfg, $virtdev_hash, $vmid) = @_;
6340
6341     my $map = {};
6342     foreach my $virtdev (sort keys %$virtdev_hash) {
6343         my $d = $virtdev_hash->{$virtdev};
6344         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6345         my $storeid = $d->{storeid};
6346         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6347
6348         # test if requested format is supported
6349         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6350         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6351         $d->{format} = $defFormat if !$supported;
6352
6353         my $name;
6354         if ($d->{is_cloudinit}) {
6355             $name = "vm-$vmid-cloudinit";
6356             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6357             if ($scfg->{path}) {
6358                 $name .= ".$d->{format}";
6359             }
6360         }
6361
6362         my $volid = PVE::Storage::vdisk_alloc(
6363             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6364
6365         print STDERR "new volume ID is '$volid'\n";
6366         $d->{volid} = $volid;
6367
6368         PVE::Storage::activate_volumes($storecfg, [$volid]);
6369
6370         $map->{$virtdev} = $volid;
6371     }
6372
6373     return $map;
6374 };
6375
6376 sub restore_update_config_line {
6377     my ($cookie, $map, $line, $unique) = @_;
6378
6379     return '' if $line =~ m/^\#qmdump\#/;
6380     return '' if $line =~ m/^\#vzdump\#/;
6381     return '' if $line =~ m/^lock:/;
6382     return '' if $line =~ m/^unused\d+:/;
6383     return '' if $line =~ m/^parent:/;
6384
6385     my $res = '';
6386
6387     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6388     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6389         # try to convert old 1.X settings
6390         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6391         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6392             my ($model, $macaddr) = split(/\=/, $devconfig);
6393             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6394             my $net = {
6395                 model => $model,
6396                 bridge => "vmbr$ind",
6397                 macaddr => $macaddr,
6398             };
6399             my $netstr = print_net($net);
6400
6401             $res .= "net$cookie->{netcount}: $netstr\n";
6402             $cookie->{netcount}++;
6403         }
6404     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6405         my ($id, $netstr) = ($1, $2);
6406         my $net = parse_net($netstr);
6407         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6408         $netstr = print_net($net);
6409         $res .= "$id: $netstr\n";
6410     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6411         my $virtdev = $1;
6412         my $value = $3;
6413         my $di = parse_drive($virtdev, $value);
6414         if (defined($di->{backup}) && !$di->{backup}) {
6415             $res .= "#$line";
6416         } elsif ($map->{$virtdev}) {
6417             delete $di->{format}; # format can change on restore
6418             $di->{file} = $map->{$virtdev};
6419             $value = print_drive($di);
6420             $res .= "$virtdev: $value\n";
6421         } else {
6422             $res .= $line;
6423         }
6424     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6425         my $vmgenid = $1;
6426         if ($vmgenid ne '0') {
6427             # always generate a new vmgenid if there was a valid one setup
6428             $vmgenid = generate_uuid();
6429         }
6430         $res .= "vmgenid: $vmgenid\n";
6431     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6432         my ($uuid, $uuid_str);
6433         UUID::generate($uuid);
6434         UUID::unparse($uuid, $uuid_str);
6435         my $smbios1 = parse_smbios1($2);
6436         $smbios1->{uuid} = $uuid_str;
6437         $res .= $1.print_smbios1($smbios1)."\n";
6438     } else {
6439         $res .= $line;
6440     }
6441
6442     return $res;
6443 }
6444
6445 my $restore_deactivate_volumes = sub {
6446     my ($storecfg, $devinfo) = @_;
6447
6448     my $vollist = [];
6449     foreach my $devname (keys %$devinfo) {
6450         my $volid = $devinfo->{$devname}->{volid};
6451         push @$vollist, $volid if $volid;
6452     }
6453
6454     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6455 };
6456
6457 my $restore_destroy_volumes = sub {
6458     my ($storecfg, $devinfo) = @_;
6459
6460     foreach my $devname (keys %$devinfo) {
6461         my $volid = $devinfo->{$devname}->{volid};
6462         next if !$volid;
6463         eval {
6464             if ($volid =~ m|^/|) {
6465                 unlink $volid || die 'unlink failed\n';
6466             } else {
6467                 PVE::Storage::vdisk_free($storecfg, $volid);
6468             }
6469             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6470         };
6471         print STDERR "unable to cleanup '$volid' - $@" if $@;
6472     }
6473 };
6474
6475 sub scan_volids {
6476     my ($cfg, $vmid) = @_;
6477
6478     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6479
6480     my $volid_hash = {};
6481     foreach my $storeid (keys %$info) {
6482         foreach my $item (@{$info->{$storeid}}) {
6483             next if !($item->{volid} && $item->{size});
6484             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6485             $volid_hash->{$item->{volid}} = $item;
6486         }
6487     }
6488
6489     return $volid_hash;
6490 }
6491
6492 sub update_disk_config {
6493     my ($vmid, $conf, $volid_hash) = @_;
6494
6495     my $changes;
6496     my $prefix = "VM $vmid";
6497
6498     # used and unused disks
6499     my $referenced = {};
6500
6501     # Note: it is allowed to define multiple storages with same path (alias), so
6502     # we need to check both 'volid' and real 'path' (two different volid can point
6503     # to the same path).
6504
6505     my $referencedpath = {};
6506
6507     # update size info
6508     PVE::QemuConfig->foreach_volume($conf, sub {
6509         my ($opt, $drive) = @_;
6510
6511         my $volid = $drive->{file};
6512         return if !$volid;
6513         my $volume = $volid_hash->{$volid};
6514
6515         # mark volid as "in-use" for next step
6516         $referenced->{$volid} = 1;
6517         if ($volume && (my $path = $volume->{path})) {
6518             $referencedpath->{$path} = 1;
6519         }
6520
6521         return if drive_is_cdrom($drive);
6522         return if !$volume;
6523
6524         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6525         if (defined($updated)) {
6526             $changes = 1;
6527             $conf->{$opt} = print_drive($updated);
6528             print "$prefix ($opt): $msg\n";
6529         }
6530     });
6531
6532     # remove 'unusedX' entry if volume is used
6533     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6534         my ($opt, $drive) = @_;
6535
6536         my $volid = $drive->{file};
6537         return if !$volid;
6538
6539         my $path;
6540         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6541         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6542             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6543             $changes = 1;
6544             delete $conf->{$opt};
6545         }
6546
6547         $referenced->{$volid} = 1;
6548         $referencedpath->{$path} = 1 if $path;
6549     });
6550
6551     foreach my $volid (sort keys %$volid_hash) {
6552         next if $volid =~ m/vm-$vmid-state-/;
6553         next if $referenced->{$volid};
6554         my $path = $volid_hash->{$volid}->{path};
6555         next if !$path; # just to be sure
6556         next if $referencedpath->{$path};
6557         $changes = 1;
6558         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6559         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6560         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6561     }
6562
6563     return $changes;
6564 }
6565
6566 sub rescan {
6567     my ($vmid, $nolock, $dryrun) = @_;
6568
6569     my $cfg = PVE::Storage::config();
6570
6571     print "rescan volumes...\n";
6572     my $volid_hash = scan_volids($cfg, $vmid);
6573
6574     my $updatefn =  sub {
6575         my ($vmid) = @_;
6576
6577         my $conf = PVE::QemuConfig->load_config($vmid);
6578
6579         PVE::QemuConfig->check_lock($conf);
6580
6581         my $vm_volids = {};
6582         foreach my $volid (keys %$volid_hash) {
6583             my $info = $volid_hash->{$volid};
6584             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6585         }
6586
6587         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6588
6589         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6590     };
6591
6592     if (defined($vmid)) {
6593         if ($nolock) {
6594             &$updatefn($vmid);
6595         } else {
6596             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6597         }
6598     } else {
6599         my $vmlist = config_list();
6600         foreach my $vmid (keys %$vmlist) {
6601             if ($nolock) {
6602                 &$updatefn($vmid);
6603             } else {
6604                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6605             }
6606         }
6607     }
6608 }
6609
6610 sub restore_proxmox_backup_archive {
6611     my ($archive, $vmid, $user, $options) = @_;
6612
6613     my $storecfg = PVE::Storage::config();
6614
6615     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6616     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6617
6618     my $fingerprint = $scfg->{fingerprint};
6619     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6620
6621     my $repo = PVE::PBSClient::get_repository($scfg);
6622
6623     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6624     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6625     local $ENV{PBS_PASSWORD} = $password;
6626     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6627
6628     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6629         PVE::Storage::parse_volname($storecfg, $archive);
6630
6631     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6632
6633     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6634
6635     my $tmpdir = "/var/tmp/vzdumptmp$$";
6636     rmtree $tmpdir;
6637     mkpath $tmpdir;
6638
6639     my $conffile = PVE::QemuConfig->config_file($vmid);
6640      # disable interrupts (always do cleanups)
6641     local $SIG{INT} =
6642         local $SIG{TERM} =
6643         local $SIG{QUIT} =
6644         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6645
6646     # Note: $oldconf is undef if VM does not exists
6647     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6648     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6649     my $new_conf_raw = '';
6650
6651     my $rpcenv = PVE::RPCEnvironment::get();
6652     my $devinfo = {};
6653
6654     eval {
6655         # enable interrupts
6656         local $SIG{INT} =
6657             local $SIG{TERM} =
6658             local $SIG{QUIT} =
6659             local $SIG{HUP} =
6660             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6661
6662         my $cfgfn = "$tmpdir/qemu-server.conf";
6663         my $firewall_config_fn = "$tmpdir/fw.conf";
6664         my $index_fn = "$tmpdir/index.json";
6665
6666         my $cmd = "restore";
6667
6668         my $param = [$pbs_backup_name, "index.json", $index_fn];
6669         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6670         my $index = PVE::Tools::file_get_contents($index_fn);
6671         $index = decode_json($index);
6672
6673         # print Dumper($index);
6674         foreach my $info (@{$index->{files}}) {
6675             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6676                 my $devname = $1;
6677                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6678                     $devinfo->{$devname}->{size} = $1;
6679                 } else {
6680                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6681                 }
6682             }
6683         }
6684
6685         my $is_qemu_server_backup = scalar(
6686             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6687         );
6688         if (!$is_qemu_server_backup) {
6689             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6690         }
6691         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6692
6693         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6694         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6695
6696         if ($has_firewall_config) {
6697             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6698             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6699
6700             my $pve_firewall_dir = '/etc/pve/firewall';
6701             mkdir $pve_firewall_dir; # make sure the dir exists
6702             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6703         }
6704
6705         my $fh = IO::File->new($cfgfn, "r") ||
6706             die "unable to read qemu-server.conf - $!\n";
6707
6708         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6709
6710         # fixme: rate limit?
6711
6712         # create empty/temp config
6713         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6714
6715         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6716
6717         # allocate volumes
6718         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6719
6720         foreach my $virtdev (sort keys %$virtdev_hash) {
6721             my $d = $virtdev_hash->{$virtdev};
6722             next if $d->{is_cloudinit}; # no need to restore cloudinit
6723
6724             # this fails if storage is unavailable
6725             my $volid = $d->{volid};
6726             my $path = PVE::Storage::path($storecfg, $volid);
6727
6728             # for live-restore we only want to preload the efidisk and TPM state
6729             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6730
6731             my $pbs_restore_cmd = [
6732                 '/usr/bin/pbs-restore',
6733                 '--repository', $repo,
6734                 $pbs_backup_name,
6735                 "$d->{devname}.img.fidx",
6736                 $path,
6737                 '--verbose',
6738                 ];
6739
6740             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6741             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6742
6743             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6744                 push @$pbs_restore_cmd, '--skip-zero';
6745             }
6746
6747             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6748             print "restore proxmox backup image: $dbg_cmdstring\n";
6749             run_command($pbs_restore_cmd);
6750         }
6751
6752         $fh->seek(0, 0) || die "seek failed - $!\n";
6753
6754         my $cookie = { netcount => 0 };
6755         while (defined(my $line = <$fh>)) {
6756             $new_conf_raw .= restore_update_config_line(
6757                 $cookie,
6758                 $map,
6759                 $line,
6760                 $options->{unique},
6761             );
6762         }
6763
6764         $fh->close();
6765     };
6766     my $err = $@;
6767
6768     if ($err || !$options->{live}) {
6769         $restore_deactivate_volumes->($storecfg, $devinfo);
6770     }
6771
6772     rmtree $tmpdir;
6773
6774     if ($err) {
6775         $restore_destroy_volumes->($storecfg, $devinfo);
6776         die $err;
6777     }
6778
6779     if ($options->{live}) {
6780         # keep lock during live-restore
6781         $new_conf_raw .= "\nlock: create";
6782     }
6783
6784     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6785
6786     PVE::Cluster::cfs_update(); # make sure we read new file
6787
6788     eval { rescan($vmid, 1); };
6789     warn $@ if $@;
6790
6791     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6792
6793     if ($options->{live}) {
6794         # enable interrupts
6795         local $SIG{INT} =
6796             local $SIG{TERM} =
6797             local $SIG{QUIT} =
6798             local $SIG{HUP} =
6799             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6800
6801         my $conf = PVE::QemuConfig->load_config($vmid);
6802         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6803
6804         # these special drives are already restored before start
6805         delete $devinfo->{'drive-efidisk0'};
6806         delete $devinfo->{'drive-tpmstate0-backup'};
6807         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6808
6809         PVE::QemuConfig->remove_lock($vmid, "create");
6810     }
6811 }
6812
6813 sub pbs_live_restore {
6814     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6815
6816     print "starting VM for live-restore\n";
6817     print "repository: '$repo', snapshot: '$snap'\n";
6818
6819     my $pbs_backing = {};
6820     for my $ds (keys %$restored_disks) {
6821         $ds =~ m/^drive-(.*)$/;
6822         my $confname = $1;
6823         $pbs_backing->{$confname} = {
6824             repository => $repo,
6825             snapshot => $snap,
6826             archive => "$ds.img.fidx",
6827         };
6828         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6829
6830         my $drive = parse_drive($confname, $conf->{$confname});
6831         print "restoring '$ds' to '$drive->{file}'\n";
6832     }
6833
6834     my $drives_streamed = 0;
6835     eval {
6836         # make sure HA doesn't interrupt our restore by stopping the VM
6837         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6838             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6839         }
6840
6841         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6842         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6843         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6844
6845         my $qmeventd_fd = register_qmeventd_handle($vmid);
6846
6847         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6848         # this will effectively collapse the backing image chain consisting of
6849         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6850         # removes itself once all backing images vanish with 'auto-remove=on')
6851         my $jobs = {};
6852         for my $ds (sort keys %$restored_disks) {
6853             my $job_id = "restore-$ds";
6854             mon_cmd($vmid, 'block-stream',
6855                 'job-id' => $job_id,
6856                 device => "$ds",
6857             );
6858             $jobs->{$job_id} = {};
6859         }
6860
6861         mon_cmd($vmid, 'cont');
6862         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6863
6864         print "restore-drive jobs finished successfully, removing all tracking block devices"
6865             ." to disconnect from Proxmox Backup Server\n";
6866
6867         for my $ds (sort keys %$restored_disks) {
6868             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6869         }
6870
6871         close($qmeventd_fd);
6872     };
6873
6874     my $err = $@;
6875
6876     if ($err) {
6877         warn "An error occurred during live-restore: $err\n";
6878         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6879         die "live-restore failed\n";
6880     }
6881 }
6882
6883 sub restore_vma_archive {
6884     my ($archive, $vmid, $user, $opts, $comp) = @_;
6885
6886     my $readfrom = $archive;
6887
6888     my $cfg = PVE::Storage::config();
6889     my $commands = [];
6890     my $bwlimit = $opts->{bwlimit};
6891
6892     my $dbg_cmdstring = '';
6893     my $add_pipe = sub {
6894         my ($cmd) = @_;
6895         push @$commands, $cmd;
6896         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6897         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6898         $readfrom = '-';
6899     };
6900
6901     my $input = undef;
6902     if ($archive eq '-') {
6903         $input = '<&STDIN';
6904     } else {
6905         # If we use a backup from a PVE defined storage we also consider that
6906         # storage's rate limit:
6907         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6908         if (defined($volid)) {
6909             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6910             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6911             if ($readlimit) {
6912                 print STDERR "applying read rate limit: $readlimit\n";
6913                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6914                 $add_pipe->($cstream);
6915             }
6916         }
6917     }
6918
6919     if ($comp) {
6920         my $info = PVE::Storage::decompressor_info('vma', $comp);
6921         my $cmd = $info->{decompressor};
6922         push @$cmd, $readfrom;
6923         $add_pipe->($cmd);
6924     }
6925
6926     my $tmpdir = "/var/tmp/vzdumptmp$$";
6927     rmtree $tmpdir;
6928
6929     # disable interrupts (always do cleanups)
6930     local $SIG{INT} =
6931         local $SIG{TERM} =
6932         local $SIG{QUIT} =
6933         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6934
6935     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6936     POSIX::mkfifo($mapfifo, 0600);
6937     my $fifofh;
6938     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6939
6940     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6941
6942     my $oldtimeout;
6943     my $timeout = 5;
6944
6945     my $devinfo = {};
6946
6947     my $rpcenv = PVE::RPCEnvironment::get();
6948
6949     my $conffile = PVE::QemuConfig->config_file($vmid);
6950
6951     # Note: $oldconf is undef if VM does not exist
6952     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6953     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6954     my $new_conf_raw = '';
6955
6956     my %storage_limits;
6957
6958     my $print_devmap = sub {
6959         my $cfgfn = "$tmpdir/qemu-server.conf";
6960
6961         # we can read the config - that is already extracted
6962         my $fh = IO::File->new($cfgfn, "r") ||
6963             die "unable to read qemu-server.conf - $!\n";
6964
6965         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6966         if (-f $fwcfgfn) {
6967             my $pve_firewall_dir = '/etc/pve/firewall';
6968             mkdir $pve_firewall_dir; # make sure the dir exists
6969             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6970         }
6971
6972         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6973
6974         foreach my $info (values %{$virtdev_hash}) {
6975             my $storeid = $info->{storeid};
6976             next if defined($storage_limits{$storeid});
6977
6978             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6979             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6980             $storage_limits{$storeid} = $limit * 1024;
6981         }
6982
6983         foreach my $devname (keys %$devinfo) {
6984             die "found no device mapping information for device '$devname'\n"
6985                 if !$devinfo->{$devname}->{virtdev};
6986         }
6987
6988         # create empty/temp config
6989         if ($oldconf) {
6990             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6991             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6992         }
6993
6994         # allocate volumes
6995         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6996
6997         # print restore information to $fifofh
6998         foreach my $virtdev (sort keys %$virtdev_hash) {
6999             my $d = $virtdev_hash->{$virtdev};
7000             next if $d->{is_cloudinit}; # no need to restore cloudinit
7001
7002             my $storeid = $d->{storeid};
7003             my $volid = $d->{volid};
7004
7005             my $map_opts = '';
7006             if (my $limit = $storage_limits{$storeid}) {
7007                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7008             }
7009
7010             my $write_zeros = 1;
7011             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7012                 $write_zeros = 0;
7013             }
7014
7015             my $path = PVE::Storage::path($cfg, $volid);
7016
7017             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7018
7019             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7020         }
7021
7022         $fh->seek(0, 0) || die "seek failed - $!\n";
7023
7024         my $cookie = { netcount => 0 };
7025         while (defined(my $line = <$fh>)) {
7026             $new_conf_raw .= restore_update_config_line(
7027                 $cookie,
7028                 $map,
7029                 $line,
7030                 $opts->{unique},
7031             );
7032         }
7033
7034         $fh->close();
7035     };
7036
7037     eval {
7038         # enable interrupts
7039         local $SIG{INT} =
7040             local $SIG{TERM} =
7041             local $SIG{QUIT} =
7042             local $SIG{HUP} =
7043             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7044         local $SIG{ALRM} = sub { die "got timeout\n"; };
7045
7046         $oldtimeout = alarm($timeout);
7047
7048         my $parser = sub {
7049             my $line = shift;
7050
7051             print "$line\n";
7052
7053             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7054                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7055                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7056             } elsif ($line =~ m/^CTIME: /) {
7057                 # we correctly received the vma config, so we can disable
7058                 # the timeout now for disk allocation (set to 10 minutes, so
7059                 # that we always timeout if something goes wrong)
7060                 alarm(600);
7061                 &$print_devmap();
7062                 print $fifofh "done\n";
7063                 my $tmp = $oldtimeout || 0;
7064                 $oldtimeout = undef;
7065                 alarm($tmp);
7066                 close($fifofh);
7067                 $fifofh = undef;
7068             }
7069         };
7070
7071         print "restore vma archive: $dbg_cmdstring\n";
7072         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7073     };
7074     my $err = $@;
7075
7076     alarm($oldtimeout) if $oldtimeout;
7077
7078     $restore_deactivate_volumes->($cfg, $devinfo);
7079
7080     close($fifofh) if $fifofh;
7081     unlink $mapfifo;
7082     rmtree $tmpdir;
7083
7084     if ($err) {
7085         $restore_destroy_volumes->($cfg, $devinfo);
7086         die $err;
7087     }
7088
7089     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7090
7091     PVE::Cluster::cfs_update(); # make sure we read new file
7092
7093     eval { rescan($vmid, 1); };
7094     warn $@ if $@;
7095
7096     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7097 }
7098
7099 sub restore_tar_archive {
7100     my ($archive, $vmid, $user, $opts) = @_;
7101
7102     if ($archive ne '-') {
7103         my $firstfile = tar_archive_read_firstfile($archive);
7104         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7105             if $firstfile ne 'qemu-server.conf';
7106     }
7107
7108     my $storecfg = PVE::Storage::config();
7109
7110     # avoid zombie disks when restoring over an existing VM -> cleanup first
7111     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7112     # skiplock=1 because qmrestore has set the 'create' lock itself already
7113     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7114     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7115
7116     my $tocmd = "/usr/lib/qemu-server/qmextract";
7117
7118     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7119     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7120     $tocmd .= ' --prealloc' if $opts->{prealloc};
7121     $tocmd .= ' --info' if $opts->{info};
7122
7123     # tar option "xf" does not autodetect compression when read from STDIN,
7124     # so we pipe to zcat
7125     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7126         PVE::Tools::shellquote("--to-command=$tocmd");
7127
7128     my $tmpdir = "/var/tmp/vzdumptmp$$";
7129     mkpath $tmpdir;
7130
7131     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7132     local $ENV{VZDUMP_VMID} = $vmid;
7133     local $ENV{VZDUMP_USER} = $user;
7134
7135     my $conffile = PVE::QemuConfig->config_file($vmid);
7136     my $new_conf_raw = '';
7137
7138     # disable interrupts (always do cleanups)
7139     local $SIG{INT} =
7140         local $SIG{TERM} =
7141         local $SIG{QUIT} =
7142         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7143
7144     eval {
7145         # enable interrupts
7146         local $SIG{INT} =
7147             local $SIG{TERM} =
7148             local $SIG{QUIT} =
7149             local $SIG{HUP} =
7150             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7151
7152         if ($archive eq '-') {
7153             print "extracting archive from STDIN\n";
7154             run_command($cmd, input => "<&STDIN");
7155         } else {
7156             print "extracting archive '$archive'\n";
7157             run_command($cmd);
7158         }
7159
7160         return if $opts->{info};
7161
7162         # read new mapping
7163         my $map = {};
7164         my $statfile = "$tmpdir/qmrestore.stat";
7165         if (my $fd = IO::File->new($statfile, "r")) {
7166             while (defined (my $line = <$fd>)) {
7167                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7168                     $map->{$1} = $2 if $1;
7169                 } else {
7170                     print STDERR "unable to parse line in statfile - $line\n";
7171                 }
7172             }
7173             $fd->close();
7174         }
7175
7176         my $confsrc = "$tmpdir/qemu-server.conf";
7177
7178         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7179
7180         my $cookie = { netcount => 0 };
7181         while (defined (my $line = <$srcfd>)) {
7182             $new_conf_raw .= restore_update_config_line(
7183                 $cookie,
7184                 $map,
7185                 $line,
7186                 $opts->{unique},
7187             );
7188         }
7189
7190         $srcfd->close();
7191     };
7192     if (my $err = $@) {
7193         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7194         die $err;
7195     }
7196
7197     rmtree $tmpdir;
7198
7199     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7200
7201     PVE::Cluster::cfs_update(); # make sure we read new file
7202
7203     eval { rescan($vmid, 1); };
7204     warn $@ if $@;
7205 };
7206
7207 sub foreach_storage_used_by_vm {
7208     my ($conf, $func) = @_;
7209
7210     my $sidhash = {};
7211
7212     PVE::QemuConfig->foreach_volume($conf, sub {
7213         my ($ds, $drive) = @_;
7214         return if drive_is_cdrom($drive);
7215
7216         my $volid = $drive->{file};
7217
7218         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7219         $sidhash->{$sid} = $sid if $sid;
7220     });
7221
7222     foreach my $sid (sort keys %$sidhash) {
7223         &$func($sid);
7224     }
7225 }
7226
7227 my $qemu_snap_storage = {
7228     rbd => 1,
7229 };
7230 sub do_snapshots_with_qemu {
7231     my ($storecfg, $volid, $deviceid) = @_;
7232
7233     return if $deviceid =~ m/tpmstate0/;
7234
7235     my $storage_name = PVE::Storage::parse_volume_id($volid);
7236     my $scfg = $storecfg->{ids}->{$storage_name};
7237     die "could not find storage '$storage_name'\n" if !defined($scfg);
7238
7239     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7240         return 1;
7241     }
7242
7243     if ($volid =~ m/\.(qcow2|qed)$/){
7244         return 1;
7245     }
7246
7247     return;
7248 }
7249
7250 sub qga_check_running {
7251     my ($vmid, $nowarn) = @_;
7252
7253     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7254     if ($@) {
7255         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7256         return 0;
7257     }
7258     return 1;
7259 }
7260
7261 sub template_create {
7262     my ($vmid, $conf, $disk) = @_;
7263
7264     my $storecfg = PVE::Storage::config();
7265
7266     PVE::QemuConfig->foreach_volume($conf, sub {
7267         my ($ds, $drive) = @_;
7268
7269         return if drive_is_cdrom($drive);
7270         return if $disk && $ds ne $disk;
7271
7272         my $volid = $drive->{file};
7273         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7274
7275         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7276         $drive->{file} = $voliddst;
7277         $conf->{$ds} = print_drive($drive);
7278         PVE::QemuConfig->write_config($vmid, $conf);
7279     });
7280 }
7281
7282 sub convert_iscsi_path {
7283     my ($path) = @_;
7284
7285     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7286         my $portal = $1;
7287         my $target = $2;
7288         my $lun = $3;
7289
7290         my $initiator_name = get_initiator_name();
7291
7292         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7293                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7294     }
7295
7296     die "cannot convert iscsi path '$path', unkown format\n";
7297 }
7298
7299 sub qemu_img_convert {
7300     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7301
7302     my $storecfg = PVE::Storage::config();
7303     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7304     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7305
7306     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7307
7308     my $cachemode;
7309     my $src_path;
7310     my $src_is_iscsi = 0;
7311     my $src_format;
7312
7313     if ($src_storeid) {
7314         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7315         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7316         $src_format = qemu_img_format($src_scfg, $src_volname);
7317         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7318         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7319         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7320     } elsif (-f $src_volid || -b $src_volid) {
7321         $src_path = $src_volid;
7322         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7323             $src_format = $1;
7324         }
7325     }
7326
7327     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7328
7329     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7330     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7331     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7332     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7333
7334     my $cmd = [];
7335     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7336     push @$cmd, '-l', "snapshot.name=$snapname"
7337         if $snapname && $src_format && $src_format eq "qcow2";
7338     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7339     push @$cmd, '-T', $cachemode if defined($cachemode);
7340
7341     if ($src_is_iscsi) {
7342         push @$cmd, '--image-opts';
7343         $src_path = convert_iscsi_path($src_path);
7344     } elsif ($src_format) {
7345         push @$cmd, '-f', $src_format;
7346     }
7347
7348     if ($dst_is_iscsi) {
7349         push @$cmd, '--target-image-opts';
7350         $dst_path = convert_iscsi_path($dst_path);
7351     } else {
7352         push @$cmd, '-O', $dst_format;
7353     }
7354
7355     push @$cmd, $src_path;
7356
7357     if (!$dst_is_iscsi && $is_zero_initialized) {
7358         push @$cmd, "zeroinit:$dst_path";
7359     } else {
7360         push @$cmd, $dst_path;
7361     }
7362
7363     my $parser = sub {
7364         my $line = shift;
7365         if($line =~ m/\((\S+)\/100\%\)/){
7366             my $percent = $1;
7367             my $transferred = int($size * $percent / 100);
7368             my $total_h = render_bytes($size, 1);
7369             my $transferred_h = render_bytes($transferred, 1);
7370
7371             print "transferred $transferred_h of $total_h ($percent%)\n";
7372         }
7373
7374     };
7375
7376     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7377     my $err = $@;
7378     die "copy failed: $err" if $err;
7379 }
7380
7381 sub qemu_img_format {
7382     my ($scfg, $volname) = @_;
7383
7384     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7385         return $1;
7386     } else {
7387         return "raw";
7388     }
7389 }
7390
7391 sub qemu_drive_mirror {
7392     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7393
7394     $jobs = {} if !$jobs;
7395
7396     my $qemu_target;
7397     my $format;
7398     $jobs->{"drive-$drive"} = {};
7399
7400     if ($dst_volid =~ /^nbd:/) {
7401         $qemu_target = $dst_volid;
7402         $format = "nbd";
7403     } else {
7404         my $storecfg = PVE::Storage::config();
7405         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7406
7407         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7408
7409         $format = qemu_img_format($dst_scfg, $dst_volname);
7410
7411         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7412
7413         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7414     }
7415
7416     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7417     $opts->{format} = $format if $format;
7418
7419     if (defined($src_bitmap)) {
7420         $opts->{sync} = 'incremental';
7421         $opts->{bitmap} = $src_bitmap;
7422         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7423     }
7424
7425     if (defined($bwlimit)) {
7426         $opts->{speed} = $bwlimit * 1024;
7427         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7428     } else {
7429         print "drive mirror is starting for drive-$drive\n";
7430     }
7431
7432     # if a job already runs for this device we get an error, catch it for cleanup
7433     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7434     if (my $err = $@) {
7435         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7436         warn "$@\n" if $@;
7437         die "mirroring error: $err\n";
7438     }
7439
7440     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7441 }
7442
7443 # $completion can be either
7444 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7445 # 'cancel': wait until all jobs are ready, block-job-cancel them
7446 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7447 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7448 sub qemu_drive_mirror_monitor {
7449     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7450
7451     $completion //= 'complete';
7452     $op //= "mirror";
7453
7454     eval {
7455         my $err_complete = 0;
7456
7457         my $starttime = time ();
7458         while (1) {
7459             die "block job ('$op') timed out\n" if $err_complete > 300;
7460
7461             my $stats = mon_cmd($vmid, "query-block-jobs");
7462             my $ctime = time();
7463
7464             my $running_jobs = {};
7465             for my $stat (@$stats) {
7466                 next if $stat->{type} ne $op;
7467                 $running_jobs->{$stat->{device}} = $stat;
7468             }
7469
7470             my $readycounter = 0;
7471
7472             for my $job_id (sort keys %$jobs) {
7473                 my $job = $running_jobs->{$job_id};
7474
7475                 my $vanished = !defined($job);
7476                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7477                 if($complete || ($vanished && $completion eq 'auto')) {
7478                     print "$job_id: $op-job finished\n";
7479                     delete $jobs->{$job_id};
7480                     next;
7481                 }
7482
7483                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7484
7485                 my $busy = $job->{busy};
7486                 my $ready = $job->{ready};
7487                 if (my $total = $job->{len}) {
7488                     my $transferred = $job->{offset} || 0;
7489                     my $remaining = $total - $transferred;
7490                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7491
7492                     my $duration = $ctime - $starttime;
7493                     my $total_h = render_bytes($total, 1);
7494                     my $transferred_h = render_bytes($transferred, 1);
7495
7496                     my $status = sprintf(
7497                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7498                         render_duration($duration),
7499                     );
7500
7501                     if ($ready) {
7502                         if ($busy) {
7503                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7504                         } else {
7505                             $status .= ", ready";
7506                         }
7507                     }
7508                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7509                     $jobs->{$job_id}->{ready} = $ready;
7510                 }
7511
7512                 $readycounter++ if $job->{ready};
7513             }
7514
7515             last if scalar(keys %$jobs) == 0;
7516
7517             if ($readycounter == scalar(keys %$jobs)) {
7518                 print "all '$op' jobs are ready\n";
7519
7520                 # do the complete later (or has already been done)
7521                 last if $completion eq 'skip' || $completion eq 'auto';
7522
7523                 if ($vmiddst && $vmiddst != $vmid) {
7524                     my $agent_running = $qga && qga_check_running($vmid);
7525                     if ($agent_running) {
7526                         print "freeze filesystem\n";
7527                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7528                         warn $@ if $@;
7529                     } else {
7530                         print "suspend vm\n";
7531                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7532                         warn $@ if $@;
7533                     }
7534
7535                     # if we clone a disk for a new target vm, we don't switch the disk
7536                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7537
7538                     if ($agent_running) {
7539                         print "unfreeze filesystem\n";
7540                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7541                         warn $@ if $@;
7542                     } else {
7543                         print "resume vm\n";
7544                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7545                         warn $@ if $@;
7546                     }
7547
7548                     last;
7549                 } else {
7550
7551                     for my $job_id (sort keys %$jobs) {
7552                         # try to switch the disk if source and destination are on the same guest
7553                         print "$job_id: Completing block job_id...\n";
7554
7555                         my $op;
7556                         if ($completion eq 'complete') {
7557                             $op = 'block-job-complete';
7558                         } elsif ($completion eq 'cancel') {
7559                             $op = 'block-job-cancel';
7560                         } else {
7561                             die "invalid completion value: $completion\n";
7562                         }
7563                         eval { mon_cmd($vmid, $op, device => $job_id) };
7564                         if ($@ =~ m/cannot be completed/) {
7565                             print "$job_id: block job cannot be completed, trying again.\n";
7566                             $err_complete++;
7567                         }else {
7568                             print "$job_id: Completed successfully.\n";
7569                             $jobs->{$job_id}->{complete} = 1;
7570                         }
7571                     }
7572                 }
7573             }
7574             sleep 1;
7575         }
7576     };
7577     my $err = $@;
7578
7579     if ($err) {
7580         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7581         die "block job ($op) error: $err";
7582     }
7583 }
7584
7585 sub qemu_blockjobs_cancel {
7586     my ($vmid, $jobs) = @_;
7587
7588     foreach my $job (keys %$jobs) {
7589         print "$job: Cancelling block job\n";
7590         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7591         $jobs->{$job}->{cancel} = 1;
7592     }
7593
7594     while (1) {
7595         my $stats = mon_cmd($vmid, "query-block-jobs");
7596
7597         my $running_jobs = {};
7598         foreach my $stat (@$stats) {
7599             $running_jobs->{$stat->{device}} = $stat;
7600         }
7601
7602         foreach my $job (keys %$jobs) {
7603
7604             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7605                 print "$job: Done.\n";
7606                 delete $jobs->{$job};
7607             }
7608         }
7609
7610         last if scalar(keys %$jobs) == 0;
7611
7612         sleep 1;
7613     }
7614 }
7615
7616 sub clone_disk {
7617     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7618
7619     my ($vmid, $running) = $source->@{qw(vmid running)};
7620     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7621
7622     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7623     my ($storage, $format) = $dest->@{qw(storage format)};
7624
7625     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7626
7627     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7628         die "cloning from/to EFI disk requires EFI disk\n"
7629             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7630         die "cloning from/to TPM state requires TPM state\n"
7631             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7632
7633         # This would lead to two device nodes in QEMU pointing to the same backing image!
7634         die "cannot change drive name when cloning disk from/to the same VM\n"
7635             if $use_drive_mirror && $vmid == $newvmid;
7636     }
7637
7638     die "cannot move TPM state while VM is running\n"
7639         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7640
7641     my $newvolid;
7642
7643     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7644     print "$src_drivename " if $src_drivename;
7645     print "($drive->{file})\n";
7646
7647     if (!$full) {
7648         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7649         push @$newvollist, $newvolid;
7650     } else {
7651
7652         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7653         $storeid = $storage if $storage;
7654
7655         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7656
7657         my $name = undef;
7658         my $size = undef;
7659         if (drive_is_cloudinit($drive)) {
7660             $name = "vm-$newvmid-cloudinit";
7661             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7662             if ($scfg->{path}) {
7663                 $name .= ".$dst_format";
7664             }
7665             $snapname = undef;
7666             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7667         } elsif ($dst_drivename eq 'efidisk0') {
7668             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7669         } elsif ($dst_drivename eq 'tpmstate0') {
7670             $dst_format = 'raw';
7671             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7672         } else {
7673             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7674         }
7675         $newvolid = PVE::Storage::vdisk_alloc(
7676             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7677         );
7678         push @$newvollist, $newvolid;
7679
7680         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7681
7682         if (drive_is_cloudinit($drive)) {
7683             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7684             # if this is the case, we have to complete any block-jobs still there from
7685             # previous drive-mirrors
7686             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7687                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7688             }
7689             goto no_data_clone;
7690         }
7691
7692         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7693         if ($use_drive_mirror) {
7694             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7695                 $completion, $qga, $bwlimit);
7696         } else {
7697             # TODO: handle bwlimits
7698             if ($dst_drivename eq 'efidisk0') {
7699                 # the relevant data on the efidisk may be smaller than the source
7700                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7701                 # that is given by the OVMF_VARS.fd
7702                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7703                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7704
7705                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7706
7707                 # better for Ceph if block size is not too small, see bug #3324
7708                 my $bs = 1024*1024;
7709
7710                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7711                 push $cmd->@*, '-l', $snapname if $src_format eq 'qcow2' && $snapname;
7712                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7713                 run_command($cmd);
7714             } else {
7715                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7716             }
7717         }
7718     }
7719
7720 no_data_clone:
7721     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7722
7723     my $disk = dclone($drive);
7724     delete $disk->{format};
7725     $disk->{file} = $newvolid;
7726     $disk->{size} = $size if defined($size);
7727
7728     return $disk;
7729 }
7730
7731 sub get_running_qemu_version {
7732     my ($vmid) = @_;
7733     my $res = mon_cmd($vmid, "query-version");
7734     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7735 }
7736
7737 sub qemu_use_old_bios_files {
7738     my ($machine_type) = @_;
7739
7740     return if !$machine_type;
7741
7742     my $use_old_bios_files = undef;
7743
7744     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7745         $machine_type = $1;
7746         $use_old_bios_files = 1;
7747     } else {
7748         my $version = extract_version($machine_type, kvm_user_version());
7749         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7750         # load new efi bios files on migration. So this hack is required to allow
7751         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7752         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7753         $use_old_bios_files = !min_version($version, 2, 4);
7754     }
7755
7756     return ($use_old_bios_files, $machine_type);
7757 }
7758
7759 sub get_efivars_size {
7760     my ($conf, $efidisk) = @_;
7761
7762     my $arch = get_vm_arch($conf);
7763     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7764     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7765     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7766     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7767     return -s $ovmf_vars;
7768 }
7769
7770 sub update_efidisk_size {
7771     my ($conf) = @_;
7772
7773     return if !defined($conf->{efidisk0});
7774
7775     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7776     $disk->{size} = get_efivars_size($conf);
7777     $conf->{efidisk0} = print_drive($disk);
7778
7779     return;
7780 }
7781
7782 sub update_tpmstate_size {
7783     my ($conf) = @_;
7784
7785     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7786     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7787     $conf->{tpmstate0} = print_drive($disk);
7788 }
7789
7790 sub create_efidisk($$$$$$$) {
7791     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7792
7793     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7794     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7795
7796     my $vars_size_b = -s $ovmf_vars;
7797     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7798     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7799     PVE::Storage::activate_volumes($storecfg, [$volid]);
7800
7801     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7802     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7803
7804     return ($volid, $size/1024);
7805 }
7806
7807 sub vm_iothreads_list {
7808     my ($vmid) = @_;
7809
7810     my $res = mon_cmd($vmid, 'query-iothreads');
7811
7812     my $iothreads = {};
7813     foreach my $iothread (@$res) {
7814         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7815     }
7816
7817     return $iothreads;
7818 }
7819
7820 sub scsihw_infos {
7821     my ($conf, $drive) = @_;
7822
7823     my $maxdev = 0;
7824
7825     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7826         $maxdev = 7;
7827     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7828         $maxdev = 1;
7829     } else {
7830         $maxdev = 256;
7831     }
7832
7833     my $controller = int($drive->{index} / $maxdev);
7834     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7835         ? "virtioscsi"
7836         : "scsihw";
7837
7838     return ($maxdev, $controller, $controller_prefix);
7839 }
7840
7841 sub windows_version {
7842     my ($ostype) = @_;
7843
7844     return 0 if !$ostype;
7845
7846     my $winversion = 0;
7847
7848     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7849         $winversion = 5;
7850     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7851         $winversion = 6;
7852     } elsif ($ostype =~ m/^win(\d+)$/) {
7853         $winversion = $1;
7854     }
7855
7856     return $winversion;
7857 }
7858
7859 sub resolve_dst_disk_format {
7860         my ($storecfg, $storeid, $src_volname, $format) = @_;
7861         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7862
7863         if (!$format) {
7864             # if no target format is specified, use the source disk format as hint
7865             if ($src_volname) {
7866                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7867                 $format = qemu_img_format($scfg, $src_volname);
7868             } else {
7869                 return $defFormat;
7870             }
7871         }
7872
7873         # test if requested format is supported - else use default
7874         my $supported = grep { $_ eq $format } @$validFormats;
7875         $format = $defFormat if !$supported;
7876         return $format;
7877 }
7878
7879 # NOTE: if this logic changes, please update docs & possibly gui logic
7880 sub find_vmstate_storage {
7881     my ($conf, $storecfg) = @_;
7882
7883     # first, return storage from conf if set
7884     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7885
7886     my ($target, $shared, $local);
7887
7888     foreach_storage_used_by_vm($conf, sub {
7889         my ($sid) = @_;
7890         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7891         my $dst = $scfg->{shared} ? \$shared : \$local;
7892         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7893     });
7894
7895     # second, use shared storage where VM has at least one disk
7896     # third, use local storage where VM has at least one disk
7897     # fall back to local storage
7898     $target = $shared // $local // 'local';
7899
7900     return $target;
7901 }
7902
7903 sub generate_uuid {
7904     my ($uuid, $uuid_str);
7905     UUID::generate($uuid);
7906     UUID::unparse($uuid, $uuid_str);
7907     return $uuid_str;
7908 }
7909
7910 sub generate_smbios1_uuid {
7911     return "uuid=".generate_uuid();
7912 }
7913
7914 sub nbd_stop {
7915     my ($vmid) = @_;
7916
7917     mon_cmd($vmid, 'nbd-server-stop');
7918 }
7919
7920 sub create_reboot_request {
7921     my ($vmid) = @_;
7922     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7923         or die "failed to create reboot trigger file: $!\n";
7924     close($fh);
7925 }
7926
7927 sub clear_reboot_request {
7928     my ($vmid) = @_;
7929     my $path = "/run/qemu-server/$vmid.reboot";
7930     my $res = 0;
7931
7932     $res = unlink($path);
7933     die "could not remove reboot request for $vmid: $!"
7934         if !$res && $! != POSIX::ENOENT;
7935
7936     return $res;
7937 }
7938
7939 sub bootorder_from_legacy {
7940     my ($conf, $bootcfg) = @_;
7941
7942     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7943     my $bootindex_hash = {};
7944     my $i = 1;
7945     foreach my $o (split(//, $boot)) {
7946         $bootindex_hash->{$o} = $i*100;
7947         $i++;
7948     }
7949
7950     my $bootorder = {};
7951
7952     PVE::QemuConfig->foreach_volume($conf, sub {
7953         my ($ds, $drive) = @_;
7954
7955         if (drive_is_cdrom ($drive, 1)) {
7956             if ($bootindex_hash->{d}) {
7957                 $bootorder->{$ds} = $bootindex_hash->{d};
7958                 $bootindex_hash->{d} += 1;
7959             }
7960         } elsif ($bootindex_hash->{c}) {
7961             $bootorder->{$ds} = $bootindex_hash->{c}
7962                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7963             $bootindex_hash->{c} += 1;
7964         }
7965     });
7966
7967     if ($bootindex_hash->{n}) {
7968         for (my $i = 0; $i < $MAX_NETS; $i++) {
7969             my $netname = "net$i";
7970             next if !$conf->{$netname};
7971             $bootorder->{$netname} = $bootindex_hash->{n};
7972             $bootindex_hash->{n} += 1;
7973         }
7974     }
7975
7976     return $bootorder;
7977 }
7978
7979 # Generate default device list for 'boot: order=' property. Matches legacy
7980 # default boot order, but with explicit device names. This is important, since
7981 # the fallback for when neither 'order' nor the old format is specified relies
7982 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7983 sub get_default_bootdevices {
7984     my ($conf) = @_;
7985
7986     my @ret = ();
7987
7988     # harddisk
7989     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7990     push @ret, $first if $first;
7991
7992     # cdrom
7993     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7994     push @ret, $first if $first;
7995
7996     # network
7997     for (my $i = 0; $i < $MAX_NETS; $i++) {
7998         my $netname = "net$i";
7999         next if !$conf->{$netname};
8000         push @ret, $netname;
8001         last;
8002     }
8003
8004     return \@ret;
8005 }
8006
8007 sub device_bootorder {
8008     my ($conf) = @_;
8009
8010     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8011
8012     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8013
8014     my $bootorder = {};
8015     if (!defined($boot) || $boot->{legacy}) {
8016         $bootorder = bootorder_from_legacy($conf, $boot);
8017     } elsif ($boot->{order}) {
8018         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8019         for my $dev (PVE::Tools::split_list($boot->{order})) {
8020             $bootorder->{$dev} = $i++;
8021         }
8022     }
8023
8024     return $bootorder;
8025 }
8026
8027 sub register_qmeventd_handle {
8028     my ($vmid) = @_;
8029
8030     my $fh;
8031     my $peer = "/var/run/qmeventd.sock";
8032     my $count = 0;
8033
8034     for (;;) {
8035         $count++;
8036         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8037         last if $fh;
8038         if ($! != EINTR && $! != EAGAIN) {
8039             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8040         }
8041         if ($count > 4) {
8042             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8043               . "after $count retries\n";
8044         }
8045         usleep(25000);
8046     }
8047
8048     # send handshake to mark VM as backing up
8049     print $fh to_json({vzdump => {vmid => "$vmid"}});
8050
8051     # return handle to be closed later when inhibit is no longer required
8052     return $fh;
8053 }
8054
8055 # bash completion helper
8056
8057 sub complete_backup_archives {
8058     my ($cmdname, $pname, $cvalue) = @_;
8059
8060     my $cfg = PVE::Storage::config();
8061
8062     my $storeid;
8063
8064     if ($cvalue =~ m/^([^:]+):/) {
8065         $storeid = $1;
8066     }
8067
8068     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8069
8070     my $res = [];
8071     foreach my $id (keys %$data) {
8072         foreach my $item (@{$data->{$id}}) {
8073             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8074             push @$res, $item->{volid} if defined($item->{volid});
8075         }
8076     }
8077
8078     return $res;
8079 }
8080
8081 my $complete_vmid_full = sub {
8082     my ($running) = @_;
8083
8084     my $idlist = vmstatus();
8085
8086     my $res = [];
8087
8088     foreach my $id (keys %$idlist) {
8089         my $d = $idlist->{$id};
8090         if (defined($running)) {
8091             next if $d->{template};
8092             next if $running && $d->{status} ne 'running';
8093             next if !$running && $d->{status} eq 'running';
8094         }
8095         push @$res, $id;
8096
8097     }
8098     return $res;
8099 };
8100
8101 sub complete_vmid {
8102     return &$complete_vmid_full();
8103 }
8104
8105 sub complete_vmid_stopped {
8106     return &$complete_vmid_full(0);
8107 }
8108
8109 sub complete_vmid_running {
8110     return &$complete_vmid_full(1);
8111 }
8112
8113 sub complete_storage {
8114
8115     my $cfg = PVE::Storage::config();
8116     my $ids = $cfg->{ids};
8117
8118     my $res = [];
8119     foreach my $sid (keys %$ids) {
8120         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8121         next if !$ids->{$sid}->{content}->{images};
8122         push @$res, $sid;
8123     }
8124
8125     return $res;
8126 }
8127
8128 sub complete_migration_storage {
8129     my ($cmd, $param, $current_value, $all_args) = @_;
8130
8131     my $targetnode = @$all_args[1];
8132
8133     my $cfg = PVE::Storage::config();
8134     my $ids = $cfg->{ids};
8135
8136     my $res = [];
8137     foreach my $sid (keys %$ids) {
8138         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8139         next if !$ids->{$sid}->{content}->{images};
8140         push @$res, $sid;
8141     }
8142
8143     return $res;
8144 }
8145
8146 sub vm_is_paused {
8147     my ($vmid) = @_;
8148     my $qmpstatus = eval {
8149         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8150         mon_cmd($vmid, "query-status");
8151     };
8152     warn "$@\n" if $@;
8153     return $qmpstatus && $qmpstatus->{status} eq "paused";
8154 }
8155
8156 sub check_volume_storage_type {
8157     my ($storecfg, $vol) = @_;
8158
8159     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8160     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8161     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8162
8163     die "storage '$storeid' does not support content-type '$vtype'\n"
8164         if !$scfg->{content}->{$vtype};
8165
8166     return 1;
8167 }
8168
8169 1;