PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. The default is read from the"
 371             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.",
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.",
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1040         return $volid;
1041     }
1042
1043     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1044     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1045     if ($@) {
1046         return if $noerr;
1047         die $@;
1048     }
1049     return $volid;
1050 }
1051
1052 my $usb_fmt = {
1053     host => {
1054         default_key => 1,
1055         type => 'string', format => 'pve-qm-usb-device',
1056         format_description => 'HOSTUSBDEVICE|spice',
1057         description => <<EODESCR,
1058 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1059
1060  'bus-port(.port)*' (decimal numbers) or
1061  'vendor_id:product_id' (hexadeciaml numbers) or
1062  'spice'
1063
1064 You can use the 'lsusb -t' command to list existing usb devices.
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 The value 'spice' can be used to add a usb redirection devices for spice.
1070 EODESCR
1071     },
1072     usb3 => {
1073         optional => 1,
1074         type => 'boolean',
1075         description => "Specifies whether if given host option is a USB3 device or port.",
1076         default => 0,
1077     },
1078 };
1079
1080 my $usbdesc = {
1081     optional => 1,
1082     type => 'string', format => $usb_fmt,
1083     description => "Configure an USB device (n is 0 to 4).",
1084 };
1085 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return  $cdrom_path if $cdrom_path;
1265
1266     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1267     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1268     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1269 }
1270
1271 sub get_iso_path {
1272     my ($storecfg, $vmid, $cdrom) = @_;
1273
1274     if ($cdrom eq 'cdrom') {
1275         return get_cdrom_path();
1276     } elsif ($cdrom eq 'none') {
1277         return '';
1278     } elsif ($cdrom =~ m|^/|) {
1279         return $cdrom;
1280     } else {
1281         return PVE::Storage::path($storecfg, $cdrom);
1282     }
1283 }
1284
1285 # try to convert old style file names to volume IDs
1286 sub filename_to_volume_id {
1287     my ($vmid, $file, $media) = @_;
1288
1289      if (!($file eq 'none' || $file eq 'cdrom' ||
1290           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1291
1292         return if $file =~ m|/|;
1293
1294         if ($media && $media eq 'cdrom') {
1295             $file = "local:iso/$file";
1296         } else {
1297             $file = "local:$vmid/$file";
1298         }
1299     }
1300
1301     return $file;
1302 }
1303
1304 sub verify_media_type {
1305     my ($opt, $vtype, $media) = @_;
1306
1307     return if !$media;
1308
1309     my $etype;
1310     if ($media eq 'disk') {
1311         $etype = 'images';
1312     } elsif ($media eq 'cdrom') {
1313         $etype = 'iso';
1314     } else {
1315         die "internal error";
1316     }
1317
1318     return if ($vtype eq $etype);
1319
1320     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1321 }
1322
1323 sub cleanup_drive_path {
1324     my ($opt, $storecfg, $drive) = @_;
1325
1326     # try to convert filesystem paths to volume IDs
1327
1328     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1329         ($drive->{file} !~ m|^/dev/.+|) &&
1330         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1331         ($drive->{file} !~ m/^\d+$/)) {
1332         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1333         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1334             if !$vtype;
1335         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1336         verify_media_type($opt, $vtype, $drive->{media});
1337         $drive->{file} = $volid;
1338     }
1339
1340     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1341 }
1342
1343 sub parse_hotplug_features {
1344     my ($data) = @_;
1345
1346     my $res = {};
1347
1348     return $res if $data eq '0';
1349
1350     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1351
1352     foreach my $feature (PVE::Tools::split_list($data)) {
1353         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1354             $res->{$1} = 1;
1355         } else {
1356             die "invalid hotplug feature '$feature'\n";
1357         }
1358     }
1359     return $res;
1360 }
1361
1362 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1363 sub pve_verify_hotplug_features {
1364     my ($value, $noerr) = @_;
1365
1366     return $value if parse_hotplug_features($value);
1367
1368     return if $noerr;
1369
1370     die "unable to parse hotplug option\n";
1371 }
1372
1373 sub scsi_inquiry {
1374     my($fh, $noerr) = @_;
1375
1376     my $SG_IO = 0x2285;
1377     my $SG_GET_VERSION_NUM = 0x2282;
1378
1379     my $versionbuf = "\x00" x 8;
1380     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1381     if (!$ret) {
1382         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1383         return;
1384     }
1385     my $version = unpack("I", $versionbuf);
1386     if ($version < 30000) {
1387         die "scsi generic interface too old\n"  if !$noerr;
1388         return;
1389     }
1390
1391     my $buf = "\x00" x 36;
1392     my $sensebuf = "\x00" x 8;
1393     my $cmd = pack("C x3 C x1", 0x12, 36);
1394
1395     # see /usr/include/scsi/sg.h
1396     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1397
1398     my $packet = pack(
1399         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1400     );
1401
1402     $ret = ioctl($fh, $SG_IO, $packet);
1403     if (!$ret) {
1404         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1405         return;
1406     }
1407
1408     my @res = unpack($sg_io_hdr_t, $packet);
1409     if ($res[17] || $res[18]) {
1410         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my $res = {};
1415     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1416
1417     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1418     $res->{type} &= 0x1F;
1419
1420     return $res;
1421 }
1422
1423 sub path_is_scsi {
1424     my ($path) = @_;
1425
1426     my $fh = IO::File->new("+<$path") || return;
1427     my $res = scsi_inquiry($fh, 1);
1428     close($fh);
1429
1430     return $res;
1431 }
1432
1433 sub print_tabletdevice_full {
1434     my ($conf, $arch) = @_;
1435
1436     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1437
1438     # we use uhci for old VMs because tablet driver was buggy in older qemu
1439     my $usbbus;
1440     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1441         $usbbus = 'ehci';
1442     } else {
1443         $usbbus = 'uhci';
1444     }
1445
1446     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1447 }
1448
1449 sub print_keyboarddevice_full {
1450     my ($conf, $arch) = @_;
1451
1452     return if $arch ne 'aarch64';
1453
1454     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1455 }
1456
1457 my sub get_drive_id {
1458     my ($drive) = @_;
1459     return "$drive->{interface}$drive->{index}";
1460 }
1461
1462 sub print_drivedevice_full {
1463     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1464
1465     my $device = '';
1466     my $maxdev = 0;
1467
1468     my $drive_id = get_drive_id($drive);
1469     if ($drive->{interface} eq 'virtio') {
1470         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1471         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1472         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1473     } elsif ($drive->{interface} eq 'scsi') {
1474
1475         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1476         my $unit = $drive->{index} % $maxdev;
1477         my $devicetype = 'hd';
1478         my $path = '';
1479         if (drive_is_cdrom($drive)) {
1480             $devicetype = 'cd';
1481         } else {
1482             if ($drive->{file} =~ m|^/|) {
1483                 $path = $drive->{file};
1484                 if (my $info = path_is_scsi($path)) {
1485                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1486                         $devicetype = 'block';
1487                     } elsif ($info->{type} == 1) { # tape
1488                         $devicetype = 'generic';
1489                     }
1490                 }
1491             } else {
1492                  $path = PVE::Storage::path($storecfg, $drive->{file});
1493             }
1494
1495             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1496             my $version = extract_version($machine_type, kvm_user_version());
1497             if ($path =~ m/^iscsi\:\/\// &&
1498                !min_version($version, 4, 1)) {
1499                 $devicetype = 'generic';
1500             }
1501         }
1502
1503         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1504             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1505         } else {
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1507                 .",lun=$drive->{index}";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1512             $device .= ",rotation_rate=1";
1513         }
1514         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1515
1516     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1517         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype";
1523         if ($drive->{interface} eq 'ide') {
1524             $device .= ",bus=ide.$controller,unit=$unit";
1525         } else {
1526             $device .= ",bus=ahci$controller.$unit";
1527         }
1528         $device .= ",drive=drive-$drive_id,id=$drive_id";
1529
1530         if ($devicetype eq 'hd') {
1531             if (my $model = $drive->{model}) {
1532                 $model = URI::Escape::uri_unescape($model);
1533                 $device .= ",model=$model";
1534             }
1535             if ($drive->{ssd}) {
1536                 $device .= ",rotation_rate=1";
1537             }
1538         }
1539         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1540     } elsif ($drive->{interface} eq 'usb') {
1541         die "implement me";
1542         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1543     } else {
1544         die "unsupported interface type";
1545     }
1546
1547     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1548
1549     if (my $serial = $drive->{serial}) {
1550         $serial = URI::Escape::uri_unescape($serial);
1551         $device .= ",serial=$serial";
1552     }
1553
1554
1555     return $device;
1556 }
1557
1558 sub get_initiator_name {
1559     my $initiator;
1560
1561     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1562     while (defined(my $line = <$fh>)) {
1563         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1564         $initiator = $1;
1565         last;
1566     }
1567     $fh->close();
1568
1569     return $initiator;
1570 }
1571
1572 sub print_drive_commandline_full {
1573     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1574
1575     my $path;
1576     my $volid = $drive->{file};
1577     my $format = $drive->{format};
1578     my $drive_id = get_drive_id($drive);
1579
1580     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1581     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1582
1583     if (drive_is_cdrom($drive)) {
1584         $path = get_iso_path($storecfg, $vmid, $volid);
1585         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1586     } else {
1587         if ($storeid) {
1588             $path = PVE::Storage::path($storecfg, $volid);
1589             $format //= qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format //= "raw";
1593         }
1594    }
1595
1596    my $is_rbd = $path =~ m/^rbd:/;
1597
1598     my $opts = '';
1599     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1600     foreach my $o (@qemu_drive_options) {
1601         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1602     }
1603
1604     # snapshot only accepts on|off
1605     if (defined($drive->{snapshot})) {
1606         my $v = $drive->{snapshot} ? 'on' : 'off';
1607         $opts .= ",snapshot=$v";
1608     }
1609
1610     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1611         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1612     }
1613
1614     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1615         my ($dir, $qmpname) = @$type;
1616         if (my $v = $drive->{"mbps$dir"}) {
1617             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1618         }
1619         if (my $v = $drive->{"mbps${dir}_max"}) {
1620             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1621         }
1622         if (my $v = $drive->{"bps${dir}_max_length"}) {
1623             $opts .= ",throttling.bps$qmpname-max-length=$v";
1624         }
1625         if (my $v = $drive->{"iops${dir}"}) {
1626             $opts .= ",throttling.iops$qmpname=$v";
1627         }
1628         if (my $v = $drive->{"iops${dir}_max"}) {
1629             $opts .= ",throttling.iops$qmpname-max=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}_max_length"}) {
1632             $opts .= ",throttling.iops$qmpname-max-length=$v";
1633         }
1634     }
1635
1636     if ($pbs_name) {
1637         $format = "rbd" if $is_rbd;
1638         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1639             if !$format;
1640         $opts .= ",format=alloc-track,file.driver=$format";
1641     } elsif ($format) {
1642         $opts .= ",format=$format";
1643     }
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # io_uring with cache mode writeback or writethrough on krbd will hang...
1655     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1656
1657     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1658     # sometimes, just plain disable...
1659     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1660
1661     if (!$drive->{aio}) {
1662         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1663             # io_uring supports all cache modes
1664             $opts .= ",aio=io_uring";
1665         } else {
1666             # aio native works only with O_DIRECT
1667             if($cache_direct) {
1668                 $opts .= ",aio=native";
1669             } else {
1670                 $opts .= ",aio=threads";
1671             }
1672         }
1673     }
1674
1675     if (!drive_is_cdrom($drive)) {
1676         my $detectzeroes;
1677         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1678             $detectzeroes = 'off';
1679         } elsif ($drive->{discard}) {
1680             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1681         } else {
1682             # This used to be our default with discard not being specified:
1683             $detectzeroes = 'on';
1684         }
1685
1686         # note: 'detect-zeroes' works per blockdev and we want it to persist
1687         # after the alloc-track is removed, so put it on 'file' directly
1688         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1689         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1690     }
1691
1692     if ($pbs_name) {
1693         $opts .= ",backing=$pbs_name";
1694         $opts .= ",auto-remove=on";
1695     }
1696
1697     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1698     my $file_param = "file";
1699     if ($pbs_name) {
1700         # non-rbd drivers require the underlying file to be a seperate block
1701         # node, so add a second .file indirection
1702         $file_param .= ".file" if !$is_rbd;
1703         $file_param .= ".filename";
1704     }
1705     my $pathinfo = $path ? "$file_param=$path," : '';
1706
1707     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1708 }
1709
1710 sub print_pbs_blockdev {
1711     my ($pbs_conf, $pbs_name) = @_;
1712     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1713     $blockdev .= ",repository=$pbs_conf->{repository}";
1714     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1715     $blockdev .= ",archive=$pbs_conf->{archive}";
1716     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1717     return $blockdev;
1718 }
1719
1720 sub print_netdevice_full {
1721     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1722
1723     my $device = $net->{model};
1724     if ($net->{model} eq 'virtio') {
1725          $device = 'virtio-net-pci';
1726      };
1727
1728     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1729     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1730     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1731         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1732         # and out of each queue plus one config interrupt and control vector queue
1733         my $vectors = $net->{queues} * 2 + 2;
1734         $tmpstr .= ",vectors=$vectors,mq=on";
1735     }
1736     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1737
1738     if (my $mtu = $net->{mtu}) {
1739         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1740             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1741             if ($mtu == 1) {
1742                  $mtu = $bridge_mtu;
1743             } elsif ($mtu < 576) {
1744                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1745             } elsif ($mtu > $bridge_mtu) {
1746                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1747             }
1748             $tmpstr .= ",host_mtu=$mtu";
1749         } else {
1750             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1751         }
1752     }
1753
1754     if ($use_old_bios_files) {
1755         my $romfile;
1756         if ($device eq 'virtio-net-pci') {
1757             $romfile = 'pxe-virtio.rom';
1758         } elsif ($device eq 'e1000') {
1759             $romfile = 'pxe-e1000.rom';
1760         } elsif ($device eq 'e1000e') {
1761             $romfile = 'pxe-e1000e.rom';
1762         } elsif ($device eq 'ne2k') {
1763             $romfile = 'pxe-ne2k_pci.rom';
1764         } elsif ($device eq 'pcnet') {
1765             $romfile = 'pxe-pcnet.rom';
1766         } elsif ($device eq 'rtl8139') {
1767             $romfile = 'pxe-rtl8139.rom';
1768         }
1769         $tmpstr .= ",romfile=$romfile" if $romfile;
1770     }
1771
1772     return $tmpstr;
1773 }
1774
1775 sub print_netdev_full {
1776     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1777
1778     my $i = '';
1779     if ($netid =~ m/^net(\d+)$/) {
1780         $i = int($1);
1781     }
1782
1783     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1784
1785     my $ifname = "tap${vmid}i$i";
1786
1787     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1788     die "interface name '$ifname' is too long (max 15 character)\n"
1789         if length($ifname) >= 16;
1790
1791     my $vhostparam = '';
1792     if (is_native($arch)) {
1793         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1794     }
1795
1796     my $vmname = $conf->{name} || "vm$vmid";
1797
1798     my $netdev = "";
1799     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1800
1801     if ($net->{bridge}) {
1802         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1803             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1804     } else {
1805         $netdev = "type=user,id=$netid,hostname=$vmname";
1806     }
1807
1808     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1809
1810     return $netdev;
1811 }
1812
1813 my $vga_map = {
1814     'cirrus' => 'cirrus-vga',
1815     'std' => 'VGA',
1816     'vmware' => 'vmware-svga',
1817     'virtio' => 'virtio-vga',
1818 };
1819
1820 sub print_vga_device {
1821     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1822
1823     my $type = $vga_map->{$vga->{type}};
1824     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1825         $type = 'virtio-gpu';
1826     }
1827     my $vgamem_mb = $vga->{memory};
1828
1829     my $max_outputs = '';
1830     if ($qxlnum) {
1831         $type = $id ? 'qxl' : 'qxl-vga';
1832
1833         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1834             # set max outputs so linux can have up to 4 qxl displays with one device
1835             if (min_version($machine_version, 4, 1)) {
1836                 $max_outputs = ",max_outputs=4";
1837             }
1838         }
1839     }
1840
1841     die "no devicetype for $vga->{type}\n" if !$type;
1842
1843     my $memory = "";
1844     if ($vgamem_mb) {
1845         if ($vga->{type} eq 'virtio') {
1846             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1847             $memory = ",max_hostmem=$bytes";
1848         } elsif ($qxlnum) {
1849             # from https://www.spice-space.org/multiple-monitors.html
1850             $memory = ",vgamem_mb=$vga->{memory}";
1851             my $ram = $vgamem_mb * 4;
1852             my $vram = $vgamem_mb * 2;
1853             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1854         } else {
1855             $memory = ",vgamem_mb=$vga->{memory}";
1856         }
1857     } elsif ($qxlnum && $id) {
1858         $memory = ",ram_size=67108864,vram_size=33554432";
1859     }
1860
1861     my $edidoff = "";
1862     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1863         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1864     }
1865
1866     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1867     my $vgaid = "vga" . ($id // '');
1868     my $pciaddr;
1869     if ($q35 && $vgaid eq 'vga') {
1870         # the first display uses pcie.0 bus on q35 machines
1871         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1872     } else {
1873         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1874     }
1875
1876     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1877 }
1878
1879 sub parse_number_sets {
1880     my ($set) = @_;
1881     my $res = [];
1882     foreach my $part (split(/;/, $set)) {
1883         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1884             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1885             push @$res, [ $1, $2 ];
1886         } else {
1887             die "invalid range: $part\n";
1888         }
1889     }
1890     return $res;
1891 }
1892
1893 sub parse_numa {
1894     my ($data) = @_;
1895
1896     my $res = parse_property_string($numa_fmt, $data);
1897     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1898     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1899     return $res;
1900 }
1901
1902 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1903 sub parse_net {
1904     my ($data) = @_;
1905
1906     my $res = eval { parse_property_string($net_fmt, $data) };
1907     if ($@) {
1908         warn $@;
1909         return;
1910     }
1911     if (!defined($res->{macaddr})) {
1912         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1913         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1914     }
1915     return $res;
1916 }
1917
1918 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1919 sub parse_ipconfig {
1920     my ($data) = @_;
1921
1922     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1923     if ($@) {
1924         warn $@;
1925         return;
1926     }
1927
1928     if ($res->{gw} && !$res->{ip}) {
1929         warn 'gateway specified without specifying an IP address';
1930         return;
1931     }
1932     if ($res->{gw6} && !$res->{ip6}) {
1933         warn 'IPv6 gateway specified without specifying an IPv6 address';
1934         return;
1935     }
1936     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1937         warn 'gateway specified together with DHCP';
1938         return;
1939     }
1940     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1941         # gw6 + auto/dhcp
1942         warn "IPv6 gateway specified together with $res->{ip6} address";
1943         return;
1944     }
1945
1946     if (!$res->{ip} && !$res->{ip6}) {
1947         return { ip => 'dhcp', ip6 => 'dhcp' };
1948     }
1949
1950     return $res;
1951 }
1952
1953 sub print_net {
1954     my $net = shift;
1955
1956     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1957 }
1958
1959 sub add_random_macs {
1960     my ($settings) = @_;
1961
1962     foreach my $opt (keys %$settings) {
1963         next if $opt !~ m/^net(\d+)$/;
1964         my $net = parse_net($settings->{$opt});
1965         next if !$net;
1966         $settings->{$opt} = print_net($net);
1967     }
1968 }
1969
1970 sub vm_is_volid_owner {
1971     my ($storecfg, $vmid, $volid) = @_;
1972
1973     if ($volid !~  m|^/|) {
1974         my ($path, $owner);
1975         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1976         if ($owner && ($owner == $vmid)) {
1977             return 1;
1978         }
1979     }
1980
1981     return;
1982 }
1983
1984 sub vmconfig_register_unused_drive {
1985     my ($storecfg, $vmid, $conf, $drive) = @_;
1986
1987     if (drive_is_cloudinit($drive)) {
1988         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1989         warn $@ if $@;
1990     } elsif (!drive_is_cdrom($drive)) {
1991         my $volid = $drive->{file};
1992         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1993             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1994         }
1995     }
1996 }
1997
1998 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1999 my $smbios1_fmt = {
2000     uuid => {
2001         type => 'string',
2002         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2003         format_description => 'UUID',
2004         description => "Set SMBIOS1 UUID.",
2005         optional => 1,
2006     },
2007     version => {
2008         type => 'string',
2009         pattern => '[A-Za-z0-9+\/]+={0,2}',
2010         format_description => 'Base64 encoded string',
2011         description => "Set SMBIOS1 version.",
2012         optional => 1,
2013     },
2014     serial => {
2015         type => 'string',
2016         pattern => '[A-Za-z0-9+\/]+={0,2}',
2017         format_description => 'Base64 encoded string',
2018         description => "Set SMBIOS1 serial number.",
2019         optional => 1,
2020     },
2021     manufacturer => {
2022         type => 'string',
2023         pattern => '[A-Za-z0-9+\/]+={0,2}',
2024         format_description => 'Base64 encoded string',
2025         description => "Set SMBIOS1 manufacturer.",
2026         optional => 1,
2027     },
2028     product => {
2029         type => 'string',
2030         pattern => '[A-Za-z0-9+\/]+={0,2}',
2031         format_description => 'Base64 encoded string',
2032         description => "Set SMBIOS1 product ID.",
2033         optional => 1,
2034     },
2035     sku => {
2036         type => 'string',
2037         pattern => '[A-Za-z0-9+\/]+={0,2}',
2038         format_description => 'Base64 encoded string',
2039         description => "Set SMBIOS1 SKU string.",
2040         optional => 1,
2041     },
2042     family => {
2043         type => 'string',
2044         pattern => '[A-Za-z0-9+\/]+={0,2}',
2045         format_description => 'Base64 encoded string',
2046         description => "Set SMBIOS1 family string.",
2047         optional => 1,
2048     },
2049     base64 => {
2050         type => 'boolean',
2051         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2052         optional => 1,
2053     },
2054 };
2055
2056 sub parse_smbios1 {
2057     my ($data) = @_;
2058
2059     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2060     warn $@ if $@;
2061     return $res;
2062 }
2063
2064 sub print_smbios1 {
2065     my ($smbios1) = @_;
2066     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2067 }
2068
2069 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2070
2071 sub parse_watchdog {
2072     my ($value) = @_;
2073
2074     return if !$value;
2075
2076     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2077     warn $@ if $@;
2078     return $res;
2079 }
2080
2081 sub parse_guest_agent {
2082     my ($conf) = @_;
2083
2084     return {} if !defined($conf->{agent});
2085
2086     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2087     warn $@ if $@;
2088
2089     # if the agent is disabled ignore the other potentially set properties
2090     return {} if !$res->{enabled};
2091     return $res;
2092 }
2093
2094 sub get_qga_key {
2095     my ($conf, $key) = @_;
2096     return undef if !defined($conf->{agent});
2097
2098     my $agent = parse_guest_agent($conf);
2099     return $agent->{$key};
2100 }
2101
2102 sub parse_vga {
2103     my ($value) = @_;
2104
2105     return {} if !$value;
2106     my $res = eval { parse_property_string($vga_fmt, $value) };
2107     warn $@ if $@;
2108     return $res;
2109 }
2110
2111 sub parse_rng {
2112     my ($value) = @_;
2113
2114     return if !$value;
2115
2116     my $res = eval { parse_property_string($rng_fmt, $value) };
2117     warn $@ if $@;
2118     return $res;
2119 }
2120
2121 sub parse_meta_info {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub new_meta_info_string {
2132     my () = @_; # for now do not allow to override any value
2133
2134     return PVE::JSONSchema::print_property_string(
2135         {
2136             'creation-qemu' => kvm_user_version(),
2137             ctime => "". int(time()),
2138         },
2139         $meta_info_fmt
2140     );
2141 }
2142
2143 sub qemu_created_version_fixups {
2144     my ($conf, $forcemachine, $kvmver) = @_;
2145
2146     my $meta = parse_meta_info($conf->{meta}) // {};
2147     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2148
2149     # check if we need to apply some handling for VMs that always use the latest machine version but
2150     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2151     # would be required (we do not want to pin machine version for non-windows OS type)
2152     if (
2153         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2154         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2155         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2156         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2157     ) {
2158         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2159         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2160             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2161             # and thus with the predictable interface naming of systemd
2162             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2163         }
2164     }
2165     return;
2166 }
2167
2168 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2169 sub verify_usb_device {
2170     my ($value, $noerr) = @_;
2171
2172     return $value if parse_usb_device($value);
2173
2174     return if $noerr;
2175
2176     die "unable to parse usb device\n";
2177 }
2178
2179 # add JSON properties for create and set function
2180 sub json_config_properties {
2181     my $prop = shift;
2182
2183     my $skip_json_config_opts = {
2184         parent => 1,
2185         snaptime => 1,
2186         vmstate => 1,
2187         runningmachine => 1,
2188         runningcpu => 1,
2189         meta => 1,
2190     };
2191
2192     foreach my $opt (keys %$confdesc) {
2193         next if $skip_json_config_opts->{$opt};
2194         $prop->{$opt} = $confdesc->{$opt};
2195     }
2196
2197     return $prop;
2198 }
2199
2200 # return copy of $confdesc_cloudinit to generate documentation
2201 sub cloudinit_config_properties {
2202
2203     return dclone($confdesc_cloudinit);
2204 }
2205
2206 sub check_type {
2207     my ($key, $value) = @_;
2208
2209     die "unknown setting '$key'\n" if !$confdesc->{$key};
2210
2211     my $type = $confdesc->{$key}->{type};
2212
2213     if (!defined($value)) {
2214         die "got undefined value\n";
2215     }
2216
2217     if ($value =~ m/[\n\r]/) {
2218         die "property contains a line feed\n";
2219     }
2220
2221     if ($type eq 'boolean') {
2222         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2223         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2224         die "type check ('boolean') failed - got '$value'\n";
2225     } elsif ($type eq 'integer') {
2226         return int($1) if $value =~ m/^(\d+)$/;
2227         die "type check ('integer') failed - got '$value'\n";
2228     } elsif ($type eq 'number') {
2229         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2230         die "type check ('number') failed - got '$value'\n";
2231     } elsif ($type eq 'string') {
2232         if (my $fmt = $confdesc->{$key}->{format}) {
2233             PVE::JSONSchema::check_format($fmt, $value);
2234             return $value;
2235         }
2236         $value =~ s/^\"(.*)\"$/$1/;
2237         return $value;
2238     } else {
2239         die "internal error"
2240     }
2241 }
2242
2243 sub destroy_vm {
2244     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2245
2246     my $conf = PVE::QemuConfig->load_config($vmid);
2247
2248     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2249
2250     if ($conf->{template}) {
2251         # check if any base image is still used by a linked clone
2252         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2253                 my ($ds, $drive) = @_;
2254                 return if drive_is_cdrom($drive);
2255
2256                 my $volid = $drive->{file};
2257                 return if !$volid || $volid =~ m|^/|;
2258
2259                 die "base volume '$volid' is still in use by linked cloned\n"
2260                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2261
2262         });
2263     }
2264
2265     my $volids = {};
2266     my $remove_owned_drive = sub {
2267         my ($ds, $drive) = @_;
2268         return if drive_is_cdrom($drive, 1);
2269
2270         my $volid = $drive->{file};
2271         return if !$volid || $volid =~ m|^/|;
2272         return if $volids->{$volid};
2273
2274         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2275         return if !$path || !$owner || ($owner != $vmid);
2276
2277         $volids->{$volid} = 1;
2278         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2279         warn "Could not remove disk '$volid', check manually: $@" if $@;
2280     };
2281
2282     # only remove disks owned by this VM (referenced in the config)
2283     my $include_opts = {
2284         include_unused => 1,
2285         extra_keys => ['vmstate'],
2286     };
2287     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2288
2289     for my $snap (values %{$conf->{snapshots}}) {
2290         next if !defined($snap->{vmstate});
2291         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2292         next if !defined($drive);
2293         $remove_owned_drive->('vmstate', $drive);
2294     }
2295
2296     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2297
2298     if ($purge_unreferenced) { # also remove unreferenced disk
2299         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2300         PVE::Storage::foreach_volid($vmdisks, sub {
2301             my ($volid, $sid, $volname, $d) = @_;
2302             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2303             warn $@ if $@;
2304         });
2305     }
2306
2307     if (defined $replacement_conf) {
2308         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2309     } else {
2310         PVE::QemuConfig->destroy_config($vmid);
2311     }
2312 }
2313
2314 sub parse_vm_config {
2315     my ($filename, $raw, $strict) = @_;
2316
2317     return if !defined($raw);
2318
2319     my $res = {
2320         digest => Digest::SHA::sha1_hex($raw),
2321         snapshots => {},
2322         pending => {},
2323     };
2324
2325     my $handle_error = sub {
2326         my ($msg) = @_;
2327
2328         if ($strict) {
2329             die $msg;
2330         } else {
2331             warn $msg;
2332         }
2333     };
2334
2335     $filename =~ m|/qemu-server/(\d+)\.conf$|
2336         || die "got strange filename '$filename'";
2337
2338     my $vmid = $1;
2339
2340     my $conf = $res;
2341     my $descr;
2342     my $section = '';
2343
2344     my @lines = split(/\n/, $raw);
2345     foreach my $line (@lines) {
2346         next if $line =~ m/^\s*$/;
2347
2348         if ($line =~ m/^\[PENDING\]\s*$/i) {
2349             $section = 'pending';
2350             if (defined($descr)) {
2351                 $descr =~ s/\s+$//;
2352                 $conf->{description} = $descr;
2353             }
2354             $descr = undef;
2355             $conf = $res->{$section} = {};
2356             next;
2357
2358         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2359             $section = $1;
2360             if (defined($descr)) {
2361                 $descr =~ s/\s+$//;
2362                 $conf->{description} = $descr;
2363             }
2364             $descr = undef;
2365             $conf = $res->{snapshots}->{$section} = {};
2366             next;
2367         }
2368
2369         if ($line =~ m/^\#(.*)\s*$/) {
2370             $descr = '' if !defined($descr);
2371             $descr .= PVE::Tools::decode_text($1) . "\n";
2372             next;
2373         }
2374
2375         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2376             $descr = '' if !defined($descr);
2377             $descr .= PVE::Tools::decode_text($2);
2378         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2379             $conf->{snapstate} = $1;
2380         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2381             my $key = $1;
2382             my $value = $2;
2383             $conf->{$key} = $value;
2384         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2385             my $value = $1;
2386             if ($section eq 'pending') {
2387                 $conf->{delete} = $value; # we parse this later
2388             } else {
2389                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2390             }
2391         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2392             my $key = $1;
2393             my $value = $2;
2394             eval { $value = check_type($key, $value); };
2395             if ($@) {
2396                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2397             } else {
2398                 $key = 'ide2' if $key eq 'cdrom';
2399                 my $fmt = $confdesc->{$key}->{format};
2400                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2401                     my $v = parse_drive($key, $value);
2402                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2403                         $v->{file} = $volid;
2404                         $value = print_drive($v);
2405                     } else {
2406                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2407                         next;
2408                     }
2409                 }
2410
2411                 $conf->{$key} = $value;
2412             }
2413         } else {
2414             $handle_error->("vm $vmid - unable to parse config: $line\n");
2415         }
2416     }
2417
2418     if (defined($descr)) {
2419         $descr =~ s/\s+$//;
2420         $conf->{description} = $descr;
2421     }
2422     delete $res->{snapstate}; # just to be sure
2423
2424     return $res;
2425 }
2426
2427 sub write_vm_config {
2428     my ($filename, $conf) = @_;
2429
2430     delete $conf->{snapstate}; # just to be sure
2431
2432     if ($conf->{cdrom}) {
2433         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2434         $conf->{ide2} = $conf->{cdrom};
2435         delete $conf->{cdrom};
2436     }
2437
2438     # we do not use 'smp' any longer
2439     if ($conf->{sockets}) {
2440         delete $conf->{smp};
2441     } elsif ($conf->{smp}) {
2442         $conf->{sockets} = $conf->{smp};
2443         delete $conf->{cores};
2444         delete $conf->{smp};
2445     }
2446
2447     my $used_volids = {};
2448
2449     my $cleanup_config = sub {
2450         my ($cref, $pending, $snapname) = @_;
2451
2452         foreach my $key (keys %$cref) {
2453             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2454                 $key eq 'snapstate' || $key eq 'pending';
2455             my $value = $cref->{$key};
2456             if ($key eq 'delete') {
2457                 die "propertry 'delete' is only allowed in [PENDING]\n"
2458                     if !$pending;
2459                 # fixme: check syntax?
2460                 next;
2461             }
2462             eval { $value = check_type($key, $value); };
2463             die "unable to parse value of '$key' - $@" if $@;
2464
2465             $cref->{$key} = $value;
2466
2467             if (!$snapname && is_valid_drivename($key)) {
2468                 my $drive = parse_drive($key, $value);
2469                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2470             }
2471         }
2472     };
2473
2474     &$cleanup_config($conf);
2475
2476     &$cleanup_config($conf->{pending}, 1);
2477
2478     foreach my $snapname (keys %{$conf->{snapshots}}) {
2479         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2480         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2481     }
2482
2483     # remove 'unusedX' settings if we re-add a volume
2484     foreach my $key (keys %$conf) {
2485         my $value = $conf->{$key};
2486         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2487             delete $conf->{$key};
2488         }
2489     }
2490
2491     my $generate_raw_config = sub {
2492         my ($conf, $pending) = @_;
2493
2494         my $raw = '';
2495
2496         # add description as comment to top of file
2497         if (defined(my $descr = $conf->{description})) {
2498             if ($descr) {
2499                 foreach my $cl (split(/\n/, $descr)) {
2500                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2501                 }
2502             } else {
2503                 $raw .= "#\n" if $pending;
2504             }
2505         }
2506
2507         foreach my $key (sort keys %$conf) {
2508             next if $key =~ /^(digest|description|pending|snapshots)$/;
2509             $raw .= "$key: $conf->{$key}\n";
2510         }
2511         return $raw;
2512     };
2513
2514     my $raw = &$generate_raw_config($conf);
2515
2516     if (scalar(keys %{$conf->{pending}})){
2517         $raw .= "\n[PENDING]\n";
2518         $raw .= &$generate_raw_config($conf->{pending}, 1);
2519     }
2520
2521     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2522         $raw .= "\n[$snapname]\n";
2523         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2524     }
2525
2526     return $raw;
2527 }
2528
2529 sub load_defaults {
2530
2531     my $res = {};
2532
2533     # we use static defaults from our JSON schema configuration
2534     foreach my $key (keys %$confdesc) {
2535         if (defined(my $default = $confdesc->{$key}->{default})) {
2536             $res->{$key} = $default;
2537         }
2538     }
2539
2540     return $res;
2541 }
2542
2543 sub config_list {
2544     my $vmlist = PVE::Cluster::get_vmlist();
2545     my $res = {};
2546     return $res if !$vmlist || !$vmlist->{ids};
2547     my $ids = $vmlist->{ids};
2548     my $nodename = nodename();
2549
2550     foreach my $vmid (keys %$ids) {
2551         my $d = $ids->{$vmid};
2552         next if !$d->{node} || $d->{node} ne $nodename;
2553         next if !$d->{type} || $d->{type} ne 'qemu';
2554         $res->{$vmid}->{exists} = 1;
2555     }
2556     return $res;
2557 }
2558
2559 # test if VM uses local resources (to prevent migration)
2560 sub check_local_resources {
2561     my ($conf, $noerr) = @_;
2562
2563     my @loc_res = ();
2564
2565     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2566     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2567
2568     push @loc_res, "ivshmem" if $conf->{ivshmem};
2569
2570     foreach my $k (keys %$conf) {
2571         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2572         # sockets are safe: they will recreated be on the target side post-migrate
2573         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2574         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2575     }
2576
2577     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2578
2579     return \@loc_res;
2580 }
2581
2582 # check if used storages are available on all nodes (use by migrate)
2583 sub check_storage_availability {
2584     my ($storecfg, $conf, $node) = @_;
2585
2586     PVE::QemuConfig->foreach_volume($conf, sub {
2587         my ($ds, $drive) = @_;
2588
2589         my $volid = $drive->{file};
2590         return if !$volid;
2591
2592         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2593         return if !$sid;
2594
2595         # check if storage is available on both nodes
2596         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2597         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2598
2599         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2600
2601         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2602             if !$scfg->{content}->{$vtype};
2603    });
2604 }
2605
2606 # list nodes where all VM images are available (used by has_feature API)
2607 sub shared_nodes {
2608     my ($conf, $storecfg) = @_;
2609
2610     my $nodelist = PVE::Cluster::get_nodelist();
2611     my $nodehash = { map { $_ => 1 } @$nodelist };
2612     my $nodename = nodename();
2613
2614     PVE::QemuConfig->foreach_volume($conf, sub {
2615         my ($ds, $drive) = @_;
2616
2617         my $volid = $drive->{file};
2618         return if !$volid;
2619
2620         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2621         if ($storeid) {
2622             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2623             if ($scfg->{disable}) {
2624                 $nodehash = {};
2625             } elsif (my $avail = $scfg->{nodes}) {
2626                 foreach my $node (keys %$nodehash) {
2627                     delete $nodehash->{$node} if !$avail->{$node};
2628                 }
2629             } elsif (!$scfg->{shared}) {
2630                 foreach my $node (keys %$nodehash) {
2631                     delete $nodehash->{$node} if $node ne $nodename
2632                 }
2633             }
2634         }
2635     });
2636
2637     return $nodehash
2638 }
2639
2640 sub check_local_storage_availability {
2641     my ($conf, $storecfg) = @_;
2642
2643     my $nodelist = PVE::Cluster::get_nodelist();
2644     my $nodehash = { map { $_ => {} } @$nodelist };
2645
2646     PVE::QemuConfig->foreach_volume($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         my $volid = $drive->{file};
2650         return if !$volid;
2651
2652         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2653         if ($storeid) {
2654             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2655
2656             if ($scfg->{disable}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2659                 }
2660             } elsif (my $avail = $scfg->{nodes}) {
2661                 foreach my $node (keys %$nodehash) {
2662                     if (!$avail->{$node}) {
2663                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2664                     }
2665                 }
2666             }
2667         }
2668     });
2669
2670     foreach my $node (values %$nodehash) {
2671         if (my $unavail = $node->{unavailable_storages}) {
2672             $node->{unavailable_storages} = [ sort keys %$unavail ];
2673         }
2674     }
2675
2676     return $nodehash
2677 }
2678
2679 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2680 sub check_running {
2681     my ($vmid, $nocheck, $node) = @_;
2682
2683     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2684     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2685 }
2686
2687 sub vzlist {
2688
2689     my $vzlist = config_list();
2690
2691     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2692
2693     while (defined(my $de = $fd->read)) {
2694         next if $de !~ m/^(\d+)\.pid$/;
2695         my $vmid = $1;
2696         next if !defined($vzlist->{$vmid});
2697         if (my $pid = check_running($vmid)) {
2698             $vzlist->{$vmid}->{pid} = $pid;
2699         }
2700     }
2701
2702     return $vzlist;
2703 }
2704
2705 our $vmstatus_return_properties = {
2706     vmid => get_standard_option('pve-vmid'),
2707     status => {
2708         description => "Qemu process status.",
2709         type => 'string',
2710         enum => ['stopped', 'running'],
2711     },
2712     maxmem => {
2713         description => "Maximum memory in bytes.",
2714         type => 'integer',
2715         optional => 1,
2716         renderer => 'bytes',
2717     },
2718     maxdisk => {
2719         description => "Root disk size in bytes.",
2720         type => 'integer',
2721         optional => 1,
2722         renderer => 'bytes',
2723     },
2724     name => {
2725         description => "VM name.",
2726         type => 'string',
2727         optional => 1,
2728     },
2729     qmpstatus => {
2730         description => "Qemu QMP agent status.",
2731         type => 'string',
2732         optional => 1,
2733     },
2734     pid => {
2735         description => "PID of running qemu process.",
2736         type => 'integer',
2737         optional => 1,
2738     },
2739     uptime => {
2740         description => "Uptime.",
2741         type => 'integer',
2742         optional => 1,
2743         renderer => 'duration',
2744     },
2745     cpus => {
2746         description => "Maximum usable CPUs.",
2747         type => 'number',
2748         optional => 1,
2749     },
2750     lock => {
2751         description => "The current config lock, if any.",
2752         type => 'string',
2753         optional => 1,
2754     },
2755     tags => {
2756         description  => "The current configured tags, if any",
2757         type => 'string',
2758         optional => 1,
2759     },
2760     'running-machine' => {
2761         description => "The currently running machine type (if running).",
2762         type => 'string',
2763         optional => 1,
2764     },
2765     'running-qemu' => {
2766         description => "The currently running QEMU version (if running).",
2767         type => 'string',
2768         optional => 1,
2769     },
2770 };
2771
2772 my $last_proc_pid_stat;
2773
2774 # get VM status information
2775 # This must be fast and should not block ($full == false)
2776 # We only query KVM using QMP if $full == true (this can be slow)
2777 sub vmstatus {
2778     my ($opt_vmid, $full) = @_;
2779
2780     my $res = {};
2781
2782     my $storecfg = PVE::Storage::config();
2783
2784     my $list = vzlist();
2785     my $defaults = load_defaults();
2786
2787     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2788
2789     my $cpucount = $cpuinfo->{cpus} || 1;
2790
2791     foreach my $vmid (keys %$list) {
2792         next if $opt_vmid && ($vmid ne $opt_vmid);
2793
2794         my $conf = PVE::QemuConfig->load_config($vmid);
2795
2796         my $d = { vmid => int($vmid) };
2797         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2798
2799         # fixme: better status?
2800         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2801
2802         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2803         if (defined($size)) {
2804             $d->{disk} = 0; # no info available
2805             $d->{maxdisk} = $size;
2806         } else {
2807             $d->{disk} = 0;
2808             $d->{maxdisk} = 0;
2809         }
2810
2811         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2812             * ($conf->{cores} || $defaults->{cores});
2813         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2814         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2815
2816         $d->{name} = $conf->{name} || "VM $vmid";
2817         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2818             : $defaults->{memory}*(1024*1024);
2819
2820         if ($conf->{balloon}) {
2821             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2822             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2823                 : $defaults->{shares};
2824         }
2825
2826         $d->{uptime} = 0;
2827         $d->{cpu} = 0;
2828         $d->{mem} = 0;
2829
2830         $d->{netout} = 0;
2831         $d->{netin} = 0;
2832
2833         $d->{diskread} = 0;
2834         $d->{diskwrite} = 0;
2835
2836         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2837
2838         $d->{serial} = 1 if conf_has_serial($conf);
2839         $d->{lock} = $conf->{lock} if $conf->{lock};
2840         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2841
2842         $res->{$vmid} = $d;
2843     }
2844
2845     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2846     foreach my $dev (keys %$netdev) {
2847         next if $dev !~ m/^tap([1-9]\d*)i/;
2848         my $vmid = $1;
2849         my $d = $res->{$vmid};
2850         next if !$d;
2851
2852         $d->{netout} += $netdev->{$dev}->{receive};
2853         $d->{netin} += $netdev->{$dev}->{transmit};
2854
2855         if ($full) {
2856             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2857             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2858         }
2859
2860     }
2861
2862     my $ctime = gettimeofday;
2863
2864     foreach my $vmid (keys %$list) {
2865
2866         my $d = $res->{$vmid};
2867         my $pid = $d->{pid};
2868         next if !$pid;
2869
2870         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2871         next if !$pstat; # not running
2872
2873         my $used = $pstat->{utime} + $pstat->{stime};
2874
2875         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2876
2877         if ($pstat->{vsize}) {
2878             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2879         }
2880
2881         my $old = $last_proc_pid_stat->{$pid};
2882         if (!$old) {
2883             $last_proc_pid_stat->{$pid} = {
2884                 time => $ctime,
2885                 used => $used,
2886                 cpu => 0,
2887             };
2888             next;
2889         }
2890
2891         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2892
2893         if ($dtime > 1000) {
2894             my $dutime = $used -  $old->{used};
2895
2896             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2897             $last_proc_pid_stat->{$pid} = {
2898                 time => $ctime,
2899                 used => $used,
2900                 cpu => $d->{cpu},
2901             };
2902         } else {
2903             $d->{cpu} = $old->{cpu};
2904         }
2905     }
2906
2907     return $res if !$full;
2908
2909     my $qmpclient = PVE::QMPClient->new();
2910
2911     my $ballooncb = sub {
2912         my ($vmid, $resp) = @_;
2913
2914         my $info = $resp->{'return'};
2915         return if !$info->{max_mem};
2916
2917         my $d = $res->{$vmid};
2918
2919         # use memory assigned to VM
2920         $d->{maxmem} = $info->{max_mem};
2921         $d->{balloon} = $info->{actual};
2922
2923         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2924             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2925             $d->{freemem} = $info->{free_mem};
2926         }
2927
2928         $d->{ballooninfo} = $info;
2929     };
2930
2931     my $blockstatscb = sub {
2932         my ($vmid, $resp) = @_;
2933         my $data = $resp->{'return'} || [];
2934         my $totalrdbytes = 0;
2935         my $totalwrbytes = 0;
2936
2937         for my $blockstat (@$data) {
2938             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2939             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2940
2941             $blockstat->{device} =~ s/drive-//;
2942             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2943         }
2944         $res->{$vmid}->{diskread} = $totalrdbytes;
2945         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2946     };
2947
2948     my $machinecb = sub {
2949         my ($vmid, $resp) = @_;
2950         my $data = $resp->{'return'} || [];
2951
2952         $res->{$vmid}->{'running-machine'} =
2953             PVE::QemuServer::Machine::current_from_query_machines($data);
2954     };
2955
2956     my $versioncb = sub {
2957         my ($vmid, $resp) = @_;
2958         my $data = $resp->{'return'} // {};
2959         my $version = 'unknown';
2960
2961         if (my $v = $data->{qemu}) {
2962             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2963         }
2964
2965         $res->{$vmid}->{'running-qemu'} = $version;
2966     };
2967
2968     my $statuscb = sub {
2969         my ($vmid, $resp) = @_;
2970
2971         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2972         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2973         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2974         # this fails if ballon driver is not loaded, so this must be
2975         # the last commnand (following command are aborted if this fails).
2976         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2977
2978         my $status = 'unknown';
2979         if (!defined($status = $resp->{'return'}->{status})) {
2980             warn "unable to get VM status\n";
2981             return;
2982         }
2983
2984         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2985     };
2986
2987     foreach my $vmid (keys %$list) {
2988         next if $opt_vmid && ($vmid ne $opt_vmid);
2989         next if !$res->{$vmid}->{pid}; # not running
2990         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2991     }
2992
2993     $qmpclient->queue_execute(undef, 2);
2994
2995     foreach my $vmid (keys %$list) {
2996         next if $opt_vmid && ($vmid ne $opt_vmid);
2997         next if !$res->{$vmid}->{pid}; #not running
2998
2999         # we can't use the $qmpclient since it might have already aborted on
3000         # 'query-balloon', but this might also fail for older versions...
3001         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3002         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3003     }
3004
3005     foreach my $vmid (keys %$list) {
3006         next if $opt_vmid && ($vmid ne $opt_vmid);
3007         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3008     }
3009
3010     return $res;
3011 }
3012
3013 sub conf_has_serial {
3014     my ($conf) = @_;
3015
3016     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3017         if ($conf->{"serial$i"}) {
3018             return 1;
3019         }
3020     }
3021
3022     return 0;
3023 }
3024
3025 sub conf_has_audio {
3026     my ($conf, $id) = @_;
3027
3028     $id //= 0;
3029     my $audio = $conf->{"audio$id"};
3030     return if !defined($audio);
3031
3032     my $audioproperties = parse_property_string($audio_fmt, $audio);
3033     my $audiodriver = $audioproperties->{driver} // 'spice';
3034
3035     return {
3036         dev => $audioproperties->{device},
3037         dev_id => "audiodev$id",
3038         backend => $audiodriver,
3039         backend_id => "$audiodriver-backend${id}",
3040     };
3041 }
3042
3043 sub audio_devs {
3044     my ($audio, $audiopciaddr, $machine_version) = @_;
3045
3046     my $devs = [];
3047
3048     my $id = $audio->{dev_id};
3049     my $audiodev = "";
3050     if (min_version($machine_version, 4, 2)) {
3051         $audiodev = ",audiodev=$audio->{backend_id}";
3052     }
3053
3054     if ($audio->{dev} eq 'AC97') {
3055         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3056     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3057         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3058         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3059         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3060     } else {
3061         die "unkown audio device '$audio->{dev}', implement me!";
3062     }
3063
3064     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3065
3066     return $devs;
3067 }
3068
3069 sub get_tpm_paths {
3070     my ($vmid) = @_;
3071     return {
3072         socket => "/var/run/qemu-server/$vmid.swtpm",
3073         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3074     };
3075 }
3076
3077 sub add_tpm_device {
3078     my ($vmid, $devices, $conf) = @_;
3079
3080     return if !$conf->{tpmstate0};
3081
3082     my $paths = get_tpm_paths($vmid);
3083
3084     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3085     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3086     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3087 }
3088
3089 sub start_swtpm {
3090     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3091
3092     return if !$tpmdrive;
3093
3094     my $state;
3095     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3096     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3097     if ($storeid) {
3098         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3099     } else {
3100         $state = $tpm->{file};
3101     }
3102
3103     my $paths = get_tpm_paths($vmid);
3104
3105     # during migration, we will get state from remote
3106     #
3107     if (!$migration) {
3108         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3109         my $setup_cmd = [
3110             "swtpm_setup",
3111             "--tpmstate",
3112             "file://$state",
3113             "--createek",
3114             "--create-ek-cert",
3115             "--create-platform-cert",
3116             "--lock-nvram",
3117             "--config",
3118             "/etc/swtpm_setup.conf", # do not use XDG configs
3119             "--runas",
3120             "0", # force creation as root, error if not possible
3121             "--not-overwrite", # ignore existing state, do not modify
3122         ];
3123
3124         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3125         # TPM 2.0 supports ECC crypto, use if possible
3126         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3127
3128         run_command($setup_cmd, outfunc => sub {
3129             print "swtpm_setup: $1\n";
3130         });
3131     }
3132
3133     my $emulator_cmd = [
3134         "swtpm",
3135         "socket",
3136         "--tpmstate",
3137         "backend-uri=file://$state,mode=0600",
3138         "--ctrl",
3139         "type=unixio,path=$paths->{socket},mode=0600",
3140         "--pid",
3141         "file=$paths->{pid}",
3142         "--terminate", # terminate on QEMU disconnect
3143         "--daemon",
3144     ];
3145     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3146     run_command($emulator_cmd, outfunc => sub { print $1; });
3147
3148     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3149     while (! -e $paths->{pid}) {
3150         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3151         usleep(50_000);
3152     }
3153
3154     # return untainted PID of swtpm daemon so it can be killed on error
3155     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3156     return $1;
3157 }
3158
3159 sub vga_conf_has_spice {
3160     my ($vga) = @_;
3161
3162     my $vgaconf = parse_vga($vga);
3163     my $vgatype = $vgaconf->{type};
3164     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3165
3166     return $1 || 1;
3167 }
3168
3169 sub is_native($) {
3170     my ($arch) = @_;
3171     return get_host_arch() eq $arch;
3172 }
3173
3174 sub get_vm_arch {
3175     my ($conf) = @_;
3176     return $conf->{arch} // get_host_arch();
3177 }
3178
3179 my $default_machines = {
3180     x86_64 => 'pc',
3181     aarch64 => 'virt',
3182 };
3183
3184 sub get_installed_machine_version {
3185     my ($kvmversion) = @_;
3186     $kvmversion = kvm_user_version() if !defined($kvmversion);
3187     $kvmversion =~ m/^(\d+\.\d+)/;
3188     return $1;
3189 }
3190
3191 sub windows_get_pinned_machine_version {
3192     my ($machine, $base_version, $kvmversion) = @_;
3193
3194     my $pin_version = $base_version;
3195     if (!defined($base_version) ||
3196         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3197     ) {
3198         $pin_version = get_installed_machine_version($kvmversion);
3199     }
3200     if (!$machine || $machine eq 'pc') {
3201         $machine = "pc-i440fx-$pin_version";
3202     } elsif ($machine eq 'q35') {
3203         $machine = "pc-q35-$pin_version";
3204     } elsif ($machine eq 'virt') {
3205         $machine = "virt-$pin_version";
3206     } else {
3207         warn "unknown machine type '$machine', not touching that!\n";
3208     }
3209
3210     return $machine;
3211 }
3212
3213 sub get_vm_machine {
3214     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3215
3216     my $machine = $forcemachine || $conf->{machine};
3217
3218     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3219         $kvmversion //= kvm_user_version();
3220         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3221         # layout which confuses windows quite a bit and may result in various regressions..
3222         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3223         if (windows_version($conf->{ostype})) {
3224             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3225         }
3226         $arch //= 'x86_64';
3227         $machine ||= $default_machines->{$arch};
3228         if ($add_pve_version) {
3229             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3230             $machine .= "+pve$pvever";
3231         }
3232     }
3233
3234     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3235         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3236         $machine = $1 if $is_pxe;
3237
3238         # for version-pinned machines that do not include a pve-version (e.g.
3239         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3240         $machine .= '+pve0';
3241
3242         $machine .= '.pxe' if $is_pxe;
3243     }
3244
3245     return $machine;
3246 }
3247
3248 sub get_ovmf_files($$$) {
3249     my ($arch, $efidisk, $smm) = @_;
3250
3251     my $types = $OVMF->{$arch}
3252         or die "no OVMF images known for architecture '$arch'\n";
3253
3254     my $type = 'default';
3255     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3256         $type = $smm ? "4m" : "4m-no-smm";
3257         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3258     }
3259
3260     return $types->{$type}->@*;
3261 }
3262
3263 my $Arch2Qemu = {
3264     aarch64 => '/usr/bin/qemu-system-aarch64',
3265     x86_64 => '/usr/bin/qemu-system-x86_64',
3266 };
3267 sub get_command_for_arch($) {
3268     my ($arch) = @_;
3269     return '/usr/bin/kvm' if is_native($arch);
3270
3271     my $cmd = $Arch2Qemu->{$arch}
3272         or die "don't know how to emulate architecture '$arch'\n";
3273     return $cmd;
3274 }
3275
3276 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3277 # to use in a QEMU command line (-cpu element), first array_intersect the result
3278 # of query_supported_ with query_understood_. This is necessary because:
3279 #
3280 # a) query_understood_ returns flags the host cannot use and
3281 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3282 #    flags, but CPU settings - with most of them being flags. Those settings
3283 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3284 #
3285 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3286 # expensive. If you need the value returned from this, you can get it much
3287 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3288 # $accel being 'kvm' or 'tcg'.
3289 #
3290 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3291 # changes, automatically populating pmxcfs.
3292 #
3293 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3294 # since kvm and tcg machines support different flags
3295 #
3296 sub query_supported_cpu_flags {
3297     my ($arch) = @_;
3298
3299     $arch //= get_host_arch();
3300     my $default_machine = $default_machines->{$arch};
3301
3302     my $flags = {};
3303
3304     # FIXME: Once this is merged, the code below should work for ARM as well:
3305     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3306     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3307         $arch eq "aarch64";
3308
3309     my $kvm_supported = defined(kvm_version());
3310     my $qemu_cmd = get_command_for_arch($arch);
3311     my $fakevmid = -1;
3312     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3313
3314     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3315     my $query_supported_run_qemu = sub {
3316         my ($kvm) = @_;
3317
3318         my $flags = {};
3319         my $cmd = [
3320             $qemu_cmd,
3321             '-machine', $default_machine,
3322             '-display', 'none',
3323             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3324             '-mon', 'chardev=qmp,mode=control',
3325             '-pidfile', $pidfile,
3326             '-S', '-daemonize'
3327         ];
3328
3329         if (!$kvm) {
3330             push @$cmd, '-accel', 'tcg';
3331         }
3332
3333         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3334         die "QEMU flag querying VM exited with code " . $rc if $rc;
3335
3336         eval {
3337             my $cmd_result = mon_cmd(
3338                 $fakevmid,
3339                 'query-cpu-model-expansion',
3340                 type => 'full',
3341                 model => { name => 'host' }
3342             );
3343
3344             my $props = $cmd_result->{model}->{props};
3345             foreach my $prop (keys %$props) {
3346                 next if $props->{$prop} ne '1';
3347                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3348                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3349                 # We only keep those with underscores, to match /proc/cpuinfo
3350                 $prop =~ s/\.|-/_/g;
3351                 $flags->{$prop} = 1;
3352             }
3353         };
3354         my $err = $@;
3355
3356         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3357         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3358
3359         die $err if $err;
3360
3361         return [ sort keys %$flags ];
3362     };
3363
3364     # We need to query QEMU twice, since KVM and TCG have different supported flags
3365     PVE::QemuConfig->lock_config($fakevmid, sub {
3366         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3367         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3368
3369         if ($kvm_supported) {
3370             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3371             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3372         }
3373     });
3374
3375     return $flags;
3376 }
3377
3378 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3379 my $understood_cpu_flag_dir = "/usr/share/kvm";
3380 sub query_understood_cpu_flags {
3381     my $arch = get_host_arch();
3382     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3383
3384     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3385         if ! -e $filepath;
3386
3387     my $raw = file_get_contents($filepath);
3388     $raw =~ s/^\s+|\s+$//g;
3389     my @flags = split(/\s+/, $raw);
3390
3391     return \@flags;
3392 }
3393
3394 my sub get_cpuunits {
3395     my ($conf) = @_;
3396     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3397
3398     my $cpuunits = $conf->{cpuunits};
3399     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3400
3401     if ($is_cgroupv2) {
3402         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3403     } else {
3404         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3405     }
3406     return $cpuunits;
3407 }
3408
3409 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3410 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3411 my sub should_disable_smm {
3412     my ($conf, $vga) = @_;
3413
3414     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3415         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3416 }
3417
3418 sub config_to_command {
3419     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3420         $pbs_backing) = @_;
3421
3422     my $cmd = [];
3423     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3424     my $devices = [];
3425     my $bridges = {};
3426     my $ostype = $conf->{ostype};
3427     my $winversion = windows_version($ostype);
3428     my $kvm = $conf->{kvm};
3429     my $nodename = nodename();
3430
3431     my $arch = get_vm_arch($conf);
3432     my $kvm_binary = get_command_for_arch($arch);
3433     my $kvmver = kvm_user_version($kvm_binary);
3434
3435     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3436         $kvmver //= "undefined";
3437         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3438     }
3439
3440     my $add_pve_version = min_version($kvmver, 4, 1);
3441
3442     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3443     my $machine_version = extract_version($machine_type, $kvmver);
3444     $kvm //= 1 if is_native($arch);
3445
3446     $machine_version =~ m/(\d+)\.(\d+)/;
3447     my ($machine_major, $machine_minor) = ($1, $2);
3448
3449     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3450         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3451     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3452         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3453             ." please upgrade node '$nodename'\n"
3454     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3455         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3456         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3457             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3458             ." node '$nodename'\n";
3459     }
3460
3461     # if a specific +pve version is required for a feature, use $version_guard
3462     # instead of min_version to allow machines to be run with the minimum
3463     # required version
3464     my $required_pve_version = 0;
3465     my $version_guard = sub {
3466         my ($major, $minor, $pve) = @_;
3467         return 0 if !min_version($machine_version, $major, $minor, $pve);
3468         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3469         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3470         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3471         return 1;
3472     };
3473
3474     if ($kvm && !defined kvm_version()) {
3475         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3476             ." or enable in BIOS.\n";
3477     }
3478
3479     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3480     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3481     my $use_old_bios_files = undef;
3482     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3483
3484     push @$cmd, $kvm_binary;
3485
3486     push @$cmd, '-id', $vmid;
3487
3488     my $vmname = $conf->{name} || "vm$vmid";
3489
3490     push @$cmd, '-name', $vmname;
3491
3492     push @$cmd, '-no-shutdown';
3493
3494     my $use_virtio = 0;
3495
3496     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3497     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3498     push @$cmd, '-mon', "chardev=qmp,mode=control";
3499
3500     if (min_version($machine_version, 2, 12)) {
3501         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3502         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3503     }
3504
3505     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3506
3507     push @$cmd, '-daemonize';
3508
3509     if ($conf->{smbios1}) {
3510         my $smbios_conf = parse_smbios1($conf->{smbios1});
3511         if ($smbios_conf->{base64}) {
3512             # Do not pass base64 flag to qemu
3513             delete $smbios_conf->{base64};
3514             my $smbios_string = "";
3515             foreach my $key (keys %$smbios_conf) {
3516                 my $value;
3517                 if ($key eq "uuid") {
3518                     $value = $smbios_conf->{uuid}
3519                 } else {
3520                     $value = decode_base64($smbios_conf->{$key});
3521                 }
3522                 # qemu accepts any binary data, only commas need escaping by double comma
3523                 $value =~ s/,/,,/g;
3524                 $smbios_string .= "," . $key . "=" . $value if $value;
3525             }
3526             push @$cmd, '-smbios', "type=1" . $smbios_string;
3527         } else {
3528             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3529         }
3530     }
3531
3532     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3533         my $d;
3534         if (my $efidisk = $conf->{efidisk0}) {
3535             $d = parse_drive('efidisk0', $efidisk);
3536         }
3537
3538         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3539         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3540
3541         my ($path, $format);
3542         my $read_only_str = '';
3543         if ($d) {
3544             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3545             $format = $d->{format};
3546             if ($storeid) {
3547                 $path = PVE::Storage::path($storecfg, $d->{file});
3548                 if (!defined($format)) {
3549                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3550                     $format = qemu_img_format($scfg, $volname);
3551                 }
3552             } else {
3553                 $path = $d->{file};
3554                 die "efidisk format must be specified\n"
3555                     if !defined($format);
3556             }
3557
3558             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3559         } else {
3560             warn "no efidisk configured! Using temporary efivars disk.\n";
3561             $path = "/tmp/$vmid-ovmf.fd";
3562             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3563             $format = 'raw';
3564         }
3565
3566         my $size_str = "";
3567
3568         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3569             $size_str = ",size=" . (-s $ovmf_vars);
3570         }
3571
3572         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3573         my $cache = "";
3574         if ($path =~ m/^rbd:/) {
3575                 $cache = ',cache=writeback';
3576                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3577         }
3578
3579         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3580         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3581     }
3582
3583     if ($q35) { # tell QEMU to load q35 config early
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3593         push @$cmd, $fixups->@*;
3594     }
3595
3596     if ($conf->{vmgenid}) {
3597         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3598     }
3599
3600     # add usb controllers
3601     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3602         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3603     push @$devices, @usbcontrollers if @usbcontrollers;
3604     my $vga = parse_vga($conf->{vga});
3605
3606     my $qxlnum = vga_conf_has_spice($conf->{vga});
3607     $vga->{type} = 'qxl' if $qxlnum;
3608
3609     if (!$vga->{type}) {
3610         if ($arch eq 'aarch64') {
3611             $vga->{type} = 'virtio';
3612         } elsif (min_version($machine_version, 2, 9)) {
3613             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3614         } else {
3615             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3616         }
3617     }
3618
3619     # enable absolute mouse coordinates (needed by vnc)
3620     my $tablet = $conf->{tablet};
3621     if (!defined($tablet)) {
3622         $tablet = $defaults->{tablet};
3623         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3624         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3625     }
3626
3627     if ($tablet) {
3628         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3629         my $kbd = print_keyboarddevice_full($conf, $arch);
3630         push @$devices, '-device', $kbd if defined($kbd);
3631     }
3632
3633     my $bootorder = device_bootorder($conf);
3634
3635     # host pci device passthrough
3636     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3637         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3638
3639     # usb devices
3640     my $usb_dev_features = {};
3641     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3642
3643     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3644         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3645     push @$devices, @usbdevices if @usbdevices;
3646
3647     # serial devices
3648     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3649         my $path = $conf->{"serial$i"} or next;
3650         if ($path eq 'socket') {
3651             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3652             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3653             # On aarch64, serial0 is the UART device. Qemu only allows
3654             # connecting UART devices via the '-serial' command line, as
3655             # the device has a fixed slot on the hardware...
3656             if ($arch eq 'aarch64' && $i == 0) {
3657                 push @$devices, '-serial', "chardev:serial$i";
3658             } else {
3659                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3660             }
3661         } else {
3662             die "no such serial device\n" if ! -c $path;
3663             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3664             push @$devices, '-device', "isa-serial,chardev=serial$i";
3665         }
3666     }
3667
3668     # parallel devices
3669     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3670         if (my $path = $conf->{"parallel$i"}) {
3671             die "no such parallel device\n" if ! -c $path;
3672             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3673             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3674             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3675         }
3676     }
3677
3678     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3679         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3680         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3681         push @$devices, @$audio_devs;
3682     }
3683
3684     add_tpm_device($vmid, $devices, $conf);
3685
3686     my $sockets = 1;
3687     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3688     $sockets = $conf->{sockets} if  $conf->{sockets};
3689
3690     my $cores = $conf->{cores} || 1;
3691
3692     my $maxcpus = $sockets * $cores;
3693
3694     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3695
3696     my $allowed_vcpus = $cpuinfo->{cpus};
3697
3698     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3699
3700     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3701         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3702         for (my $i = 2; $i <= $vcpus; $i++)  {
3703             my $cpustr = print_cpu_device($conf,$i);
3704             push @$cmd, '-device', $cpustr;
3705         }
3706
3707     } else {
3708
3709         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3710     }
3711     push @$cmd, '-nodefaults';
3712
3713     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3714
3715     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3716
3717     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3718
3719     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3720         push @$devices, '-device', print_vga_device(
3721             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3722         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3723         push @$cmd,  '-vnc', "unix:$socket,password=on";
3724     } else {
3725         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3726         push @$cmd, '-nographic';
3727     }
3728
3729     # time drift fix
3730     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3731     my $useLocaltime = $conf->{localtime};
3732
3733     if ($winversion >= 5) { # windows
3734         $useLocaltime = 1 if !defined($conf->{localtime});
3735
3736         # use time drift fix when acpi is enabled
3737         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3738             $tdf = 1 if !defined($conf->{tdf});
3739         }
3740     }
3741
3742     if ($winversion >= 6) {
3743         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3744         push @$cmd, '-no-hpet';
3745     }
3746
3747     push @$rtcFlags, 'driftfix=slew' if $tdf;
3748
3749     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3750         push @$rtcFlags, "base=$conf->{startdate}";
3751     } elsif ($useLocaltime) {
3752         push @$rtcFlags, 'base=localtime';
3753     }
3754
3755     if ($forcecpu) {
3756         push @$cmd, '-cpu', $forcecpu;
3757     } else {
3758         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3759     }
3760
3761     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3762
3763     push @$cmd, '-S' if $conf->{freeze};
3764
3765     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3766
3767     my $guest_agent = parse_guest_agent($conf);
3768
3769     if ($guest_agent->{enabled}) {
3770         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3771         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3772
3773         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3774             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3775             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3776             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3777         } elsif ($guest_agent->{type} eq 'isa') {
3778             push @$devices, '-device', "isa-serial,chardev=qga0";
3779         }
3780     }
3781
3782     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3783     if ($rng && $version_guard->(4, 1, 2)) {
3784         check_rng_source($rng->{source});
3785
3786         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3787         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3788         my $limiter_str = "";
3789         if ($max_bytes) {
3790             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3791         }
3792
3793         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3794         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3795         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3796     }
3797
3798     my $spice_port;
3799
3800     if ($qxlnum) {
3801         if ($qxlnum > 1) {
3802             if ($winversion){
3803                 for (my $i = 1; $i < $qxlnum; $i++){
3804                     push @$devices, '-device', print_vga_device(
3805                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3806                 }
3807             } else {
3808                 # assume other OS works like Linux
3809                 my ($ram, $vram) = ("134217728", "67108864");
3810                 if ($vga->{memory}) {
3811                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3812                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3813                 }
3814                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3815                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3816             }
3817         }
3818
3819         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3820
3821         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3822         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3823         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3824
3825         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3826         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3827         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3828
3829         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3830         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3831
3832         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3833         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3834         if ($spice_enhancement->{foldersharing}) {
3835             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3836             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3837         }
3838
3839         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3840         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3841             if $spice_enhancement->{videostreaming};
3842
3843         push @$devices, '-spice', "$spice_opts";
3844     }
3845
3846     # enable balloon by default, unless explicitly disabled
3847     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3848         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3849         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3850     }
3851
3852     if ($conf->{watchdog}) {
3853         my $wdopts = parse_watchdog($conf->{watchdog});
3854         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3855         my $watchdog = $wdopts->{model} || 'i6300esb';
3856         push @$devices, '-device', "$watchdog$pciaddr";
3857         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3858     }
3859
3860     my $vollist = [];
3861     my $scsicontroller = {};
3862     my $ahcicontroller = {};
3863     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3864
3865     # Add iscsi initiator name if available
3866     if (my $initiator = get_initiator_name()) {
3867         push @$devices, '-iscsi', "initiator-name=$initiator";
3868     }
3869
3870     PVE::QemuConfig->foreach_volume($conf, sub {
3871         my ($ds, $drive) = @_;
3872
3873         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3874             check_volume_storage_type($storecfg, $drive->{file});
3875             push @$vollist, $drive->{file};
3876         }
3877
3878         # ignore efidisk here, already added in bios/fw handling code above
3879         return if $drive->{interface} eq 'efidisk';
3880         # similar for TPM
3881         return if $drive->{interface} eq 'tpmstate';
3882
3883         $use_virtio = 1 if $ds =~ m/^virtio/;
3884
3885         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3886
3887         if ($drive->{interface} eq 'virtio'){
3888            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3889         }
3890
3891         if ($drive->{interface} eq 'scsi') {
3892
3893             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3894
3895             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3896                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3897
3898             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3899             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3900
3901             my $iothread = '';
3902             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3903                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3904                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3905             } elsif ($drive->{iothread}) {
3906                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3907             }
3908
3909             my $queues = '';
3910             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3911                 $queues = ",num_queues=$drive->{queues}";
3912             }
3913
3914             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3915                 if !$scsicontroller->{$controller};
3916             $scsicontroller->{$controller}=1;
3917         }
3918
3919         if ($drive->{interface} eq 'sata') {
3920             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3921             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3922             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3923                 if !$ahcicontroller->{$controller};
3924             $ahcicontroller->{$controller}=1;
3925         }
3926
3927         my $pbs_conf = $pbs_backing->{$ds};
3928         my $pbs_name = undef;
3929         if ($pbs_conf) {
3930             $pbs_name = "drive-$ds-pbs";
3931             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3932         }
3933
3934         my $drive_cmd = print_drive_commandline_full(
3935             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3936
3937         # extra protection for templates, but SATA and IDE don't support it..
3938         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3939
3940         push @$devices, '-drive',$drive_cmd;
3941         push @$devices, '-device', print_drivedevice_full(
3942             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3943     });
3944
3945     for (my $i = 0; $i < $MAX_NETS; $i++) {
3946         my $netname = "net$i";
3947
3948         next if !$conf->{$netname};
3949         my $d = parse_net($conf->{$netname});
3950         next if !$d;
3951
3952         $use_virtio = 1 if $d->{model} eq 'virtio';
3953
3954         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3955
3956         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3957         push @$devices, '-netdev', $netdevfull;
3958
3959         my $netdevicefull = print_netdevice_full(
3960             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3961
3962         push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3980             .",size=$ivshmem->{size}M";
3981     }
3982
3983     # pci.4 is nested in pci.1
3984     $bridges->{1} = 1 if $bridges->{4};
3985
3986     if (!$q35) { # add pci bridges
3987         if (min_version($machine_version, 2, 3)) {
3988            $bridges->{1} = 1;
3989            $bridges->{2} = 1;
3990         }
3991         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3992     }
3993
3994     for my $k (sort {$b cmp $a} keys %$bridges) {
3995         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3996
3997         my $k_name = $k;
3998         if ($k == 2 && $legacy_igd) {
3999             $k_name = "$k-igd";
4000         }
4001         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4002         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4003
4004         if ($q35) { # add after -readconfig pve-q35.cfg
4005             splice @$devices, 2, 0, '-device', $devstr;
4006         } else {
4007             unshift @$devices, '-device', $devstr if $k > 0;
4008         }
4009     }
4010
4011     if (!$kvm) {
4012         push @$machineFlags, 'accel=tcg';
4013     }
4014
4015     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4016
4017     my $machine_type_min = $machine_type;
4018     if ($add_pve_version) {
4019         $machine_type_min =~ s/\+pve\d+$//;
4020         $machine_type_min .= "+pve$required_pve_version";
4021     }
4022     push @$machineFlags, "type=${machine_type_min}";
4023
4024     push @$cmd, @$devices;
4025     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4026     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4027     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4028
4029     if (my $vmstate = $conf->{vmstate}) {
4030         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4031         push @$vollist, $vmstate;
4032         push @$cmd, '-loadstate', $statepath;
4033         print "activating and using '$vmstate' as vmstate\n";
4034     }
4035
4036     if (PVE::QemuConfig->is_template($conf)) {
4037         # needed to workaround base volumes being read-only
4038         push @$cmd, '-snapshot';
4039     }
4040
4041     # add custom args
4042     if ($conf->{args}) {
4043         my $aa = PVE::Tools::split_args($conf->{args});
4044         push @$cmd, @$aa;
4045     }
4046
4047     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4048 }
4049
4050 sub check_rng_source {
4051     my ($source) = @_;
4052
4053     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4054     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4055         if ! -e $source;
4056
4057     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4058     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4059         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4060         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4061         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4062             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4063             ." to the host.\n";
4064     }
4065 }
4066
4067 sub spice_port {
4068     my ($vmid) = @_;
4069
4070     my $res = mon_cmd($vmid, 'query-spice');
4071
4072     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4073 }
4074
4075 sub vm_devices_list {
4076     my ($vmid) = @_;
4077
4078     my $res = mon_cmd($vmid, 'query-pci');
4079     my $devices_to_check = [];
4080     my $devices = {};
4081     foreach my $pcibus (@$res) {
4082         push @$devices_to_check, @{$pcibus->{devices}},
4083     }
4084
4085     while (@$devices_to_check) {
4086         my $to_check = [];
4087         for my $d (@$devices_to_check) {
4088             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4089             next if !$d->{'pci_bridge'};
4090
4091             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4092             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4093         }
4094         $devices_to_check = $to_check;
4095     }
4096
4097     my $resblock = mon_cmd($vmid, 'query-block');
4098     foreach my $block (@$resblock) {
4099         if($block->{device} =~ m/^drive-(\S+)/){
4100                 $devices->{$1} = 1;
4101         }
4102     }
4103
4104     my $resmice = mon_cmd($vmid, 'query-mice');
4105     foreach my $mice (@$resmice) {
4106         if ($mice->{name} eq 'QEMU HID Tablet') {
4107             $devices->{tablet} = 1;
4108             last;
4109         }
4110     }
4111
4112     # for usb devices there is no query-usb
4113     # but we can iterate over the entries in
4114     # qom-list path=/machine/peripheral
4115     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4116     foreach my $per (@$resperipheral) {
4117         if ($per->{name} =~ m/^usb\d+$/) {
4118             $devices->{$per->{name}} = 1;
4119         }
4120     }
4121
4122     return $devices;
4123 }
4124
4125 sub vm_deviceplug {
4126     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4127
4128     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4129
4130     my $devices_list = vm_devices_list($vmid);
4131     return 1 if defined($devices_list->{$deviceid});
4132
4133     # add PCI bridge if we need it for the device
4134     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4135
4136     if ($deviceid eq 'tablet') {
4137         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4138     } elsif ($deviceid eq 'keyboard') {
4139         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4140     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4141         die "usb hotplug currently not reliable\n";
4142         # since we can't reliably hot unplug all added usb devices and usb
4143         # passthrough breaks live migration we disable usb hotplugging for now
4144         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4145     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4146         qemu_iothread_add($vmid, $deviceid, $device);
4147
4148         qemu_driveadd($storecfg, $vmid, $device);
4149         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4150
4151         qemu_deviceadd($vmid, $devicefull);
4152         eval { qemu_deviceaddverify($vmid, $deviceid); };
4153         if (my $err = $@) {
4154             eval { qemu_drivedel($vmid, $deviceid); };
4155             warn $@ if $@;
4156             die $err;
4157         }
4158     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4160         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4161         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4162
4163         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4164
4165         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4166             qemu_iothread_add($vmid, $deviceid, $device);
4167             $devicefull .= ",iothread=iothread-$deviceid";
4168         }
4169
4170         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4171             $devicefull .= ",num_queues=$device->{queues}";
4172         }
4173
4174         qemu_deviceadd($vmid, $devicefull);
4175         qemu_deviceaddverify($vmid, $deviceid);
4176     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4177         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4178         qemu_driveadd($storecfg, $vmid, $device);
4179
4180         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4181         eval { qemu_deviceadd($vmid, $devicefull); };
4182         if (my $err = $@) {
4183             eval { qemu_drivedel($vmid, $deviceid); };
4184             warn $@ if $@;
4185             die $err;
4186         }
4187     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4188         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4189
4190         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4191         my $use_old_bios_files = undef;
4192         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4193
4194         my $netdevicefull = print_netdevice_full(
4195             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4196         qemu_deviceadd($vmid, $netdevicefull);
4197         eval {
4198             qemu_deviceaddverify($vmid, $deviceid);
4199             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4200         };
4201         if (my $err = $@) {
4202             eval { qemu_netdevdel($vmid, $deviceid); };
4203             warn $@ if $@;
4204             die $err;
4205         }
4206     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4207         my $bridgeid = $2;
4208         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4209         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4210
4211         qemu_deviceadd($vmid, $devicefull);
4212         qemu_deviceaddverify($vmid, $deviceid);
4213     } else {
4214         die "can't hotplug device '$deviceid'\n";
4215     }
4216
4217     return 1;
4218 }
4219
4220 # fixme: this should raise exceptions on error!
4221 sub vm_deviceunplug {
4222     my ($vmid, $conf, $deviceid) = @_;
4223
4224     my $devices_list = vm_devices_list($vmid);
4225     return 1 if !defined($devices_list->{$deviceid});
4226
4227     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4228     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4229
4230     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4231         qemu_devicedel($vmid, $deviceid);
4232     } elsif ($deviceid =~ m/^usb\d+$/) {
4233         die "usb hotplug currently not reliable\n";
4234         # when unplugging usb devices this way, there may be remaining usb
4235         # controllers/hubs so we disable it for now
4236         #qemu_devicedel($vmid, $deviceid);
4237         #qemu_devicedelverify($vmid, $deviceid);
4238     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4239         my $device = parse_drive($deviceid, $conf->{$deviceid});
4240
4241         qemu_devicedel($vmid, $deviceid);
4242         qemu_devicedelverify($vmid, $deviceid);
4243         qemu_drivedel($vmid, $deviceid);
4244         qemu_iothread_del($vmid, $deviceid, $device);
4245     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4246         qemu_devicedel($vmid, $deviceid);
4247         qemu_devicedelverify($vmid, $deviceid);
4248     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4249         my $device = parse_drive($deviceid, $conf->{$deviceid});
4250
4251         qemu_devicedel($vmid, $deviceid);
4252         qemu_devicedelverify($vmid, $deviceid);
4253         qemu_drivedel($vmid, $deviceid);
4254         qemu_deletescsihw($conf, $vmid, $deviceid);
4255
4256         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4257             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4258     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4259         qemu_devicedel($vmid, $deviceid);
4260         qemu_devicedelverify($vmid, $deviceid);
4261         qemu_netdevdel($vmid, $deviceid);
4262     } else {
4263         die "can't unplug device '$deviceid'\n";
4264     }
4265
4266     return 1;
4267 }
4268
4269 sub qemu_deviceadd {
4270     my ($vmid, $devicefull) = @_;
4271
4272     $devicefull = "driver=".$devicefull;
4273     my %options =  split(/[=,]/, $devicefull);
4274
4275     mon_cmd($vmid, "device_add" , %options);
4276 }
4277
4278 sub qemu_devicedel {
4279     my ($vmid, $deviceid) = @_;
4280
4281     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4282 }
4283
4284 sub qemu_iothread_add {
4285     my ($vmid, $deviceid, $device) = @_;
4286
4287     if ($device->{iothread}) {
4288         my $iothreads = vm_iothreads_list($vmid);
4289         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4290     }
4291 }
4292
4293 sub qemu_iothread_del {
4294     my ($vmid, $deviceid, $device) = @_;
4295
4296     if ($device->{iothread}) {
4297         my $iothreads = vm_iothreads_list($vmid);
4298         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4299     }
4300 }
4301
4302 sub qemu_objectadd {
4303     my ($vmid, $objectid, $qomtype) = @_;
4304
4305     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4306
4307     return 1;
4308 }
4309
4310 sub qemu_objectdel {
4311     my ($vmid, $objectid) = @_;
4312
4313     mon_cmd($vmid, "object-del", id => $objectid);
4314
4315     return 1;
4316 }
4317
4318 sub qemu_driveadd {
4319     my ($storecfg, $vmid, $device) = @_;
4320
4321     my $kvmver = get_running_qemu_version($vmid);
4322     my $io_uring = min_version($kvmver, 6, 0);
4323     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4324     $drive =~ s/\\/\\\\/g;
4325     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4326
4327     # If the command succeeds qemu prints: "OK"
4328     return 1 if $ret =~ m/OK/s;
4329
4330     die "adding drive failed: $ret\n";
4331 }
4332
4333 sub qemu_drivedel {
4334     my ($vmid, $deviceid) = @_;
4335
4336     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4337     $ret =~ s/^\s+//;
4338
4339     return 1 if $ret eq "";
4340
4341     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4342     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4343
4344     die "deleting drive $deviceid failed : $ret\n";
4345 }
4346
4347 sub qemu_deviceaddverify {
4348     my ($vmid, $deviceid) = @_;
4349
4350     for (my $i = 0; $i <= 5; $i++) {
4351          my $devices_list = vm_devices_list($vmid);
4352          return 1 if defined($devices_list->{$deviceid});
4353          sleep 1;
4354     }
4355
4356     die "error on hotplug device '$deviceid'\n";
4357 }
4358
4359
4360 sub qemu_devicedelverify {
4361     my ($vmid, $deviceid) = @_;
4362
4363     # need to verify that the device is correctly removed as device_del
4364     # is async and empty return is not reliable
4365
4366     for (my $i = 0; $i <= 5; $i++) {
4367          my $devices_list = vm_devices_list($vmid);
4368          return 1 if !defined($devices_list->{$deviceid});
4369          sleep 1;
4370     }
4371
4372     die "error on hot-unplugging device '$deviceid'\n";
4373 }
4374
4375 sub qemu_findorcreatescsihw {
4376     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4377
4378     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4379
4380     my $scsihwid="$controller_prefix$controller";
4381     my $devices_list = vm_devices_list($vmid);
4382
4383     if (!defined($devices_list->{$scsihwid})) {
4384         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4385     }
4386
4387     return 1;
4388 }
4389
4390 sub qemu_deletescsihw {
4391     my ($conf, $vmid, $opt) = @_;
4392
4393     my $device = parse_drive($opt, $conf->{$opt});
4394
4395     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4396         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4397         return 1;
4398     }
4399
4400     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4401
4402     my $devices_list = vm_devices_list($vmid);
4403     foreach my $opt (keys %{$devices_list}) {
4404         if (is_valid_drivename($opt)) {
4405             my $drive = parse_drive($opt, $conf->{$opt});
4406             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4407                 return 1;
4408             }
4409         }
4410     }
4411
4412     my $scsihwid="scsihw$controller";
4413
4414     vm_deviceunplug($vmid, $conf, $scsihwid);
4415
4416     return 1;
4417 }
4418
4419 sub qemu_add_pci_bridge {
4420     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4421
4422     my $bridges = {};
4423
4424     my $bridgeid;
4425
4426     print_pci_addr($device, $bridges, $arch, $machine_type);
4427
4428     while (my ($k, $v) = each %$bridges) {
4429         $bridgeid = $k;
4430     }
4431     return 1 if !defined($bridgeid) || $bridgeid < 1;
4432
4433     my $bridge = "pci.$bridgeid";
4434     my $devices_list = vm_devices_list($vmid);
4435
4436     if (!defined($devices_list->{$bridge})) {
4437         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4438     }
4439
4440     return 1;
4441 }
4442
4443 sub qemu_set_link_status {
4444     my ($vmid, $device, $up) = @_;
4445
4446     mon_cmd($vmid, "set_link", name => $device,
4447                up => $up ? JSON::true : JSON::false);
4448 }
4449
4450 sub qemu_netdevadd {
4451     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4452
4453     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4454     my %options =  split(/[=,]/, $netdev);
4455
4456     if (defined(my $vhost = $options{vhost})) {
4457         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4458     }
4459
4460     if (defined(my $queues = $options{queues})) {
4461         $options{queues} = $queues + 0;
4462     }
4463
4464     mon_cmd($vmid, "netdev_add",  %options);
4465     return 1;
4466 }
4467
4468 sub qemu_netdevdel {
4469     my ($vmid, $deviceid) = @_;
4470
4471     mon_cmd($vmid, "netdev_del", id => $deviceid);
4472 }
4473
4474 sub qemu_usb_hotplug {
4475     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4476
4477     return if !$device;
4478
4479     # remove the old one first
4480     vm_deviceunplug($vmid, $conf, $deviceid);
4481
4482     # check if xhci controller is necessary and available
4483     if ($device->{usb3}) {
4484
4485         my $devicelist = vm_devices_list($vmid);
4486
4487         if (!$devicelist->{xhci}) {
4488             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4489             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4490         }
4491     }
4492     my $d = parse_usb_device($device->{host});
4493     $d->{usb3} = $device->{usb3};
4494
4495     # add the new one
4496     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4497 }
4498
4499 sub qemu_cpu_hotplug {
4500     my ($vmid, $conf, $vcpus) = @_;
4501
4502     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4503
4504     my $sockets = 1;
4505     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4506     $sockets = $conf->{sockets} if  $conf->{sockets};
4507     my $cores = $conf->{cores} || 1;
4508     my $maxcpus = $sockets * $cores;
4509
4510     $vcpus = $maxcpus if !$vcpus;
4511
4512     die "you can't add more vcpus than maxcpus\n"
4513         if $vcpus > $maxcpus;
4514
4515     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4516
4517     if ($vcpus < $currentvcpus) {
4518
4519         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4520
4521             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4522                 qemu_devicedel($vmid, "cpu$i");
4523                 my $retry = 0;
4524                 my $currentrunningvcpus = undef;
4525                 while (1) {
4526                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4527                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4528                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4529                     $retry++;
4530                     sleep 1;
4531                 }
4532                 #update conf after each succesfull cpu unplug
4533                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4534                 PVE::QemuConfig->write_config($vmid, $conf);
4535             }
4536         } else {
4537             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4538         }
4539
4540         return;
4541     }
4542
4543     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4544     die "vcpus in running vm does not match its configuration\n"
4545         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4546
4547     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4548
4549         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4550             my $cpustr = print_cpu_device($conf, $i);
4551             qemu_deviceadd($vmid, $cpustr);
4552
4553             my $retry = 0;
4554             my $currentrunningvcpus = undef;
4555             while (1) {
4556                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4557                 last if scalar(@{$currentrunningvcpus}) == $i;
4558                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4559                 sleep 1;
4560                 $retry++;
4561             }
4562             #update conf after each succesfull cpu hotplug
4563             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4564             PVE::QemuConfig->write_config($vmid, $conf);
4565         }
4566     } else {
4567
4568         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4569             mon_cmd($vmid, "cpu-add", id => int($i));
4570         }
4571     }
4572 }
4573
4574 sub qemu_block_set_io_throttle {
4575     my ($vmid, $deviceid,
4576         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4577         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4578         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4579         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4580
4581     return if !check_running($vmid) ;
4582
4583     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4584         bps => int($bps),
4585         bps_rd => int($bps_rd),
4586         bps_wr => int($bps_wr),
4587         iops => int($iops),
4588         iops_rd => int($iops_rd),
4589         iops_wr => int($iops_wr),
4590         bps_max => int($bps_max),
4591         bps_rd_max => int($bps_rd_max),
4592         bps_wr_max => int($bps_wr_max),
4593         iops_max => int($iops_max),
4594         iops_rd_max => int($iops_rd_max),
4595         iops_wr_max => int($iops_wr_max),
4596         bps_max_length => int($bps_max_length),
4597         bps_rd_max_length => int($bps_rd_max_length),
4598         bps_wr_max_length => int($bps_wr_max_length),
4599         iops_max_length => int($iops_max_length),
4600         iops_rd_max_length => int($iops_rd_max_length),
4601         iops_wr_max_length => int($iops_wr_max_length),
4602     );
4603
4604 }
4605
4606 sub qemu_block_resize {
4607     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4608
4609     my $running = check_running($vmid);
4610
4611     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4612
4613     return if !$running;
4614
4615     my $padding = (1024 - $size % 1024) % 1024;
4616     $size = $size + $padding;
4617
4618     mon_cmd(
4619         $vmid,
4620         "block_resize",
4621         device => $deviceid,
4622         size => int($size),
4623         timeout => 60,
4624     );
4625 }
4626
4627 sub qemu_volume_snapshot {
4628     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4629
4630     my $running = check_running($vmid);
4631
4632     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4633         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4634     } else {
4635         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4636     }
4637 }
4638
4639 sub qemu_volume_snapshot_delete {
4640     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4641
4642     my $running = check_running($vmid);
4643
4644     if($running) {
4645
4646         $running = undef;
4647         my $conf = PVE::QemuConfig->load_config($vmid);
4648         PVE::QemuConfig->foreach_volume($conf, sub {
4649             my ($ds, $drive) = @_;
4650             $running = 1 if $drive->{file} eq $volid;
4651         });
4652     }
4653
4654     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4655         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4656     } else {
4657         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4658     }
4659 }
4660
4661 sub set_migration_caps {
4662     my ($vmid, $savevm) = @_;
4663
4664     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4665
4666     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4667     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4668
4669     my $cap_ref = [];
4670
4671     my $enabled_cap = {
4672         "auto-converge" => 1,
4673         "xbzrle" => 1,
4674         "x-rdma-pin-all" => 0,
4675         "zero-blocks" => 0,
4676         "compress" => 0,
4677         "dirty-bitmaps" => $dirty_bitmaps,
4678     };
4679
4680     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4681
4682     for my $supported_capability (@$supported_capabilities) {
4683         push @$cap_ref, {
4684             capability => $supported_capability->{capability},
4685             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4686         };
4687     }
4688
4689     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4690 }
4691
4692 sub foreach_volid {
4693     my ($conf, $func, @param) = @_;
4694
4695     my $volhash = {};
4696
4697     my $test_volid = sub {
4698         my ($key, $drive, $snapname) = @_;
4699
4700         my $volid = $drive->{file};
4701         return if !$volid;
4702
4703         $volhash->{$volid}->{cdrom} //= 1;
4704         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4705
4706         my $replicate = $drive->{replicate} // 1;
4707         $volhash->{$volid}->{replicate} //= 0;
4708         $volhash->{$volid}->{replicate} = 1 if $replicate;
4709
4710         $volhash->{$volid}->{shared} //= 0;
4711         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4712
4713         $volhash->{$volid}->{referenced_in_config} //= 0;
4714         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4715
4716         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4717             if defined($snapname);
4718
4719         my $size = $drive->{size};
4720         $volhash->{$volid}->{size} //= $size if $size;
4721
4722         $volhash->{$volid}->{is_vmstate} //= 0;
4723         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4724
4725         $volhash->{$volid}->{is_tpmstate} //= 0;
4726         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4727
4728         $volhash->{$volid}->{is_unused} //= 0;
4729         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4730
4731         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4732     };
4733
4734     my $include_opts = {
4735         extra_keys => ['vmstate'],
4736         include_unused => 1,
4737     };
4738
4739     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4740     foreach my $snapname (keys %{$conf->{snapshots}}) {
4741         my $snap = $conf->{snapshots}->{$snapname};
4742         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4743     }
4744
4745     foreach my $volid (keys %$volhash) {
4746         &$func($volid, $volhash->{$volid}, @param);
4747     }
4748 }
4749
4750 my $fast_plug_option = {
4751     'lock' => 1,
4752     'name' => 1,
4753     'onboot' => 1,
4754     'shares' => 1,
4755     'startup' => 1,
4756     'description' => 1,
4757     'protection' => 1,
4758     'vmstatestorage' => 1,
4759     'hookscript' => 1,
4760     'tags' => 1,
4761 };
4762
4763 # hotplug changes in [PENDING]
4764 # $selection hash can be used to only apply specified options, for
4765 # example: { cores => 1 } (only apply changed 'cores')
4766 # $errors ref is used to return error messages
4767 sub vmconfig_hotplug_pending {
4768     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4769
4770     my $defaults = load_defaults();
4771     my $arch = get_vm_arch($conf);
4772     my $machine_type = get_vm_machine($conf, undef, $arch);
4773
4774     # commit values which do not have any impact on running VM first
4775     # Note: those option cannot raise errors, we we do not care about
4776     # $selection and always apply them.
4777
4778     my $add_error = sub {
4779         my ($opt, $msg) = @_;
4780         $errors->{$opt} = "hotplug problem - $msg";
4781     };
4782
4783     my $changes = 0;
4784     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4785         if ($fast_plug_option->{$opt}) {
4786             $conf->{$opt} = $conf->{pending}->{$opt};
4787             delete $conf->{pending}->{$opt};
4788             $changes = 1;
4789         }
4790     }
4791
4792     if ($changes) {
4793         PVE::QemuConfig->write_config($vmid, $conf);
4794     }
4795
4796     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4797
4798     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4799     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4800     foreach my $opt (sort keys %$pending_delete_hash) {
4801         next if $selection && !$selection->{$opt};
4802         my $force = $pending_delete_hash->{$opt}->{force};
4803         eval {
4804             if ($opt eq 'hotplug') {
4805                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4806             } elsif ($opt eq 'tablet') {
4807                 die "skip\n" if !$hotplug_features->{usb};
4808                 if ($defaults->{tablet}) {
4809                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4810                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4811                         if $arch eq 'aarch64';
4812                 } else {
4813                     vm_deviceunplug($vmid, $conf, 'tablet');
4814                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4815                 }
4816             } elsif ($opt =~ m/^usb\d+/) {
4817                 die "skip\n";
4818                 # since we cannot reliably hot unplug usb devices we are disabling it
4819                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4820                 #vm_deviceunplug($vmid, $conf, $opt);
4821             } elsif ($opt eq 'vcpus') {
4822                 die "skip\n" if !$hotplug_features->{cpu};
4823                 qemu_cpu_hotplug($vmid, $conf, undef);
4824             } elsif ($opt eq 'balloon') {
4825                 # enable balloon device is not hotpluggable
4826                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4827                 # here we reset the ballooning value to memory
4828                 my $balloon = $conf->{memory} || $defaults->{memory};
4829                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4830             } elsif ($fast_plug_option->{$opt}) {
4831                 # do nothing
4832             } elsif ($opt =~ m/^net(\d+)$/) {
4833                 die "skip\n" if !$hotplug_features->{network};
4834                 vm_deviceunplug($vmid, $conf, $opt);
4835             } elsif (is_valid_drivename($opt)) {
4836                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4837                 vm_deviceunplug($vmid, $conf, $opt);
4838                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4839             } elsif ($opt =~ m/^memory$/) {
4840                 die "skip\n" if !$hotplug_features->{memory};
4841                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4842             } elsif ($opt eq 'cpuunits') {
4843                 $cgroup->change_cpu_shares(undef, 1024);
4844             } elsif ($opt eq 'cpulimit') {
4845                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4846             } else {
4847                 die "skip\n";
4848             }
4849         };
4850         if (my $err = $@) {
4851             &$add_error($opt, $err) if $err ne "skip\n";
4852         } else {
4853             delete $conf->{$opt};
4854             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4855         }
4856     }
4857
4858     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4859     $apply_pending_cloudinit = sub {
4860         return if $apply_pending_cloudinit_done; # once is enough
4861         $apply_pending_cloudinit_done = 1; # once is enough
4862
4863         my ($key, $value) = @_;
4864
4865         my @cloudinit_opts = keys %$confdesc_cloudinit;
4866         foreach my $opt (keys %{$conf->{pending}}) {
4867             next if !grep { $_ eq $opt } @cloudinit_opts;
4868             $conf->{$opt} = delete $conf->{pending}->{$opt};
4869         }
4870
4871         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4872         foreach my $opt (sort keys %$pending_delete_hash) {
4873             next if !grep { $_ eq $opt } @cloudinit_opts;
4874             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4875             delete $conf->{$opt};
4876         }
4877
4878         my $new_conf = { %$conf };
4879         $new_conf->{$key} = $value;
4880         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4881     };
4882
4883     foreach my $opt (keys %{$conf->{pending}}) {
4884         next if $selection && !$selection->{$opt};
4885         my $value = $conf->{pending}->{$opt};
4886         eval {
4887             if ($opt eq 'hotplug') {
4888                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4889             } elsif ($opt eq 'tablet') {
4890                 die "skip\n" if !$hotplug_features->{usb};
4891                 if ($value == 1) {
4892                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4893                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4894                         if $arch eq 'aarch64';
4895                 } elsif ($value == 0) {
4896                     vm_deviceunplug($vmid, $conf, 'tablet');
4897                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4898                 }
4899             } elsif ($opt =~ m/^usb\d+$/) {
4900                 die "skip\n";
4901                 # since we cannot reliably hot unplug usb devices we disable it for now
4902                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4903                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4904                 #die "skip\n" if !$d;
4905                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4906             } elsif ($opt eq 'vcpus') {
4907                 die "skip\n" if !$hotplug_features->{cpu};
4908                 qemu_cpu_hotplug($vmid, $conf, $value);
4909             } elsif ($opt eq 'balloon') {
4910                 # enable/disable balloning device is not hotpluggable
4911                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4912                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4913                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4914
4915                 # allow manual ballooning if shares is set to zero
4916                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4917                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4918                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4919                 }
4920             } elsif ($opt =~ m/^net(\d+)$/) {
4921                 # some changes can be done without hotplug
4922                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4923                                     $vmid, $opt, $value, $arch, $machine_type);
4924             } elsif (is_valid_drivename($opt)) {
4925                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4926                 # some changes can be done without hotplug
4927                 my $drive = parse_drive($opt, $value);
4928                 if (drive_is_cloudinit($drive)) {
4929                     &$apply_pending_cloudinit($opt, $value);
4930                 }
4931                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4932                                      $vmid, $opt, $value, $arch, $machine_type);
4933             } elsif ($opt =~ m/^memory$/) { #dimms
4934                 die "skip\n" if !$hotplug_features->{memory};
4935                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4936             } elsif ($opt eq 'cpuunits') {
4937                 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4938                 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4939             } elsif ($opt eq 'cpulimit') {
4940                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4941                 $cgroup->change_cpu_quota($cpulimit, 100000);
4942             } elsif ($opt eq 'agent') {
4943                 vmconfig_update_agent($conf, $opt, $value);
4944             } else {
4945                 die "skip\n";  # skip non-hot-pluggable options
4946             }
4947         };
4948         if (my $err = $@) {
4949             &$add_error($opt, $err) if $err ne "skip\n";
4950         } else {
4951             $conf->{$opt} = $value;
4952             delete $conf->{pending}->{$opt};
4953         }
4954     }
4955
4956     PVE::QemuConfig->write_config($vmid, $conf);
4957 }
4958
4959 sub try_deallocate_drive {
4960     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4961
4962     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4963         my $volid = $drive->{file};
4964         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4965             my $sid = PVE::Storage::parse_volume_id($volid);
4966             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4967
4968             # check if the disk is really unused
4969             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4970                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4971             PVE::Storage::vdisk_free($storecfg, $volid);
4972             return 1;
4973         } else {
4974             # If vm is not owner of this disk remove from config
4975             return 1;
4976         }
4977     }
4978
4979     return;
4980 }
4981
4982 sub vmconfig_delete_or_detach_drive {
4983     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4984
4985     my $drive = parse_drive($opt, $conf->{$opt});
4986
4987     my $rpcenv = PVE::RPCEnvironment::get();
4988     my $authuser = $rpcenv->get_user();
4989
4990     if ($force) {
4991         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4992         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4993     } else {
4994         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4995     }
4996 }
4997
4998
4999
5000 sub vmconfig_apply_pending {
5001     my ($vmid, $conf, $storecfg, $errors) = @_;
5002
5003     return if !scalar(keys %{$conf->{pending}});
5004
5005     my $add_apply_error = sub {
5006         my ($opt, $msg) = @_;
5007         my $err_msg = "unable to apply pending change $opt : $msg";
5008         $errors->{$opt} = $err_msg;
5009         warn $err_msg;
5010     };
5011
5012     # cold plug
5013
5014     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5015     foreach my $opt (sort keys %$pending_delete_hash) {
5016         my $force = $pending_delete_hash->{$opt}->{force};
5017         eval {
5018             if ($opt =~ m/^unused/) {
5019                 die "internal error";
5020             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5021                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5022             }
5023         };
5024         if (my $err = $@) {
5025             $add_apply_error->($opt, $err);
5026         } else {
5027             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5028             delete $conf->{$opt};
5029         }
5030     }
5031
5032     PVE::QemuConfig->cleanup_pending($conf);
5033
5034     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5035         next if $opt eq 'delete'; # just to be sure
5036         eval {
5037             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5038                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5039             }
5040         };
5041         if (my $err = $@) {
5042             $add_apply_error->($opt, $err);
5043         } else {
5044             $conf->{$opt} = delete $conf->{pending}->{$opt};
5045         }
5046     }
5047
5048     # write all changes at once to avoid unnecessary i/o
5049     PVE::QemuConfig->write_config($vmid, $conf);
5050 }
5051
5052 sub vmconfig_update_net {
5053     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5054
5055     my $newnet = parse_net($value);
5056
5057     if ($conf->{$opt}) {
5058         my $oldnet = parse_net($conf->{$opt});
5059
5060         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5061             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5062             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5063             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5064
5065             # for non online change, we try to hot-unplug
5066             die "skip\n" if !$hotplug;
5067             vm_deviceunplug($vmid, $conf, $opt);
5068         } else {
5069
5070             die "internal error" if $opt !~ m/net(\d+)/;
5071             my $iface = "tap${vmid}i$1";
5072
5073             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5074                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5075                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5076                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5077                 PVE::Network::tap_unplug($iface);
5078
5079                 if ($have_sdn) {
5080                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5081                 } else {
5082                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5083                 }
5084             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5085                 # Rate can be applied on its own but any change above needs to
5086                 # include the rate in tap_plug since OVS resets everything.
5087                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5088             }
5089
5090             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5091                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5092             }
5093
5094             return 1;
5095         }
5096     }
5097
5098     if ($hotplug) {
5099         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5100     } else {
5101         die "skip\n";
5102     }
5103 }
5104
5105 sub vmconfig_update_agent {
5106     my ($conf, $opt, $value) = @_;
5107
5108     die "skip\n" if !$conf->{$opt};
5109
5110     my $hotplug_options = { fstrim_cloned_disks => 1 };
5111
5112     my $old_agent = parse_guest_agent($conf);
5113     my $agent = parse_guest_agent({$opt => $value});
5114
5115     for my $option (keys %$agent) { # added/changed options
5116         next if defined($hotplug_options->{$option});
5117         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5118     }
5119
5120     for my $option (keys %$old_agent) { # removed options
5121         next if defined($hotplug_options->{$option});
5122         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5123     }
5124
5125     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5126 }
5127
5128 sub vmconfig_update_disk {
5129     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5130
5131     my $drive = parse_drive($opt, $value);
5132
5133     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5134         my $media = $drive->{media} || 'disk';
5135         my $oldmedia = $old_drive->{media} || 'disk';
5136         die "unable to change media type\n" if $media ne $oldmedia;
5137
5138         if (!drive_is_cdrom($old_drive)) {
5139
5140             if ($drive->{file} ne $old_drive->{file}) {
5141
5142                 die "skip\n" if !$hotplug;
5143
5144                 # unplug and register as unused
5145                 vm_deviceunplug($vmid, $conf, $opt);
5146                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5147
5148             } else {
5149                 # update existing disk
5150
5151                 # skip non hotpluggable value
5152                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5153                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5154                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5155                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5156                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5157                     die "skip\n";
5158                 }
5159
5160                 # apply throttle
5161                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5162                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5163                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5164                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5165                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5166                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5167                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5168                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5169                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5170                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5171                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5172                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5173                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5174                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5175                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5176                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5177                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5178                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5179
5180                     qemu_block_set_io_throttle(
5181                         $vmid,"drive-$opt",
5182                         ($drive->{mbps} || 0)*1024*1024,
5183                         ($drive->{mbps_rd} || 0)*1024*1024,
5184                         ($drive->{mbps_wr} || 0)*1024*1024,
5185                          $drive->{iops} || 0,
5186                          $drive->{iops_rd} || 0,
5187                          $drive->{iops_wr} || 0,
5188                         ($drive->{mbps_max} || 0)*1024*1024,
5189                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5190                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5191                          $drive->{iops_max} || 0,
5192                          $drive->{iops_rd_max} || 0,
5193                          $drive->{iops_wr_max} || 0,
5194                          $drive->{bps_max_length} || 1,
5195                          $drive->{bps_rd_max_length} || 1,
5196                          $drive->{bps_wr_max_length} || 1,
5197                          $drive->{iops_max_length} || 1,
5198                          $drive->{iops_rd_max_length} || 1,
5199                          $drive->{iops_wr_max_length} || 1,
5200                     );
5201
5202                 }
5203
5204                 return 1;
5205             }
5206
5207         } else { # cdrom
5208
5209             if ($drive->{file} eq 'none') {
5210                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5211                 if (drive_is_cloudinit($old_drive)) {
5212                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5213                 }
5214             } else {
5215                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5216
5217                 # force eject if locked
5218                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5219
5220                 if ($path) {
5221                     mon_cmd($vmid, "blockdev-change-medium",
5222                         id => "$opt", filename => "$path");
5223                 }
5224             }
5225
5226             return 1;
5227         }
5228     }
5229
5230     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5231     # hotplug new disks
5232     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5233     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5234 }
5235
5236 # called in locked context by incoming migration
5237 sub vm_migrate_get_nbd_disks {
5238     my ($storecfg, $conf, $replicated_volumes) = @_;
5239
5240     my $local_volumes = {};
5241     PVE::QemuConfig->foreach_volume($conf, sub {
5242         my ($ds, $drive) = @_;
5243
5244         return if drive_is_cdrom($drive);
5245         return if $ds eq 'tpmstate0';
5246
5247         my $volid = $drive->{file};
5248
5249         return if !$volid;
5250
5251         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5252
5253         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5254         return if $scfg->{shared};
5255
5256         # replicated disks re-use existing state via bitmap
5257         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5258         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5259     });
5260     return $local_volumes;
5261 }
5262
5263 # called in locked context by incoming migration
5264 sub vm_migrate_alloc_nbd_disks {
5265     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5266
5267     my $nbd = {};
5268     foreach my $opt (sort keys %$source_volumes) {
5269         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5270
5271         if ($use_existing) {
5272             $nbd->{$opt}->{drivestr} = print_drive($drive);
5273             $nbd->{$opt}->{volid} = $volid;
5274             $nbd->{$opt}->{replicated} = 1;
5275             next;
5276         }
5277
5278         # storage mapping + volname = regular migration
5279         # storage mapping + format = remote migration
5280         # order of precedence, filtered by whether storage supports it:
5281         # 1. explicit requested format
5282         # 2. format of current volume
5283         # 3. default format of storage
5284         if (!$storagemap->{identity}) {
5285             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5286             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5287             if (!$format || !grep { $format eq $_ } @$validFormats) {
5288                 if ($volname) {
5289                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5290                     my $fileFormat = qemu_img_format($scfg, $volname);
5291                     $format = $fileFormat
5292                         if grep { $fileFormat eq $_ } @$validFormats;
5293                 }
5294                 $format //= $defFormat;
5295             }
5296         } else {
5297             # can't happen for remote migration, so $volname is always defined
5298             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5299             $format = qemu_img_format($scfg, $volname);
5300         }
5301
5302         my $size = $drive->{size} / 1024;
5303         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5304         my $newdrive = $drive;
5305         $newdrive->{format} = $format;
5306         $newdrive->{file} = $newvolid;
5307         my $drivestr = print_drive($newdrive);
5308         $nbd->{$opt}->{drivestr} = $drivestr;
5309         $nbd->{$opt}->{volid} = $newvolid;
5310     }
5311
5312     return $nbd;
5313 }
5314
5315 # see vm_start_nolock for parameters, additionally:
5316 # migrate_opts:
5317 #   storagemap = parsed storage map for allocating NBD disks
5318 sub vm_start {
5319     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5320
5321     return PVE::QemuConfig->lock_config($vmid, sub {
5322         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5323
5324         die "you can't start a vm if it's a template\n"
5325             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5326
5327         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5328         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5329
5330         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5331
5332         if ($has_backup_lock && $running) {
5333             # a backup is currently running, attempt to start the guest in the
5334             # existing QEMU instance
5335             return vm_resume($vmid);
5336         }
5337
5338         PVE::QemuConfig->check_lock($conf)
5339             if !($params->{skiplock} || $has_suspended_lock);
5340
5341         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5342
5343         die "VM $vmid already running\n" if $running;
5344
5345         if (my $storagemap = $migrate_opts->{storagemap}) {
5346             my $replicated = $migrate_opts->{replicated_volumes};
5347             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5348             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5349
5350             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5351                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5352             }
5353         }
5354
5355         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5356     });
5357 }
5358
5359
5360 # params:
5361 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5362 #   skiplock => 0/1, skip checking for config lock
5363 #   skiptemplate => 0/1, skip checking whether VM is template
5364 #   forcemachine => to force Qemu machine (rollback/migration)
5365 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5366 #   timeout => in seconds
5367 #   paused => start VM in paused state (backup)
5368 #   resume => resume from hibernation
5369 #   pbs-backing => {
5370 #      sata0 => {
5371 #         repository
5372 #         snapshot
5373 #         keyfile
5374 #         archive
5375 #      },
5376 #      virtio2 => ...
5377 #   }
5378 # migrate_opts:
5379 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5380 #   migratedfrom => source node
5381 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5382 #   network => CIDR of migration network
5383 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5384 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5385 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5386 #   tpmstate_vol => new volid of tpmstate0, not yet contained in config
5387 sub vm_start_nolock {
5388     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5389
5390     my $statefile = $params->{statefile};
5391     my $resume = $params->{resume};
5392
5393     my $migratedfrom = $migrate_opts->{migratedfrom};
5394     my $migration_type = $migrate_opts->{type};
5395
5396     my $res = {};
5397
5398     # clean up leftover reboot request files
5399     eval { clear_reboot_request($vmid); };
5400     warn $@ if $@;
5401
5402     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5403         vmconfig_apply_pending($vmid, $conf, $storecfg);
5404         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5405     }
5406
5407     # don't regenerate the ISO if the VM is started as part of a live migration
5408     # this way we can reuse the old ISO with the correct config
5409     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5410
5411     # override TPM state vol if migrated, conf is out of date still
5412     if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
5413         my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
5414         $parsed->{file} = $tpmvol;
5415         $conf->{tpmstate0} = print_drive($parsed);
5416     }
5417
5418     my $defaults = load_defaults();
5419
5420     # set environment variable useful inside network script
5421     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5422
5423     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5424
5425     my $forcemachine = $params->{forcemachine};
5426     my $forcecpu = $params->{forcecpu};
5427     if ($resume) {
5428         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5429         $forcemachine = $conf->{runningmachine};
5430         $forcecpu = $conf->{runningcpu};
5431         print "Resuming suspended VM\n";
5432     }
5433
5434     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5435         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5436
5437     my $migration_ip;
5438     my $get_migration_ip = sub {
5439         my ($nodename) = @_;
5440
5441         return $migration_ip if defined($migration_ip);
5442
5443         my $cidr = $migrate_opts->{network};
5444
5445         if (!defined($cidr)) {
5446             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5447             $cidr = $dc_conf->{migration}->{network};
5448         }
5449
5450         if (defined($cidr)) {
5451             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5452
5453             die "could not get IP: no address configured on local " .
5454                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5455
5456             die "could not get IP: multiple addresses configured on local " .
5457                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5458
5459             $migration_ip = @$ips[0];
5460         }
5461
5462         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5463             if !defined($migration_ip);
5464
5465         return $migration_ip;
5466     };
5467
5468     my $migrate_uri;
5469     if ($statefile) {
5470         if ($statefile eq 'tcp') {
5471             my $localip = "localhost";
5472             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5473             my $nodename = nodename();
5474
5475             if (!defined($migration_type)) {
5476                 if (defined($datacenterconf->{migration}->{type})) {
5477                     $migration_type = $datacenterconf->{migration}->{type};
5478                 } else {
5479                     $migration_type = 'secure';
5480                 }
5481             }
5482
5483             if ($migration_type eq 'insecure') {
5484                 $localip = $get_migration_ip->($nodename);
5485                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5486             }
5487
5488             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5489             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5490             $migrate_uri = "tcp:${localip}:${migrate_port}";
5491             push @$cmd, '-incoming', $migrate_uri;
5492             push @$cmd, '-S';
5493
5494         } elsif ($statefile eq 'unix') {
5495             # should be default for secure migrations as a ssh TCP forward
5496             # tunnel is not deterministic reliable ready and fails regurarly
5497             # to set up in time, so use UNIX socket forwards
5498             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5499             unlink $socket_addr;
5500
5501             $migrate_uri = "unix:$socket_addr";
5502
5503             push @$cmd, '-incoming', $migrate_uri;
5504             push @$cmd, '-S';
5505
5506         } elsif (-e $statefile) {
5507             push @$cmd, '-loadstate', $statefile;
5508         } else {
5509             my $statepath = PVE::Storage::path($storecfg, $statefile);
5510             push @$vollist, $statefile;
5511             push @$cmd, '-loadstate', $statepath;
5512         }
5513     } elsif ($params->{paused}) {
5514         push @$cmd, '-S';
5515     }
5516
5517     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5518
5519     my $pci_devices = {}; # host pci devices
5520     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5521         my $dev = $conf->{"hostpci$i"} or next;
5522         $pci_devices->{$i} = parse_hostpci($dev);
5523     }
5524
5525     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5526     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5527
5528     # map to a flat list of pci ids
5529     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5530
5531     # reserve all PCI IDs before actually doing anything with them
5532     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5533
5534     eval {
5535         for my $id (sort keys %$pci_devices) {
5536             my $d = $pci_devices->{$id};
5537             for my $dev ($d->{pciid}->@*) {
5538                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5539             }
5540         }
5541     };
5542     if (my $err = $@) {
5543         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5544         warn $@ if $@;
5545         die $err;
5546     }
5547
5548     PVE::Storage::activate_volumes($storecfg, $vollist);
5549
5550     eval {
5551         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5552     };
5553     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5554     # timeout should be more than enough here...
5555     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5556
5557     my $cpuunits = get_cpuunits($conf);
5558
5559     my %run_params = (
5560         timeout => $statefile ? undef : $start_timeout,
5561         umask => 0077,
5562         noerr => 1,
5563     );
5564
5565     # when migrating, prefix QEMU output so other side can pick up any
5566     # errors that might occur and show the user
5567     if ($migratedfrom) {
5568         $run_params{quiet} = 1;
5569         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5570     }
5571
5572     my %systemd_properties = (
5573         Slice => 'qemu.slice',
5574         KillMode => 'process',
5575         SendSIGKILL => 0,
5576         TimeoutStopUSec => ULONG_MAX, # infinity
5577     );
5578
5579     if (PVE::CGroup::cgroup_mode() == 2) {
5580         $systemd_properties{CPUWeight} = $cpuunits;
5581     } else {
5582         $systemd_properties{CPUShares} = $cpuunits;
5583     }
5584
5585     if (my $cpulimit = $conf->{cpulimit}) {
5586         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5587     }
5588     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5589
5590     my $run_qemu = sub {
5591         PVE::Tools::run_fork sub {
5592             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5593
5594             my $tpmpid;
5595             if (my $tpm = $conf->{tpmstate0}) {
5596                 # start the TPM emulator so QEMU can connect on start
5597                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5598             }
5599
5600             my $exitcode = run_command($cmd, %run_params);
5601             if ($exitcode) {
5602                 if ($tpmpid) {
5603                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5604                     kill 'TERM', $tpmpid;
5605                 }
5606                 die "QEMU exited with code $exitcode\n";
5607             }
5608         };
5609     };
5610
5611     if ($conf->{hugepages}) {
5612
5613         my $code = sub {
5614             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5615             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5616
5617             PVE::QemuServer::Memory::hugepages_mount();
5618             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5619
5620             eval { $run_qemu->() };
5621             if (my $err = $@) {
5622                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5623                     if !$conf->{keephugepages};
5624                 die $err;
5625             }
5626
5627             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5628                 if !$conf->{keephugepages};
5629         };
5630         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5631
5632     } else {
5633         eval { $run_qemu->() };
5634     }
5635
5636     if (my $err = $@) {
5637         # deactivate volumes if start fails
5638         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5639         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5640
5641         die "start failed: $err";
5642     }
5643
5644     # re-reserve all PCI IDs now that we can know the actual VM PID
5645     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5646     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5647     warn $@ if $@;
5648
5649     print "migration listens on $migrate_uri\n" if $migrate_uri;
5650     $res->{migrate_uri} = $migrate_uri;
5651
5652     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5653         eval { mon_cmd($vmid, "cont"); };
5654         warn $@ if $@;
5655     }
5656
5657     #start nbd server for storage migration
5658     if (my $nbd = $migrate_opts->{nbd}) {
5659         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5660
5661         my $migrate_storage_uri;
5662         # nbd_protocol_version > 0 for unix socket support
5663         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5664             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5665             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5666             $migrate_storage_uri = "nbd:unix:$socket_path";
5667         } else {
5668             my $nodename = nodename();
5669             my $localip = $get_migration_ip->($nodename);
5670             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5671             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5672
5673             mon_cmd($vmid, "nbd-server-start", addr => {
5674                 type => 'inet',
5675                 data => {
5676                     host => "${localip}",
5677                     port => "${storage_migrate_port}",
5678                 },
5679             });
5680             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5681             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5682         }
5683
5684         $res->{migrate_storage_uri} = $migrate_storage_uri;
5685
5686         foreach my $opt (sort keys %$nbd) {
5687             my $drivestr = $nbd->{$opt}->{drivestr};
5688             my $volid = $nbd->{$opt}->{volid};
5689             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5690             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5691             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5692             print "re-using replicated volume: $opt - $volid\n"
5693                 if $nbd->{$opt}->{replicated};
5694
5695             $res->{drives}->{$opt} = $nbd->{$opt};
5696             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5697         }
5698     }
5699
5700     if ($migratedfrom) {
5701         eval {
5702             set_migration_caps($vmid);
5703         };
5704         warn $@ if $@;
5705
5706         if ($spice_port) {
5707             print "spice listens on port $spice_port\n";
5708             $res->{spice_port} = $spice_port;
5709             if ($migrate_opts->{spice_ticket}) {
5710                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5711                     $migrate_opts->{spice_ticket});
5712                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5713             }
5714         }
5715
5716     } else {
5717         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5718             if !$statefile && $conf->{balloon};
5719
5720         foreach my $opt (keys %$conf) {
5721             next if $opt !~  m/^net\d+$/;
5722             my $nicconf = parse_net($conf->{$opt});
5723             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5724         }
5725     }
5726
5727     mon_cmd($vmid, 'qom-set',
5728                 path => "machine/peripheral/balloon0",
5729                 property => "guest-stats-polling-interval",
5730                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5731
5732     if ($resume) {
5733         print "Resumed VM, removing state\n";
5734         if (my $vmstate = $conf->{vmstate}) {
5735             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5736             PVE::Storage::vdisk_free($storecfg, $vmstate);
5737         }
5738         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5739         PVE::QemuConfig->write_config($vmid, $conf);
5740     }
5741
5742     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5743
5744     return $res;
5745 }
5746
5747 sub vm_commandline {
5748     my ($storecfg, $vmid, $snapname) = @_;
5749
5750     my $conf = PVE::QemuConfig->load_config($vmid);
5751
5752     my ($forcemachine, $forcecpu);
5753     if ($snapname) {
5754         my $snapshot = $conf->{snapshots}->{$snapname};
5755         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5756
5757         # check for machine or CPU overrides in snapshot
5758         $forcemachine = $snapshot->{runningmachine};
5759         $forcecpu = $snapshot->{runningcpu};
5760
5761         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5762
5763         $conf = $snapshot;
5764     }
5765
5766     my $defaults = load_defaults();
5767
5768     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5769
5770     return PVE::Tools::cmd2string($cmd);
5771 }
5772
5773 sub vm_reset {
5774     my ($vmid, $skiplock) = @_;
5775
5776     PVE::QemuConfig->lock_config($vmid, sub {
5777
5778         my $conf = PVE::QemuConfig->load_config($vmid);
5779
5780         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5781
5782         mon_cmd($vmid, "system_reset");
5783     });
5784 }
5785
5786 sub get_vm_volumes {
5787     my ($conf) = @_;
5788
5789     my $vollist = [];
5790     foreach_volid($conf, sub {
5791         my ($volid, $attr) = @_;
5792
5793         return if $volid =~ m|^/|;
5794
5795         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5796         return if !$sid;
5797
5798         push @$vollist, $volid;
5799     });
5800
5801     return $vollist;
5802 }
5803
5804 sub vm_stop_cleanup {
5805     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5806
5807     eval {
5808
5809         if (!$keepActive) {
5810             my $vollist = get_vm_volumes($conf);
5811             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5812
5813             if (my $tpmdrive = $conf->{tpmstate0}) {
5814                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5815                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5816                 if ($storeid) {
5817                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5818                 }
5819             }
5820         }
5821
5822         foreach my $ext (qw(mon qmp pid vnc qga)) {
5823             unlink "/var/run/qemu-server/${vmid}.$ext";
5824         }
5825
5826         if ($conf->{ivshmem}) {
5827             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5828             # just delete it for now, VMs which have this already open do not
5829             # are affected, but new VMs will get a separated one. If this
5830             # becomes an issue we either add some sort of ref-counting or just
5831             # add a "don't delete on stop" flag to the ivshmem format.
5832             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5833         }
5834
5835         my $ids = [];
5836         foreach my $key (keys %$conf) {
5837             next if $key !~ m/^hostpci(\d+)$/;
5838             my $hostpciindex = $1;
5839             my $d = parse_hostpci($conf->{$key});
5840             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5841
5842             foreach my $pci (@{$d->{pciid}}) {
5843                 my $pciid = $pci->{id};
5844                 push @$ids, $pci->{id};
5845                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5846             }
5847         }
5848         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5849
5850         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5851     };
5852     warn $@ if $@; # avoid errors - just warn
5853 }
5854
5855 # call only in locked context
5856 sub _do_vm_stop {
5857     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5858
5859     my $pid = check_running($vmid, $nocheck);
5860     return if !$pid;
5861
5862     my $conf;
5863     if (!$nocheck) {
5864         $conf = PVE::QemuConfig->load_config($vmid);
5865         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5866         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5867             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5868             $timeout = $opts->{down} if $opts->{down};
5869         }
5870         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5871     }
5872
5873     eval {
5874         if ($shutdown) {
5875             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5876                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5877             } else {
5878                 mon_cmd($vmid, "system_powerdown");
5879             }
5880         } else {
5881             mon_cmd($vmid, "quit");
5882         }
5883     };
5884     my $err = $@;
5885
5886     if (!$err) {
5887         $timeout = 60 if !defined($timeout);
5888
5889         my $count = 0;
5890         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5891             $count++;
5892             sleep 1;
5893         }
5894
5895         if ($count >= $timeout) {
5896             if ($force) {
5897                 warn "VM still running - terminating now with SIGTERM\n";
5898                 kill 15, $pid;
5899             } else {
5900                 die "VM quit/powerdown failed - got timeout\n";
5901             }
5902         } else {
5903             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5904             return;
5905         }
5906     } else {
5907        if (!check_running($vmid, $nocheck)) {
5908             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5909             return;
5910        }
5911        if ($force) {
5912             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5913             kill 15, $pid;
5914         } else {
5915             die "VM quit/powerdown failed\n";
5916         }
5917     }
5918
5919     # wait again
5920     $timeout = 10;
5921
5922     my $count = 0;
5923     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5924         $count++;
5925         sleep 1;
5926     }
5927
5928     if ($count >= $timeout) {
5929         warn "VM still running - terminating now with SIGKILL\n";
5930         kill 9, $pid;
5931         sleep 1;
5932     }
5933
5934     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5935 }
5936
5937 # Note: use $nocheck to skip tests if VM configuration file exists.
5938 # We need that when migration VMs to other nodes (files already moved)
5939 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5940 sub vm_stop {
5941     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5942
5943     $force = 1 if !defined($force) && !$shutdown;
5944
5945     if ($migratedfrom){
5946         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5947         kill 15, $pid if $pid;
5948         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5949         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5950         return;
5951     }
5952
5953     PVE::QemuConfig->lock_config($vmid, sub {
5954         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5955    });
5956 }
5957
5958 sub vm_reboot {
5959     my ($vmid, $timeout) = @_;
5960
5961     PVE::QemuConfig->lock_config($vmid, sub {
5962         eval {
5963
5964             # only reboot if running, as qmeventd starts it again on a stop event
5965             return if !check_running($vmid);
5966
5967             create_reboot_request($vmid);
5968
5969             my $storecfg = PVE::Storage::config();
5970             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5971
5972         };
5973         if (my $err = $@) {
5974             # avoid that the next normal shutdown will be confused for a reboot
5975             clear_reboot_request($vmid);
5976             die $err;
5977         }
5978    });
5979 }
5980
5981 # note: if using the statestorage parameter, the caller has to check privileges
5982 sub vm_suspend {
5983     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5984
5985     my $conf;
5986     my $path;
5987     my $storecfg;
5988     my $vmstate;
5989
5990     PVE::QemuConfig->lock_config($vmid, sub {
5991
5992         $conf = PVE::QemuConfig->load_config($vmid);
5993
5994         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5995         PVE::QemuConfig->check_lock($conf)
5996             if !($skiplock || $is_backing_up);
5997
5998         die "cannot suspend to disk during backup\n"
5999             if $is_backing_up && $includestate;
6000
6001         if ($includestate) {
6002             $conf->{lock} = 'suspending';
6003             my $date = strftime("%Y-%m-%d", localtime(time()));
6004             $storecfg = PVE::Storage::config();
6005             if (!$statestorage) {
6006                 $statestorage = find_vmstate_storage($conf, $storecfg);
6007                 # check permissions for the storage
6008                 my $rpcenv = PVE::RPCEnvironment::get();
6009                 if ($rpcenv->{type} ne 'cli') {
6010                     my $authuser = $rpcenv->get_user();
6011                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6012                 }
6013             }
6014
6015
6016             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6017                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6018             $path = PVE::Storage::path($storecfg, $vmstate);
6019             PVE::QemuConfig->write_config($vmid, $conf);
6020         } else {
6021             mon_cmd($vmid, "stop");
6022         }
6023     });
6024
6025     if ($includestate) {
6026         # save vm state
6027         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6028
6029         eval {
6030             set_migration_caps($vmid, 1);
6031             mon_cmd($vmid, "savevm-start", statefile => $path);
6032             for(;;) {
6033                 my $state = mon_cmd($vmid, "query-savevm");
6034                 if (!$state->{status}) {
6035                     die "savevm not active\n";
6036                 } elsif ($state->{status} eq 'active') {
6037                     sleep(1);
6038                     next;
6039                 } elsif ($state->{status} eq 'completed') {
6040                     print "State saved, quitting\n";
6041                     last;
6042                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6043                     die "query-savevm failed with error '$state->{error}'\n"
6044                 } else {
6045                     die "query-savevm returned status '$state->{status}'\n";
6046                 }
6047             }
6048         };
6049         my $err = $@;
6050
6051         PVE::QemuConfig->lock_config($vmid, sub {
6052             $conf = PVE::QemuConfig->load_config($vmid);
6053             if ($err) {
6054                 # cleanup, but leave suspending lock, to indicate something went wrong
6055                 eval {
6056                     mon_cmd($vmid, "savevm-end");
6057                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6058                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6059                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6060                     PVE::QemuConfig->write_config($vmid, $conf);
6061                 };
6062                 warn $@ if $@;
6063                 die $err;
6064             }
6065
6066             die "lock changed unexpectedly\n"
6067                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6068
6069             mon_cmd($vmid, "quit");
6070             $conf->{lock} = 'suspended';
6071             PVE::QemuConfig->write_config($vmid, $conf);
6072         });
6073     }
6074 }
6075
6076 sub vm_resume {
6077     my ($vmid, $skiplock, $nocheck) = @_;
6078
6079     PVE::QemuConfig->lock_config($vmid, sub {
6080         my $res = mon_cmd($vmid, 'query-status');
6081         my $resume_cmd = 'cont';
6082         my $reset = 0;
6083
6084         if ($res->{status}) {
6085             return if $res->{status} eq 'running'; # job done, go home
6086             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6087             $reset = 1 if $res->{status} eq 'shutdown';
6088         }
6089
6090         if (!$nocheck) {
6091
6092             my $conf = PVE::QemuConfig->load_config($vmid);
6093
6094             PVE::QemuConfig->check_lock($conf)
6095                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6096         }
6097
6098         if ($reset) {
6099             # required if a VM shuts down during a backup and we get a resume
6100             # request before the backup finishes for example
6101             mon_cmd($vmid, "system_reset");
6102         }
6103         mon_cmd($vmid, $resume_cmd);
6104     });
6105 }
6106
6107 sub vm_sendkey {
6108     my ($vmid, $skiplock, $key) = @_;
6109
6110     PVE::QemuConfig->lock_config($vmid, sub {
6111
6112         my $conf = PVE::QemuConfig->load_config($vmid);
6113
6114         # there is no qmp command, so we use the human monitor command
6115         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6116         die $res if $res ne '';
6117     });
6118 }
6119
6120 # vzdump restore implementaion
6121
6122 sub tar_archive_read_firstfile {
6123     my $archive = shift;
6124
6125     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6126
6127     # try to detect archive type first
6128     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6129         die "unable to open file '$archive'\n";
6130     my $firstfile = <$fh>;
6131     kill 15, $pid;
6132     close $fh;
6133
6134     die "ERROR: archive contaions no data\n" if !$firstfile;
6135     chomp $firstfile;
6136
6137     return $firstfile;
6138 }
6139
6140 sub tar_restore_cleanup {
6141     my ($storecfg, $statfile) = @_;
6142
6143     print STDERR "starting cleanup\n";
6144
6145     if (my $fd = IO::File->new($statfile, "r")) {
6146         while (defined(my $line = <$fd>)) {
6147             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6148                 my $volid = $2;
6149                 eval {
6150                     if ($volid =~ m|^/|) {
6151                         unlink $volid || die 'unlink failed\n';
6152                     } else {
6153                         PVE::Storage::vdisk_free($storecfg, $volid);
6154                     }
6155                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6156                 };
6157                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6158             } else {
6159                 print STDERR "unable to parse line in statfile - $line";
6160             }
6161         }
6162         $fd->close();
6163     }
6164 }
6165
6166 sub restore_file_archive {
6167     my ($archive, $vmid, $user, $opts) = @_;
6168
6169     return restore_vma_archive($archive, $vmid, $user, $opts)
6170         if $archive eq '-';
6171
6172     my $info = PVE::Storage::archive_info($archive);
6173     my $format = $opts->{format} // $info->{format};
6174     my $comp = $info->{compression};
6175
6176     # try to detect archive format
6177     if ($format eq 'tar') {
6178         return restore_tar_archive($archive, $vmid, $user, $opts);
6179     } else {
6180         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6181     }
6182 }
6183
6184 # hepler to remove disks that will not be used after restore
6185 my $restore_cleanup_oldconf = sub {
6186     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6187
6188     PVE::QemuConfig->foreach_volume($oldconf, sub {
6189         my ($ds, $drive) = @_;
6190
6191         return if drive_is_cdrom($drive, 1);
6192
6193         my $volid = $drive->{file};
6194         return if !$volid || $volid =~ m|^/|;
6195
6196         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6197         return if !$path || !$owner || ($owner != $vmid);
6198
6199         # Note: only delete disk we want to restore
6200         # other volumes will become unused
6201         if ($virtdev_hash->{$ds}) {
6202             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6203             if (my $err = $@) {
6204                 warn $err;
6205             }
6206         }
6207     });
6208
6209     # delete vmstate files, after the restore we have no snapshots anymore
6210     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6211         my $snap = $oldconf->{snapshots}->{$snapname};
6212         if ($snap->{vmstate}) {
6213             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6214             if (my $err = $@) {
6215                 warn $err;
6216             }
6217         }
6218     }
6219 };
6220
6221 # Helper to parse vzdump backup device hints
6222 #
6223 # $rpcenv: Environment, used to ckeck storage permissions
6224 # $user: User ID, to check storage permissions
6225 # $storecfg: Storage configuration
6226 # $fh: the file handle for reading the configuration
6227 # $devinfo: should contain device sizes for all backu-up'ed devices
6228 # $options: backup options (pool, default storage)
6229 #
6230 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6231 my $parse_backup_hints = sub {
6232     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6233
6234     my $check_storage = sub { # assert if an image can be allocate
6235         my ($storeid, $scfg) = @_;
6236         die "Content type 'images' is not available on storage '$storeid'\n"
6237             if !$scfg->{content}->{images};
6238         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6239             if $user ne 'root@pam';
6240     };
6241
6242     my $virtdev_hash = {};
6243     while (defined(my $line = <$fh>)) {
6244         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6245             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6246             die "archive does not contain data for drive '$virtdev'\n"
6247                 if !$devinfo->{$devname};
6248
6249             if (defined($options->{storage})) {
6250                 $storeid = $options->{storage} || 'local';
6251             } elsif (!$storeid) {
6252                 $storeid = 'local';
6253             }
6254             $format = 'raw' if !$format;
6255             $devinfo->{$devname}->{devname} = $devname;
6256             $devinfo->{$devname}->{virtdev} = $virtdev;
6257             $devinfo->{$devname}->{format} = $format;
6258             $devinfo->{$devname}->{storeid} = $storeid;
6259
6260             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6261             $check_storage->($storeid, $scfg); # permission and content type check
6262
6263             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6264         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6265             my $virtdev = $1;
6266             my $drive = parse_drive($virtdev, $2);
6267
6268             if (drive_is_cloudinit($drive)) {
6269                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6270                 $storeid = $options->{storage} if defined ($options->{storage});
6271                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6272                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6273
6274                 $check_storage->($storeid, $scfg); # permission and content type check
6275
6276                 $virtdev_hash->{$virtdev} = {
6277                     format => $format,
6278                     storeid => $storeid,
6279                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6280                     is_cloudinit => 1,
6281                 };
6282             }
6283         }
6284     }
6285
6286     return $virtdev_hash;
6287 };
6288
6289 # Helper to allocate and activate all volumes required for a restore
6290 #
6291 # $storecfg: Storage configuration
6292 # $virtdev_hash: as returned by parse_backup_hints()
6293 #
6294 # Returns: { $virtdev => $volid }
6295 my $restore_allocate_devices = sub {
6296     my ($storecfg, $virtdev_hash, $vmid) = @_;
6297
6298     my $map = {};
6299     foreach my $virtdev (sort keys %$virtdev_hash) {
6300         my $d = $virtdev_hash->{$virtdev};
6301         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6302         my $storeid = $d->{storeid};
6303         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6304
6305         # test if requested format is supported
6306         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6307         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6308         $d->{format} = $defFormat if !$supported;
6309
6310         my $name;
6311         if ($d->{is_cloudinit}) {
6312             $name = "vm-$vmid-cloudinit";
6313             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6314             if ($scfg->{path}) {
6315                 $name .= ".$d->{format}";
6316             }
6317         }
6318
6319         my $volid = PVE::Storage::vdisk_alloc(
6320             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6321
6322         print STDERR "new volume ID is '$volid'\n";
6323         $d->{volid} = $volid;
6324
6325         PVE::Storage::activate_volumes($storecfg, [$volid]);
6326
6327         $map->{$virtdev} = $volid;
6328     }
6329
6330     return $map;
6331 };
6332
6333 sub restore_update_config_line {
6334     my ($cookie, $map, $line, $unique) = @_;
6335
6336     return '' if $line =~ m/^\#qmdump\#/;
6337     return '' if $line =~ m/^\#vzdump\#/;
6338     return '' if $line =~ m/^lock:/;
6339     return '' if $line =~ m/^unused\d+:/;
6340     return '' if $line =~ m/^parent:/;
6341
6342     my $res = '';
6343
6344     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6345     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6346         # try to convert old 1.X settings
6347         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6348         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6349             my ($model, $macaddr) = split(/\=/, $devconfig);
6350             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6351             my $net = {
6352                 model => $model,
6353                 bridge => "vmbr$ind",
6354                 macaddr => $macaddr,
6355             };
6356             my $netstr = print_net($net);
6357
6358             $res .= "net$cookie->{netcount}: $netstr\n";
6359             $cookie->{netcount}++;
6360         }
6361     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6362         my ($id, $netstr) = ($1, $2);
6363         my $net = parse_net($netstr);
6364         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6365         $netstr = print_net($net);
6366         $res .= "$id: $netstr\n";
6367     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6368         my $virtdev = $1;
6369         my $value = $3;
6370         my $di = parse_drive($virtdev, $value);
6371         if (defined($di->{backup}) && !$di->{backup}) {
6372             $res .= "#$line";
6373         } elsif ($map->{$virtdev}) {
6374             delete $di->{format}; # format can change on restore
6375             $di->{file} = $map->{$virtdev};
6376             $value = print_drive($di);
6377             $res .= "$virtdev: $value\n";
6378         } else {
6379             $res .= $line;
6380         }
6381     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6382         my $vmgenid = $1;
6383         if ($vmgenid ne '0') {
6384             # always generate a new vmgenid if there was a valid one setup
6385             $vmgenid = generate_uuid();
6386         }
6387         $res .= "vmgenid: $vmgenid\n";
6388     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6389         my ($uuid, $uuid_str);
6390         UUID::generate($uuid);
6391         UUID::unparse($uuid, $uuid_str);
6392         my $smbios1 = parse_smbios1($2);
6393         $smbios1->{uuid} = $uuid_str;
6394         $res .= $1.print_smbios1($smbios1)."\n";
6395     } else {
6396         $res .= $line;
6397     }
6398
6399     return $res;
6400 }
6401
6402 my $restore_deactivate_volumes = sub {
6403     my ($storecfg, $devinfo) = @_;
6404
6405     my $vollist = [];
6406     foreach my $devname (keys %$devinfo) {
6407         my $volid = $devinfo->{$devname}->{volid};
6408         push @$vollist, $volid if $volid;
6409     }
6410
6411     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6412 };
6413
6414 my $restore_destroy_volumes = sub {
6415     my ($storecfg, $devinfo) = @_;
6416
6417     foreach my $devname (keys %$devinfo) {
6418         my $volid = $devinfo->{$devname}->{volid};
6419         next if !$volid;
6420         eval {
6421             if ($volid =~ m|^/|) {
6422                 unlink $volid || die 'unlink failed\n';
6423             } else {
6424                 PVE::Storage::vdisk_free($storecfg, $volid);
6425             }
6426             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6427         };
6428         print STDERR "unable to cleanup '$volid' - $@" if $@;
6429     }
6430 };
6431
6432 sub scan_volids {
6433     my ($cfg, $vmid) = @_;
6434
6435     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6436
6437     my $volid_hash = {};
6438     foreach my $storeid (keys %$info) {
6439         foreach my $item (@{$info->{$storeid}}) {
6440             next if !($item->{volid} && $item->{size});
6441             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6442             $volid_hash->{$item->{volid}} = $item;
6443         }
6444     }
6445
6446     return $volid_hash;
6447 }
6448
6449 sub update_disk_config {
6450     my ($vmid, $conf, $volid_hash) = @_;
6451
6452     my $changes;
6453     my $prefix = "VM $vmid";
6454
6455     # used and unused disks
6456     my $referenced = {};
6457
6458     # Note: it is allowed to define multiple storages with same path (alias), so
6459     # we need to check both 'volid' and real 'path' (two different volid can point
6460     # to the same path).
6461
6462     my $referencedpath = {};
6463
6464     # update size info
6465     PVE::QemuConfig->foreach_volume($conf, sub {
6466         my ($opt, $drive) = @_;
6467
6468         my $volid = $drive->{file};
6469         return if !$volid;
6470         my $volume = $volid_hash->{$volid};
6471
6472         # mark volid as "in-use" for next step
6473         $referenced->{$volid} = 1;
6474         if ($volume && (my $path = $volume->{path})) {
6475             $referencedpath->{$path} = 1;
6476         }
6477
6478         return if drive_is_cdrom($drive);
6479         return if !$volume;
6480
6481         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6482         if (defined($updated)) {
6483             $changes = 1;
6484             $conf->{$opt} = print_drive($updated);
6485             print "$prefix ($opt): $msg\n";
6486         }
6487     });
6488
6489     # remove 'unusedX' entry if volume is used
6490     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6491         my ($opt, $drive) = @_;
6492
6493         my $volid = $drive->{file};
6494         return if !$volid;
6495
6496         my $path;
6497         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6498         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6499             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6500             $changes = 1;
6501             delete $conf->{$opt};
6502         }
6503
6504         $referenced->{$volid} = 1;
6505         $referencedpath->{$path} = 1 if $path;
6506     });
6507
6508     foreach my $volid (sort keys %$volid_hash) {
6509         next if $volid =~ m/vm-$vmid-state-/;
6510         next if $referenced->{$volid};
6511         my $path = $volid_hash->{$volid}->{path};
6512         next if !$path; # just to be sure
6513         next if $referencedpath->{$path};
6514         $changes = 1;
6515         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6516         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6517         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6518     }
6519
6520     return $changes;
6521 }
6522
6523 sub rescan {
6524     my ($vmid, $nolock, $dryrun) = @_;
6525
6526     my $cfg = PVE::Storage::config();
6527
6528     print "rescan volumes...\n";
6529     my $volid_hash = scan_volids($cfg, $vmid);
6530
6531     my $updatefn =  sub {
6532         my ($vmid) = @_;
6533
6534         my $conf = PVE::QemuConfig->load_config($vmid);
6535
6536         PVE::QemuConfig->check_lock($conf);
6537
6538         my $vm_volids = {};
6539         foreach my $volid (keys %$volid_hash) {
6540             my $info = $volid_hash->{$volid};
6541             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6542         }
6543
6544         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6545
6546         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6547     };
6548
6549     if (defined($vmid)) {
6550         if ($nolock) {
6551             &$updatefn($vmid);
6552         } else {
6553             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6554         }
6555     } else {
6556         my $vmlist = config_list();
6557         foreach my $vmid (keys %$vmlist) {
6558             if ($nolock) {
6559                 &$updatefn($vmid);
6560             } else {
6561                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6562             }
6563         }
6564     }
6565 }
6566
6567 sub restore_proxmox_backup_archive {
6568     my ($archive, $vmid, $user, $options) = @_;
6569
6570     my $storecfg = PVE::Storage::config();
6571
6572     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6573     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6574
6575     my $fingerprint = $scfg->{fingerprint};
6576     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6577
6578     my $repo = PVE::PBSClient::get_repository($scfg);
6579
6580     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6581     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6582     local $ENV{PBS_PASSWORD} = $password;
6583     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6584
6585     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6586         PVE::Storage::parse_volname($storecfg, $archive);
6587
6588     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6589
6590     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6591
6592     my $tmpdir = "/var/tmp/vzdumptmp$$";
6593     rmtree $tmpdir;
6594     mkpath $tmpdir;
6595
6596     my $conffile = PVE::QemuConfig->config_file($vmid);
6597      # disable interrupts (always do cleanups)
6598     local $SIG{INT} =
6599         local $SIG{TERM} =
6600         local $SIG{QUIT} =
6601         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6602
6603     # Note: $oldconf is undef if VM does not exists
6604     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6605     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6606     my $new_conf_raw = '';
6607
6608     my $rpcenv = PVE::RPCEnvironment::get();
6609     my $devinfo = {};
6610
6611     eval {
6612         # enable interrupts
6613         local $SIG{INT} =
6614             local $SIG{TERM} =
6615             local $SIG{QUIT} =
6616             local $SIG{HUP} =
6617             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6618
6619         my $cfgfn = "$tmpdir/qemu-server.conf";
6620         my $firewall_config_fn = "$tmpdir/fw.conf";
6621         my $index_fn = "$tmpdir/index.json";
6622
6623         my $cmd = "restore";
6624
6625         my $param = [$pbs_backup_name, "index.json", $index_fn];
6626         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6627         my $index = PVE::Tools::file_get_contents($index_fn);
6628         $index = decode_json($index);
6629
6630         # print Dumper($index);
6631         foreach my $info (@{$index->{files}}) {
6632             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6633                 my $devname = $1;
6634                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6635                     $devinfo->{$devname}->{size} = $1;
6636                 } else {
6637                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6638                 }
6639             }
6640         }
6641
6642         my $is_qemu_server_backup = scalar(
6643             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6644         );
6645         if (!$is_qemu_server_backup) {
6646             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6647         }
6648         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6649
6650         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6651         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6652
6653         if ($has_firewall_config) {
6654             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6655             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6656
6657             my $pve_firewall_dir = '/etc/pve/firewall';
6658             mkdir $pve_firewall_dir; # make sure the dir exists
6659             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6660         }
6661
6662         my $fh = IO::File->new($cfgfn, "r") ||
6663             die "unable to read qemu-server.conf - $!\n";
6664
6665         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6666
6667         # fixme: rate limit?
6668
6669         # create empty/temp config
6670         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6671
6672         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6673
6674         # allocate volumes
6675         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6676
6677         foreach my $virtdev (sort keys %$virtdev_hash) {
6678             my $d = $virtdev_hash->{$virtdev};
6679             next if $d->{is_cloudinit}; # no need to restore cloudinit
6680
6681             # this fails if storage is unavailable
6682             my $volid = $d->{volid};
6683             my $path = PVE::Storage::path($storecfg, $volid);
6684
6685             # for live-restore we only want to preload the efidisk and TPM state
6686             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6687
6688             my $pbs_restore_cmd = [
6689                 '/usr/bin/pbs-restore',
6690                 '--repository', $repo,
6691                 $pbs_backup_name,
6692                 "$d->{devname}.img.fidx",
6693                 $path,
6694                 '--verbose',
6695                 ];
6696
6697             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6698             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6699
6700             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6701                 push @$pbs_restore_cmd, '--skip-zero';
6702             }
6703
6704             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6705             print "restore proxmox backup image: $dbg_cmdstring\n";
6706             run_command($pbs_restore_cmd);
6707         }
6708
6709         $fh->seek(0, 0) || die "seek failed - $!\n";
6710
6711         my $cookie = { netcount => 0 };
6712         while (defined(my $line = <$fh>)) {
6713             $new_conf_raw .= restore_update_config_line(
6714                 $cookie,
6715                 $map,
6716                 $line,
6717                 $options->{unique},
6718             );
6719         }
6720
6721         $fh->close();
6722     };
6723     my $err = $@;
6724
6725     if ($err || !$options->{live}) {
6726         $restore_deactivate_volumes->($storecfg, $devinfo);
6727     }
6728
6729     rmtree $tmpdir;
6730
6731     if ($err) {
6732         $restore_destroy_volumes->($storecfg, $devinfo);
6733         die $err;
6734     }
6735
6736     if ($options->{live}) {
6737         # keep lock during live-restore
6738         $new_conf_raw .= "\nlock: create";
6739     }
6740
6741     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6742
6743     PVE::Cluster::cfs_update(); # make sure we read new file
6744
6745     eval { rescan($vmid, 1); };
6746     warn $@ if $@;
6747
6748     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6749
6750     if ($options->{live}) {
6751         # enable interrupts
6752         local $SIG{INT} =
6753             local $SIG{TERM} =
6754             local $SIG{QUIT} =
6755             local $SIG{HUP} =
6756             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6757
6758         my $conf = PVE::QemuConfig->load_config($vmid);
6759         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6760
6761         # these special drives are already restored before start
6762         delete $devinfo->{'drive-efidisk0'};
6763         delete $devinfo->{'drive-tpmstate0-backup'};
6764         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6765
6766         PVE::QemuConfig->remove_lock($vmid, "create");
6767     }
6768 }
6769
6770 sub pbs_live_restore {
6771     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6772
6773     print "starting VM for live-restore\n";
6774     print "repository: '$repo', snapshot: '$snap'\n";
6775
6776     my $pbs_backing = {};
6777     for my $ds (keys %$restored_disks) {
6778         $ds =~ m/^drive-(.*)$/;
6779         my $confname = $1;
6780         $pbs_backing->{$confname} = {
6781             repository => $repo,
6782             snapshot => $snap,
6783             archive => "$ds.img.fidx",
6784         };
6785         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6786
6787         my $drive = parse_drive($confname, $conf->{$confname});
6788         print "restoring '$ds' to '$drive->{file}'\n";
6789     }
6790
6791     my $drives_streamed = 0;
6792     eval {
6793         # make sure HA doesn't interrupt our restore by stopping the VM
6794         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6795             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6796         }
6797
6798         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6799         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6800         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6801
6802         my $qmeventd_fd = register_qmeventd_handle($vmid);
6803
6804         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6805         # this will effectively collapse the backing image chain consisting of
6806         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6807         # removes itself once all backing images vanish with 'auto-remove=on')
6808         my $jobs = {};
6809         for my $ds (sort keys %$restored_disks) {
6810             my $job_id = "restore-$ds";
6811             mon_cmd($vmid, 'block-stream',
6812                 'job-id' => $job_id,
6813                 device => "$ds",
6814             );
6815             $jobs->{$job_id} = {};
6816         }
6817
6818         mon_cmd($vmid, 'cont');
6819         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6820
6821         print "restore-drive jobs finished successfully, removing all tracking block devices"
6822             ." to disconnect from Proxmox Backup Server\n";
6823
6824         for my $ds (sort keys %$restored_disks) {
6825             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6826         }
6827
6828         close($qmeventd_fd);
6829     };
6830
6831     my $err = $@;
6832
6833     if ($err) {
6834         warn "An error occured during live-restore: $err\n";
6835         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6836         die "live-restore failed\n";
6837     }
6838 }
6839
6840 sub restore_vma_archive {
6841     my ($archive, $vmid, $user, $opts, $comp) = @_;
6842
6843     my $readfrom = $archive;
6844
6845     my $cfg = PVE::Storage::config();
6846     my $commands = [];
6847     my $bwlimit = $opts->{bwlimit};
6848
6849     my $dbg_cmdstring = '';
6850     my $add_pipe = sub {
6851         my ($cmd) = @_;
6852         push @$commands, $cmd;
6853         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6854         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6855         $readfrom = '-';
6856     };
6857
6858     my $input = undef;
6859     if ($archive eq '-') {
6860         $input = '<&STDIN';
6861     } else {
6862         # If we use a backup from a PVE defined storage we also consider that
6863         # storage's rate limit:
6864         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6865         if (defined($volid)) {
6866             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6867             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6868             if ($readlimit) {
6869                 print STDERR "applying read rate limit: $readlimit\n";
6870                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6871                 $add_pipe->($cstream);
6872             }
6873         }
6874     }
6875
6876     if ($comp) {
6877         my $info = PVE::Storage::decompressor_info('vma', $comp);
6878         my $cmd = $info->{decompressor};
6879         push @$cmd, $readfrom;
6880         $add_pipe->($cmd);
6881     }
6882
6883     my $tmpdir = "/var/tmp/vzdumptmp$$";
6884     rmtree $tmpdir;
6885
6886     # disable interrupts (always do cleanups)
6887     local $SIG{INT} =
6888         local $SIG{TERM} =
6889         local $SIG{QUIT} =
6890         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6891
6892     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6893     POSIX::mkfifo($mapfifo, 0600);
6894     my $fifofh;
6895     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6896
6897     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6898
6899     my $oldtimeout;
6900     my $timeout = 5;
6901
6902     my $devinfo = {};
6903
6904     my $rpcenv = PVE::RPCEnvironment::get();
6905
6906     my $conffile = PVE::QemuConfig->config_file($vmid);
6907
6908     # Note: $oldconf is undef if VM does not exist
6909     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6910     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6911     my $new_conf_raw = '';
6912
6913     my %storage_limits;
6914
6915     my $print_devmap = sub {
6916         my $cfgfn = "$tmpdir/qemu-server.conf";
6917
6918         # we can read the config - that is already extracted
6919         my $fh = IO::File->new($cfgfn, "r") ||
6920             die "unable to read qemu-server.conf - $!\n";
6921
6922         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6923         if (-f $fwcfgfn) {
6924             my $pve_firewall_dir = '/etc/pve/firewall';
6925             mkdir $pve_firewall_dir; # make sure the dir exists
6926             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6927         }
6928
6929         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6930
6931         foreach my $info (values %{$virtdev_hash}) {
6932             my $storeid = $info->{storeid};
6933             next if defined($storage_limits{$storeid});
6934
6935             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6936             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6937             $storage_limits{$storeid} = $limit * 1024;
6938         }
6939
6940         foreach my $devname (keys %$devinfo) {
6941             die "found no device mapping information for device '$devname'\n"
6942                 if !$devinfo->{$devname}->{virtdev};
6943         }
6944
6945         # create empty/temp config
6946         if ($oldconf) {
6947             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6948             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6949         }
6950
6951         # allocate volumes
6952         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6953
6954         # print restore information to $fifofh
6955         foreach my $virtdev (sort keys %$virtdev_hash) {
6956             my $d = $virtdev_hash->{$virtdev};
6957             next if $d->{is_cloudinit}; # no need to restore cloudinit
6958
6959             my $storeid = $d->{storeid};
6960             my $volid = $d->{volid};
6961
6962             my $map_opts = '';
6963             if (my $limit = $storage_limits{$storeid}) {
6964                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6965             }
6966
6967             my $write_zeros = 1;
6968             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6969                 $write_zeros = 0;
6970             }
6971
6972             my $path = PVE::Storage::path($cfg, $volid);
6973
6974             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6975
6976             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6977         }
6978
6979         $fh->seek(0, 0) || die "seek failed - $!\n";
6980
6981         my $cookie = { netcount => 0 };
6982         while (defined(my $line = <$fh>)) {
6983             $new_conf_raw .= restore_update_config_line(
6984                 $cookie,
6985                 $map,
6986                 $line,
6987                 $opts->{unique},
6988             );
6989         }
6990
6991         $fh->close();
6992     };
6993
6994     eval {
6995         # enable interrupts
6996         local $SIG{INT} =
6997             local $SIG{TERM} =
6998             local $SIG{QUIT} =
6999             local $SIG{HUP} =
7000             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7001         local $SIG{ALRM} = sub { die "got timeout\n"; };
7002
7003         $oldtimeout = alarm($timeout);
7004
7005         my $parser = sub {
7006             my $line = shift;
7007
7008             print "$line\n";
7009
7010             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7011                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7012                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7013             } elsif ($line =~ m/^CTIME: /) {
7014                 # we correctly received the vma config, so we can disable
7015                 # the timeout now for disk allocation (set to 10 minutes, so
7016                 # that we always timeout if something goes wrong)
7017                 alarm(600);
7018                 &$print_devmap();
7019                 print $fifofh "done\n";
7020                 my $tmp = $oldtimeout || 0;
7021                 $oldtimeout = undef;
7022                 alarm($tmp);
7023                 close($fifofh);
7024                 $fifofh = undef;
7025             }
7026         };
7027
7028         print "restore vma archive: $dbg_cmdstring\n";
7029         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7030     };
7031     my $err = $@;
7032
7033     alarm($oldtimeout) if $oldtimeout;
7034
7035     $restore_deactivate_volumes->($cfg, $devinfo);
7036
7037     close($fifofh) if $fifofh;
7038     unlink $mapfifo;
7039     rmtree $tmpdir;
7040
7041     if ($err) {
7042         $restore_destroy_volumes->($cfg, $devinfo);
7043         die $err;
7044     }
7045
7046     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7047
7048     PVE::Cluster::cfs_update(); # make sure we read new file
7049
7050     eval { rescan($vmid, 1); };
7051     warn $@ if $@;
7052
7053     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7054 }
7055
7056 sub restore_tar_archive {
7057     my ($archive, $vmid, $user, $opts) = @_;
7058
7059     if ($archive ne '-') {
7060         my $firstfile = tar_archive_read_firstfile($archive);
7061         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7062             if $firstfile ne 'qemu-server.conf';
7063     }
7064
7065     my $storecfg = PVE::Storage::config();
7066
7067     # avoid zombie disks when restoring over an existing VM -> cleanup first
7068     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7069     # skiplock=1 because qmrestore has set the 'create' lock itself already
7070     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7071     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7072
7073     my $tocmd = "/usr/lib/qemu-server/qmextract";
7074
7075     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7076     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7077     $tocmd .= ' --prealloc' if $opts->{prealloc};
7078     $tocmd .= ' --info' if $opts->{info};
7079
7080     # tar option "xf" does not autodetect compression when read from STDIN,
7081     # so we pipe to zcat
7082     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7083         PVE::Tools::shellquote("--to-command=$tocmd");
7084
7085     my $tmpdir = "/var/tmp/vzdumptmp$$";
7086     mkpath $tmpdir;
7087
7088     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7089     local $ENV{VZDUMP_VMID} = $vmid;
7090     local $ENV{VZDUMP_USER} = $user;
7091
7092     my $conffile = PVE::QemuConfig->config_file($vmid);
7093     my $new_conf_raw = '';
7094
7095     # disable interrupts (always do cleanups)
7096     local $SIG{INT} =
7097         local $SIG{TERM} =
7098         local $SIG{QUIT} =
7099         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7100
7101     eval {
7102         # enable interrupts
7103         local $SIG{INT} =
7104             local $SIG{TERM} =
7105             local $SIG{QUIT} =
7106             local $SIG{HUP} =
7107             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7108
7109         if ($archive eq '-') {
7110             print "extracting archive from STDIN\n";
7111             run_command($cmd, input => "<&STDIN");
7112         } else {
7113             print "extracting archive '$archive'\n";
7114             run_command($cmd);
7115         }
7116
7117         return if $opts->{info};
7118
7119         # read new mapping
7120         my $map = {};
7121         my $statfile = "$tmpdir/qmrestore.stat";
7122         if (my $fd = IO::File->new($statfile, "r")) {
7123             while (defined (my $line = <$fd>)) {
7124                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7125                     $map->{$1} = $2 if $1;
7126                 } else {
7127                     print STDERR "unable to parse line in statfile - $line\n";
7128                 }
7129             }
7130             $fd->close();
7131         }
7132
7133         my $confsrc = "$tmpdir/qemu-server.conf";
7134
7135         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7136
7137         my $cookie = { netcount => 0 };
7138         while (defined (my $line = <$srcfd>)) {
7139             $new_conf_raw .= restore_update_config_line(
7140                 $cookie,
7141                 $map,
7142                 $line,
7143                 $opts->{unique},
7144             );
7145         }
7146
7147         $srcfd->close();
7148     };
7149     if (my $err = $@) {
7150         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7151         die $err;
7152     }
7153
7154     rmtree $tmpdir;
7155
7156     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7157
7158     PVE::Cluster::cfs_update(); # make sure we read new file
7159
7160     eval { rescan($vmid, 1); };
7161     warn $@ if $@;
7162 };
7163
7164 sub foreach_storage_used_by_vm {
7165     my ($conf, $func) = @_;
7166
7167     my $sidhash = {};
7168
7169     PVE::QemuConfig->foreach_volume($conf, sub {
7170         my ($ds, $drive) = @_;
7171         return if drive_is_cdrom($drive);
7172
7173         my $volid = $drive->{file};
7174
7175         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7176         $sidhash->{$sid} = $sid if $sid;
7177     });
7178
7179     foreach my $sid (sort keys %$sidhash) {
7180         &$func($sid);
7181     }
7182 }
7183
7184 my $qemu_snap_storage = {
7185     rbd => 1,
7186 };
7187 sub do_snapshots_with_qemu {
7188     my ($storecfg, $volid, $deviceid) = @_;
7189
7190     return if $deviceid =~ m/tpmstate0/;
7191
7192     my $storage_name = PVE::Storage::parse_volume_id($volid);
7193     my $scfg = $storecfg->{ids}->{$storage_name};
7194     die "could not find storage '$storage_name'\n" if !defined($scfg);
7195
7196     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7197         return 1;
7198     }
7199
7200     if ($volid =~ m/\.(qcow2|qed)$/){
7201         return 1;
7202     }
7203
7204     return;
7205 }
7206
7207 sub qga_check_running {
7208     my ($vmid, $nowarn) = @_;
7209
7210     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7211     if ($@) {
7212         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7213         return 0;
7214     }
7215     return 1;
7216 }
7217
7218 sub template_create {
7219     my ($vmid, $conf, $disk) = @_;
7220
7221     my $storecfg = PVE::Storage::config();
7222
7223     PVE::QemuConfig->foreach_volume($conf, sub {
7224         my ($ds, $drive) = @_;
7225
7226         return if drive_is_cdrom($drive);
7227         return if $disk && $ds ne $disk;
7228
7229         my $volid = $drive->{file};
7230         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7231
7232         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7233         $drive->{file} = $voliddst;
7234         $conf->{$ds} = print_drive($drive);
7235         PVE::QemuConfig->write_config($vmid, $conf);
7236     });
7237 }
7238
7239 sub convert_iscsi_path {
7240     my ($path) = @_;
7241
7242     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7243         my $portal = $1;
7244         my $target = $2;
7245         my $lun = $3;
7246
7247         my $initiator_name = get_initiator_name();
7248
7249         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7250                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7251     }
7252
7253     die "cannot convert iscsi path '$path', unkown format\n";
7254 }
7255
7256 sub qemu_img_convert {
7257     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7258
7259     my $storecfg = PVE::Storage::config();
7260     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7261     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7262
7263     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7264
7265     my $cachemode;
7266     my $src_path;
7267     my $src_is_iscsi = 0;
7268     my $src_format;
7269
7270     if ($src_storeid) {
7271         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7272         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7273         $src_format = qemu_img_format($src_scfg, $src_volname);
7274         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7275         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7276         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7277     } elsif (-f $src_volid) {
7278         $src_path = $src_volid;
7279         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7280             $src_format = $1;
7281         }
7282     }
7283
7284     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7285
7286     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7287     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7288     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7289     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7290
7291     my $cmd = [];
7292     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7293     push @$cmd, '-l', "snapshot.name=$snapname"
7294         if $snapname && $src_format && $src_format eq "qcow2";
7295     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7296     push @$cmd, '-T', $cachemode if defined($cachemode);
7297
7298     if ($src_is_iscsi) {
7299         push @$cmd, '--image-opts';
7300         $src_path = convert_iscsi_path($src_path);
7301     } elsif ($src_format) {
7302         push @$cmd, '-f', $src_format;
7303     }
7304
7305     if ($dst_is_iscsi) {
7306         push @$cmd, '--target-image-opts';
7307         $dst_path = convert_iscsi_path($dst_path);
7308     } else {
7309         push @$cmd, '-O', $dst_format;
7310     }
7311
7312     push @$cmd, $src_path;
7313
7314     if (!$dst_is_iscsi && $is_zero_initialized) {
7315         push @$cmd, "zeroinit:$dst_path";
7316     } else {
7317         push @$cmd, $dst_path;
7318     }
7319
7320     my $parser = sub {
7321         my $line = shift;
7322         if($line =~ m/\((\S+)\/100\%\)/){
7323             my $percent = $1;
7324             my $transferred = int($size * $percent / 100);
7325             my $total_h = render_bytes($size, 1);
7326             my $transferred_h = render_bytes($transferred, 1);
7327
7328             print "transferred $transferred_h of $total_h ($percent%)\n";
7329         }
7330
7331     };
7332
7333     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7334     my $err = $@;
7335     die "copy failed: $err" if $err;
7336 }
7337
7338 sub qemu_img_format {
7339     my ($scfg, $volname) = @_;
7340
7341     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7342         return $1;
7343     } else {
7344         return "raw";
7345     }
7346 }
7347
7348 sub qemu_drive_mirror {
7349     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7350
7351     $jobs = {} if !$jobs;
7352
7353     my $qemu_target;
7354     my $format;
7355     $jobs->{"drive-$drive"} = {};
7356
7357     if ($dst_volid =~ /^nbd:/) {
7358         $qemu_target = $dst_volid;
7359         $format = "nbd";
7360     } else {
7361         my $storecfg = PVE::Storage::config();
7362         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7363
7364         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7365
7366         $format = qemu_img_format($dst_scfg, $dst_volname);
7367
7368         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7369
7370         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7371     }
7372
7373     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7374     $opts->{format} = $format if $format;
7375
7376     if (defined($src_bitmap)) {
7377         $opts->{sync} = 'incremental';
7378         $opts->{bitmap} = $src_bitmap;
7379         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7380     }
7381
7382     if (defined($bwlimit)) {
7383         $opts->{speed} = $bwlimit * 1024;
7384         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7385     } else {
7386         print "drive mirror is starting for drive-$drive\n";
7387     }
7388
7389     # if a job already runs for this device we get an error, catch it for cleanup
7390     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7391     if (my $err = $@) {
7392         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7393         warn "$@\n" if $@;
7394         die "mirroring error: $err\n";
7395     }
7396
7397     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7398 }
7399
7400 # $completion can be either
7401 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7402 # 'cancel': wait until all jobs are ready, block-job-cancel them
7403 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7404 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7405 sub qemu_drive_mirror_monitor {
7406     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7407
7408     $completion //= 'complete';
7409     $op //= "mirror";
7410
7411     eval {
7412         my $err_complete = 0;
7413
7414         my $starttime = time ();
7415         while (1) {
7416             die "block job ('$op') timed out\n" if $err_complete > 300;
7417
7418             my $stats = mon_cmd($vmid, "query-block-jobs");
7419             my $ctime = time();
7420
7421             my $running_jobs = {};
7422             for my $stat (@$stats) {
7423                 next if $stat->{type} ne $op;
7424                 $running_jobs->{$stat->{device}} = $stat;
7425             }
7426
7427             my $readycounter = 0;
7428
7429             for my $job_id (sort keys %$jobs) {
7430                 my $job = $running_jobs->{$job_id};
7431
7432                 my $vanished = !defined($job);
7433                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7434                 if($complete || ($vanished && $completion eq 'auto')) {
7435                     print "$job_id: $op-job finished\n";
7436                     delete $jobs->{$job_id};
7437                     next;
7438                 }
7439
7440                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7441
7442                 my $busy = $job->{busy};
7443                 my $ready = $job->{ready};
7444                 if (my $total = $job->{len}) {
7445                     my $transferred = $job->{offset} || 0;
7446                     my $remaining = $total - $transferred;
7447                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7448
7449                     my $duration = $ctime - $starttime;
7450                     my $total_h = render_bytes($total, 1);
7451                     my $transferred_h = render_bytes($transferred, 1);
7452
7453                     my $status = sprintf(
7454                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7455                         render_duration($duration),
7456                     );
7457
7458                     if ($ready) {
7459                         if ($busy) {
7460                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7461                         } else {
7462                             $status .= ", ready";
7463                         }
7464                     }
7465                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7466                     $jobs->{$job_id}->{ready} = $ready;
7467                 }
7468
7469                 $readycounter++ if $job->{ready};
7470             }
7471
7472             last if scalar(keys %$jobs) == 0;
7473
7474             if ($readycounter == scalar(keys %$jobs)) {
7475                 print "all '$op' jobs are ready\n";
7476
7477                 # do the complete later (or has already been done)
7478                 last if $completion eq 'skip' || $completion eq 'auto';
7479
7480                 if ($vmiddst && $vmiddst != $vmid) {
7481                     my $agent_running = $qga && qga_check_running($vmid);
7482                     if ($agent_running) {
7483                         print "freeze filesystem\n";
7484                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7485                         warn $@ if $@;
7486                     } else {
7487                         print "suspend vm\n";
7488                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7489                         warn $@ if $@;
7490                     }
7491
7492                     # if we clone a disk for a new target vm, we don't switch the disk
7493                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7494
7495                     if ($agent_running) {
7496                         print "unfreeze filesystem\n";
7497                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7498                         warn $@ if $@;
7499                     } else {
7500                         print "resume vm\n";
7501                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7502                         warn $@ if $@;
7503                     }
7504
7505                     last;
7506                 } else {
7507
7508                     for my $job_id (sort keys %$jobs) {
7509                         # try to switch the disk if source and destination are on the same guest
7510                         print "$job_id: Completing block job_id...\n";
7511
7512                         my $op;
7513                         if ($completion eq 'complete') {
7514                             $op = 'block-job-complete';
7515                         } elsif ($completion eq 'cancel') {
7516                             $op = 'block-job-cancel';
7517                         } else {
7518                             die "invalid completion value: $completion\n";
7519                         }
7520                         eval { mon_cmd($vmid, $op, device => $job_id) };
7521                         if ($@ =~ m/cannot be completed/) {
7522                             print "$job_id: block job cannot be completed, trying again.\n";
7523                             $err_complete++;
7524                         }else {
7525                             print "$job_id: Completed successfully.\n";
7526                             $jobs->{$job_id}->{complete} = 1;
7527                         }
7528                     }
7529                 }
7530             }
7531             sleep 1;
7532         }
7533     };
7534     my $err = $@;
7535
7536     if ($err) {
7537         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7538         die "block job ($op) error: $err";
7539     }
7540 }
7541
7542 sub qemu_blockjobs_cancel {
7543     my ($vmid, $jobs) = @_;
7544
7545     foreach my $job (keys %$jobs) {
7546         print "$job: Cancelling block job\n";
7547         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7548         $jobs->{$job}->{cancel} = 1;
7549     }
7550
7551     while (1) {
7552         my $stats = mon_cmd($vmid, "query-block-jobs");
7553
7554         my $running_jobs = {};
7555         foreach my $stat (@$stats) {
7556             $running_jobs->{$stat->{device}} = $stat;
7557         }
7558
7559         foreach my $job (keys %$jobs) {
7560
7561             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7562                 print "$job: Done.\n";
7563                 delete $jobs->{$job};
7564             }
7565         }
7566
7567         last if scalar(keys %$jobs) == 0;
7568
7569         sleep 1;
7570     }
7571 }
7572
7573 sub clone_disk {
7574     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7575
7576     my ($vmid, $running) = $source->@{qw(vmid running)};
7577     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7578
7579     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7580     my ($storage, $format) = $dest->@{qw(storage format)};
7581
7582     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7583         die "cloning from/to EFI disk requires EFI disk\n"
7584             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7585         die "cloning from/to TPM state requires TPM state\n"
7586             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7587     }
7588
7589     my $newvolid;
7590
7591     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7592     print "$src_drivename " if $src_drivename;
7593     print "($drive->{file})\n";
7594
7595     if (!$full) {
7596         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7597         push @$newvollist, $newvolid;
7598     } else {
7599
7600         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7601         $storeid = $storage if $storage;
7602
7603         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7604
7605         my $name = undef;
7606         my $size = undef;
7607         if (drive_is_cloudinit($drive)) {
7608             $name = "vm-$newvmid-cloudinit";
7609             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7610             if ($scfg->{path}) {
7611                 $name .= ".$dst_format";
7612             }
7613             $snapname = undef;
7614             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7615         } elsif ($dst_drivename eq 'efidisk0') {
7616             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7617         } elsif ($dst_drivename eq 'tpmstate0') {
7618             $dst_format = 'raw';
7619             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7620         } else {
7621             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7622         }
7623         $newvolid = PVE::Storage::vdisk_alloc(
7624             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7625         );
7626         push @$newvollist, $newvolid;
7627
7628         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7629
7630         if (drive_is_cloudinit($drive)) {
7631             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7632             # if this is the case, we have to complete any block-jobs still there from
7633             # previous drive-mirrors
7634             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7635                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7636             }
7637             goto no_data_clone;
7638         }
7639
7640         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7641         if (!$running || !$src_drivename || $snapname) {
7642             # TODO: handle bwlimits
7643             if ($dst_drivename eq 'efidisk0') {
7644                 # the relevant data on the efidisk may be smaller than the source
7645                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7646                 # that is given by the OVMF_VARS.fd
7647                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7648                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7649
7650                 # better for Ceph if block size is not too small, see bug #3324
7651                 my $bs = 1024*1024;
7652
7653                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7654                     "if=$src_path", "of=$dst_path"]);
7655             } else {
7656                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7657             }
7658         } else {
7659             die "cannot move TPM state while VM is running\n" if $src_drivename eq 'tpmstate0';
7660
7661             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7662                 $completion, $qga, $bwlimit);
7663         }
7664     }
7665
7666 no_data_clone:
7667     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7668
7669     my $disk = dclone($drive);
7670     delete $disk->{format};
7671     $disk->{file} = $newvolid;
7672     $disk->{size} = $size if defined($size);
7673
7674     return $disk;
7675 }
7676
7677 sub get_running_qemu_version {
7678     my ($vmid) = @_;
7679     my $res = mon_cmd($vmid, "query-version");
7680     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7681 }
7682
7683 sub qemu_use_old_bios_files {
7684     my ($machine_type) = @_;
7685
7686     return if !$machine_type;
7687
7688     my $use_old_bios_files = undef;
7689
7690     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7691         $machine_type = $1;
7692         $use_old_bios_files = 1;
7693     } else {
7694         my $version = extract_version($machine_type, kvm_user_version());
7695         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7696         # load new efi bios files on migration. So this hack is required to allow
7697         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7698         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7699         $use_old_bios_files = !min_version($version, 2, 4);
7700     }
7701
7702     return ($use_old_bios_files, $machine_type);
7703 }
7704
7705 sub get_efivars_size {
7706     my ($conf, $efidisk) = @_;
7707
7708     my $arch = get_vm_arch($conf);
7709     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7710     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7711     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7712     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7713     return -s $ovmf_vars;
7714 }
7715
7716 sub update_efidisk_size {
7717     my ($conf) = @_;
7718
7719     return if !defined($conf->{efidisk0});
7720
7721     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7722     $disk->{size} = get_efivars_size($conf);
7723     $conf->{efidisk0} = print_drive($disk);
7724
7725     return;
7726 }
7727
7728 sub update_tpmstate_size {
7729     my ($conf) = @_;
7730
7731     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7732     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7733     $conf->{tpmstate0} = print_drive($disk);
7734 }
7735
7736 sub create_efidisk($$$$$$$) {
7737     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7738
7739     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7740     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7741
7742     my $vars_size_b = -s $ovmf_vars;
7743     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7744     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7745     PVE::Storage::activate_volumes($storecfg, [$volid]);
7746
7747     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7748     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7749
7750     return ($volid, $size/1024);
7751 }
7752
7753 sub vm_iothreads_list {
7754     my ($vmid) = @_;
7755
7756     my $res = mon_cmd($vmid, 'query-iothreads');
7757
7758     my $iothreads = {};
7759     foreach my $iothread (@$res) {
7760         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7761     }
7762
7763     return $iothreads;
7764 }
7765
7766 sub scsihw_infos {
7767     my ($conf, $drive) = @_;
7768
7769     my $maxdev = 0;
7770
7771     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7772         $maxdev = 7;
7773     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7774         $maxdev = 1;
7775     } else {
7776         $maxdev = 256;
7777     }
7778
7779     my $controller = int($drive->{index} / $maxdev);
7780     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7781         ? "virtioscsi"
7782         : "scsihw";
7783
7784     return ($maxdev, $controller, $controller_prefix);
7785 }
7786
7787 sub windows_version {
7788     my ($ostype) = @_;
7789
7790     return 0 if !$ostype;
7791
7792     my $winversion = 0;
7793
7794     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7795         $winversion = 5;
7796     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7797         $winversion = 6;
7798     } elsif ($ostype =~ m/^win(\d+)$/) {
7799         $winversion = $1;
7800     }
7801
7802     return $winversion;
7803 }
7804
7805 sub resolve_dst_disk_format {
7806         my ($storecfg, $storeid, $src_volname, $format) = @_;
7807         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7808
7809         if (!$format) {
7810             # if no target format is specified, use the source disk format as hint
7811             if ($src_volname) {
7812                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7813                 $format = qemu_img_format($scfg, $src_volname);
7814             } else {
7815                 return $defFormat;
7816             }
7817         }
7818
7819         # test if requested format is supported - else use default
7820         my $supported = grep { $_ eq $format } @$validFormats;
7821         $format = $defFormat if !$supported;
7822         return $format;
7823 }
7824
7825 # NOTE: if this logic changes, please update docs & possibly gui logic
7826 sub find_vmstate_storage {
7827     my ($conf, $storecfg) = @_;
7828
7829     # first, return storage from conf if set
7830     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7831
7832     my ($target, $shared, $local);
7833
7834     foreach_storage_used_by_vm($conf, sub {
7835         my ($sid) = @_;
7836         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7837         my $dst = $scfg->{shared} ? \$shared : \$local;
7838         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7839     });
7840
7841     # second, use shared storage where VM has at least one disk
7842     # third, use local storage where VM has at least one disk
7843     # fall back to local storage
7844     $target = $shared // $local // 'local';
7845
7846     return $target;
7847 }
7848
7849 sub generate_uuid {
7850     my ($uuid, $uuid_str);
7851     UUID::generate($uuid);
7852     UUID::unparse($uuid, $uuid_str);
7853     return $uuid_str;
7854 }
7855
7856 sub generate_smbios1_uuid {
7857     return "uuid=".generate_uuid();
7858 }
7859
7860 sub nbd_stop {
7861     my ($vmid) = @_;
7862
7863     mon_cmd($vmid, 'nbd-server-stop');
7864 }
7865
7866 sub create_reboot_request {
7867     my ($vmid) = @_;
7868     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7869         or die "failed to create reboot trigger file: $!\n";
7870     close($fh);
7871 }
7872
7873 sub clear_reboot_request {
7874     my ($vmid) = @_;
7875     my $path = "/run/qemu-server/$vmid.reboot";
7876     my $res = 0;
7877
7878     $res = unlink($path);
7879     die "could not remove reboot request for $vmid: $!"
7880         if !$res && $! != POSIX::ENOENT;
7881
7882     return $res;
7883 }
7884
7885 sub bootorder_from_legacy {
7886     my ($conf, $bootcfg) = @_;
7887
7888     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7889     my $bootindex_hash = {};
7890     my $i = 1;
7891     foreach my $o (split(//, $boot)) {
7892         $bootindex_hash->{$o} = $i*100;
7893         $i++;
7894     }
7895
7896     my $bootorder = {};
7897
7898     PVE::QemuConfig->foreach_volume($conf, sub {
7899         my ($ds, $drive) = @_;
7900
7901         if (drive_is_cdrom ($drive, 1)) {
7902             if ($bootindex_hash->{d}) {
7903                 $bootorder->{$ds} = $bootindex_hash->{d};
7904                 $bootindex_hash->{d} += 1;
7905             }
7906         } elsif ($bootindex_hash->{c}) {
7907             $bootorder->{$ds} = $bootindex_hash->{c}
7908                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7909             $bootindex_hash->{c} += 1;
7910         }
7911     });
7912
7913     if ($bootindex_hash->{n}) {
7914         for (my $i = 0; $i < $MAX_NETS; $i++) {
7915             my $netname = "net$i";
7916             next if !$conf->{$netname};
7917             $bootorder->{$netname} = $bootindex_hash->{n};
7918             $bootindex_hash->{n} += 1;
7919         }
7920     }
7921
7922     return $bootorder;
7923 }
7924
7925 # Generate default device list for 'boot: order=' property. Matches legacy
7926 # default boot order, but with explicit device names. This is important, since
7927 # the fallback for when neither 'order' nor the old format is specified relies
7928 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7929 sub get_default_bootdevices {
7930     my ($conf) = @_;
7931
7932     my @ret = ();
7933
7934     # harddisk
7935     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7936     push @ret, $first if $first;
7937
7938     # cdrom
7939     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7940     push @ret, $first if $first;
7941
7942     # network
7943     for (my $i = 0; $i < $MAX_NETS; $i++) {
7944         my $netname = "net$i";
7945         next if !$conf->{$netname};
7946         push @ret, $netname;
7947         last;
7948     }
7949
7950     return \@ret;
7951 }
7952
7953 sub device_bootorder {
7954     my ($conf) = @_;
7955
7956     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7957
7958     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7959
7960     my $bootorder = {};
7961     if (!defined($boot) || $boot->{legacy}) {
7962         $bootorder = bootorder_from_legacy($conf, $boot);
7963     } elsif ($boot->{order}) {
7964         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7965         for my $dev (PVE::Tools::split_list($boot->{order})) {
7966             $bootorder->{$dev} = $i++;
7967         }
7968     }
7969
7970     return $bootorder;
7971 }
7972
7973 sub register_qmeventd_handle {
7974     my ($vmid) = @_;
7975
7976     my $fh;
7977     my $peer = "/var/run/qmeventd.sock";
7978     my $count = 0;
7979
7980     for (;;) {
7981         $count++;
7982         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7983         last if $fh;
7984         if ($! != EINTR && $! != EAGAIN) {
7985             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7986         }
7987         if ($count > 4) {
7988             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7989               . "after $count retries\n";
7990         }
7991         usleep(25000);
7992     }
7993
7994     # send handshake to mark VM as backing up
7995     print $fh to_json({vzdump => {vmid => "$vmid"}});
7996
7997     # return handle to be closed later when inhibit is no longer required
7998     return $fh;
7999 }
8000
8001 # bash completion helper
8002
8003 sub complete_backup_archives {
8004     my ($cmdname, $pname, $cvalue) = @_;
8005
8006     my $cfg = PVE::Storage::config();
8007
8008     my $storeid;
8009
8010     if ($cvalue =~ m/^([^:]+):/) {
8011         $storeid = $1;
8012     }
8013
8014     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8015
8016     my $res = [];
8017     foreach my $id (keys %$data) {
8018         foreach my $item (@{$data->{$id}}) {
8019             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8020             push @$res, $item->{volid} if defined($item->{volid});
8021         }
8022     }
8023
8024     return $res;
8025 }
8026
8027 my $complete_vmid_full = sub {
8028     my ($running) = @_;
8029
8030     my $idlist = vmstatus();
8031
8032     my $res = [];
8033
8034     foreach my $id (keys %$idlist) {
8035         my $d = $idlist->{$id};
8036         if (defined($running)) {
8037             next if $d->{template};
8038             next if $running && $d->{status} ne 'running';
8039             next if !$running && $d->{status} eq 'running';
8040         }
8041         push @$res, $id;
8042
8043     }
8044     return $res;
8045 };
8046
8047 sub complete_vmid {
8048     return &$complete_vmid_full();
8049 }
8050
8051 sub complete_vmid_stopped {
8052     return &$complete_vmid_full(0);
8053 }
8054
8055 sub complete_vmid_running {
8056     return &$complete_vmid_full(1);
8057 }
8058
8059 sub complete_storage {
8060
8061     my $cfg = PVE::Storage::config();
8062     my $ids = $cfg->{ids};
8063
8064     my $res = [];
8065     foreach my $sid (keys %$ids) {
8066         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8067         next if !$ids->{$sid}->{content}->{images};
8068         push @$res, $sid;
8069     }
8070
8071     return $res;
8072 }
8073
8074 sub complete_migration_storage {
8075     my ($cmd, $param, $current_value, $all_args) = @_;
8076
8077     my $targetnode = @$all_args[1];
8078
8079     my $cfg = PVE::Storage::config();
8080     my $ids = $cfg->{ids};
8081
8082     my $res = [];
8083     foreach my $sid (keys %$ids) {
8084         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8085         next if !$ids->{$sid}->{content}->{images};
8086         push @$res, $sid;
8087     }
8088
8089     return $res;
8090 }
8091
8092 sub vm_is_paused {
8093     my ($vmid) = @_;
8094     my $qmpstatus = eval {
8095         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8096         mon_cmd($vmid, "query-status");
8097     };
8098     warn "$@\n" if $@;
8099     return $qmpstatus && $qmpstatus->{status} eq "paused";
8100 }
8101
8102 sub check_volume_storage_type {
8103     my ($storecfg, $vol) = @_;
8104
8105     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8106     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8107     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8108
8109     die "storage '$storeid' does not support content-type '$vtype'\n"
8110         if !$scfg->{content}->{$vtype};
8111
8112     return 1;
8113 }
8114
8115 1;