PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. The default is read from the"
 371             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.",
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.",
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1040         return $volid;
1041     }
1042
1043     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1044     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1045     if ($@) {
1046         return if $noerr;
1047         die $@;
1048     }
1049     return $volid;
1050 }
1051
1052 my $usb_fmt = {
1053     host => {
1054         default_key => 1,
1055         type => 'string', format => 'pve-qm-usb-device',
1056         format_description => 'HOSTUSBDEVICE|spice',
1057         description => <<EODESCR,
1058 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1059
1060  'bus-port(.port)*' (decimal numbers) or
1061  'vendor_id:product_id' (hexadeciaml numbers) or
1062  'spice'
1063
1064 You can use the 'lsusb -t' command to list existing usb devices.
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1067 machines - use with special care.
1068
1069 The value 'spice' can be used to add a usb redirection devices for spice.
1070 EODESCR
1071     },
1072     usb3 => {
1073         optional => 1,
1074         type => 'boolean',
1075         description => "Specifies whether if given host option is a USB3 device or port.",
1076         default => 0,
1077     },
1078 };
1079
1080 my $usbdesc = {
1081     optional => 1,
1082     type => 'string', format => $usb_fmt,
1083     description => "Configure an USB device (n is 0 to 4).",
1084 };
1085 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1098 use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 my $paralleldesc= {
1105         optional => 1,
1106         type => 'string',
1107         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1108         description =>  "Map host parallel devices (n is 0 to 2).",
1109         verbose_description =>  <<EODESCR,
1110 Map host parallel devices (n is 0 to 2).
1111
1112 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1113 machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1129 }
1130
1131 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1132     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1133 }
1134
1135 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1136     $confdesc->{"usb$i"} = $usbdesc;
1137 }
1138
1139 my $boot_fmt = {
1140     legacy => {
1141         optional => 1,
1142         default_key => 1,
1143         type => 'string',
1144         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1145                      . " Deprecated, use 'order=' instead.",
1146         pattern => '[acdn]{1,4}',
1147         format_description => "[acdn]{1,4}",
1148
1149         # note: this is also the fallback if boot: is not given at all
1150         default => 'cdn',
1151     },
1152     order => {
1153         optional => 1,
1154         type => 'string',
1155         format => 'pve-qm-bootdev-list',
1156         format_description => "device[;device...]",
1157         description => <<EODESC,
1158 The guest will attempt to boot from devices in the order they appear here.
1159
1160 Disks, optical drives and passed-through storage USB devices will be directly
1161 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1162 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1163
1164 Note that only devices in this list will be marked as bootable and thus loaded
1165 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1166 (e.g. software-raid), you need to specify all of them here.
1167
1168 Overrides the deprecated 'legacy=[acdn]*' value when given.
1169 EODESC
1170     },
1171 };
1172 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1173
1174 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1175 sub verify_bootdev {
1176     my ($dev, $noerr) = @_;
1177
1178     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1179     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1180
1181     my $check = sub {
1182         my ($base) = @_;
1183         return 0 if $dev !~ m/^$base\d+$/;
1184         return 0 if !$confdesc->{$dev};
1185         return 1;
1186     };
1187
1188     return $dev if $check->("net");
1189     return $dev if $check->("usb");
1190     return $dev if $check->("hostpci");
1191
1192     return if $noerr;
1193     die "invalid boot device '$dev'\n";
1194 }
1195
1196 sub print_bootorder {
1197     my ($devs) = @_;
1198     return "" if !@$devs;
1199     my $data = { order => join(';', @$devs) };
1200     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1201 }
1202
1203 my $kvm_api_version = 0;
1204
1205 sub kvm_version {
1206     return $kvm_api_version if $kvm_api_version;
1207
1208     open my $fh, '<', '/dev/kvm' or return;
1209
1210     # 0xae00 => KVM_GET_API_VERSION
1211     $kvm_api_version = ioctl($fh, 0xae00, 0);
1212     close($fh);
1213
1214     return $kvm_api_version;
1215 }
1216
1217 my $kvm_user_version = {};
1218 my $kvm_mtime = {};
1219
1220 sub kvm_user_version {
1221     my ($binary) = @_;
1222
1223     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1224     my $st = stat($binary);
1225
1226     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1227     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1228         $cachedmtime == $st->mtime;
1229
1230     $kvm_user_version->{$binary} = 'unknown';
1231     $kvm_mtime->{$binary} = $st->mtime;
1232
1233     my $code = sub {
1234         my $line = shift;
1235         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1236             $kvm_user_version->{$binary} = $2;
1237         }
1238     };
1239
1240     eval { run_command([$binary, '--version'], outfunc => $code); };
1241     warn $@ if $@;
1242
1243     return $kvm_user_version->{$binary};
1244
1245 }
1246 my sub extract_version {
1247     my ($machine_type, $version) = @_;
1248     $version = kvm_user_version() if !defined($version);
1249     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1250 }
1251
1252 sub kernel_has_vhost_net {
1253     return -c '/dev/vhost-net';
1254 }
1255
1256 sub option_exists {
1257     my $key = shift;
1258     return defined($confdesc->{$key});
1259 }
1260
1261 my $cdrom_path;
1262 sub get_cdrom_path {
1263
1264     return  $cdrom_path if $cdrom_path;
1265
1266     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1267     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1268     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1269 }
1270
1271 sub get_iso_path {
1272     my ($storecfg, $vmid, $cdrom) = @_;
1273
1274     if ($cdrom eq 'cdrom') {
1275         return get_cdrom_path();
1276     } elsif ($cdrom eq 'none') {
1277         return '';
1278     } elsif ($cdrom =~ m|^/|) {
1279         return $cdrom;
1280     } else {
1281         return PVE::Storage::path($storecfg, $cdrom);
1282     }
1283 }
1284
1285 # try to convert old style file names to volume IDs
1286 sub filename_to_volume_id {
1287     my ($vmid, $file, $media) = @_;
1288
1289      if (!($file eq 'none' || $file eq 'cdrom' ||
1290           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1291
1292         return if $file =~ m|/|;
1293
1294         if ($media && $media eq 'cdrom') {
1295             $file = "local:iso/$file";
1296         } else {
1297             $file = "local:$vmid/$file";
1298         }
1299     }
1300
1301     return $file;
1302 }
1303
1304 sub verify_media_type {
1305     my ($opt, $vtype, $media) = @_;
1306
1307     return if !$media;
1308
1309     my $etype;
1310     if ($media eq 'disk') {
1311         $etype = 'images';
1312     } elsif ($media eq 'cdrom') {
1313         $etype = 'iso';
1314     } else {
1315         die "internal error";
1316     }
1317
1318     return if ($vtype eq $etype);
1319
1320     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1321 }
1322
1323 sub cleanup_drive_path {
1324     my ($opt, $storecfg, $drive) = @_;
1325
1326     # try to convert filesystem paths to volume IDs
1327
1328     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1329         ($drive->{file} !~ m|^/dev/.+|) &&
1330         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1331         ($drive->{file} !~ m/^\d+$/)) {
1332         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1333         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1334             if !$vtype;
1335         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1336         verify_media_type($opt, $vtype, $drive->{media});
1337         $drive->{file} = $volid;
1338     }
1339
1340     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1341 }
1342
1343 sub parse_hotplug_features {
1344     my ($data) = @_;
1345
1346     my $res = {};
1347
1348     return $res if $data eq '0';
1349
1350     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1351
1352     foreach my $feature (PVE::Tools::split_list($data)) {
1353         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1354             $res->{$1} = 1;
1355         } else {
1356             die "invalid hotplug feature '$feature'\n";
1357         }
1358     }
1359     return $res;
1360 }
1361
1362 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1363 sub pve_verify_hotplug_features {
1364     my ($value, $noerr) = @_;
1365
1366     return $value if parse_hotplug_features($value);
1367
1368     return if $noerr;
1369
1370     die "unable to parse hotplug option\n";
1371 }
1372
1373 sub scsi_inquiry {
1374     my($fh, $noerr) = @_;
1375
1376     my $SG_IO = 0x2285;
1377     my $SG_GET_VERSION_NUM = 0x2282;
1378
1379     my $versionbuf = "\x00" x 8;
1380     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1381     if (!$ret) {
1382         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1383         return;
1384     }
1385     my $version = unpack("I", $versionbuf);
1386     if ($version < 30000) {
1387         die "scsi generic interface too old\n"  if !$noerr;
1388         return;
1389     }
1390
1391     my $buf = "\x00" x 36;
1392     my $sensebuf = "\x00" x 8;
1393     my $cmd = pack("C x3 C x1", 0x12, 36);
1394
1395     # see /usr/include/scsi/sg.h
1396     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1397
1398     my $packet = pack(
1399         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1400     );
1401
1402     $ret = ioctl($fh, $SG_IO, $packet);
1403     if (!$ret) {
1404         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1405         return;
1406     }
1407
1408     my @res = unpack($sg_io_hdr_t, $packet);
1409     if ($res[17] || $res[18]) {
1410         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my $res = {};
1415     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1416
1417     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1418     $res->{type} &= 0x1F;
1419
1420     return $res;
1421 }
1422
1423 sub path_is_scsi {
1424     my ($path) = @_;
1425
1426     my $fh = IO::File->new("+<$path") || return;
1427     my $res = scsi_inquiry($fh, 1);
1428     close($fh);
1429
1430     return $res;
1431 }
1432
1433 sub print_tabletdevice_full {
1434     my ($conf, $arch) = @_;
1435
1436     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1437
1438     # we use uhci for old VMs because tablet driver was buggy in older qemu
1439     my $usbbus;
1440     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1441         $usbbus = 'ehci';
1442     } else {
1443         $usbbus = 'uhci';
1444     }
1445
1446     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1447 }
1448
1449 sub print_keyboarddevice_full {
1450     my ($conf, $arch) = @_;
1451
1452     return if $arch ne 'aarch64';
1453
1454     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1455 }
1456
1457 my sub get_drive_id {
1458     my ($drive) = @_;
1459     return "$drive->{interface}$drive->{index}";
1460 }
1461
1462 sub print_drivedevice_full {
1463     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1464
1465     my $device = '';
1466     my $maxdev = 0;
1467
1468     my $drive_id = get_drive_id($drive);
1469     if ($drive->{interface} eq 'virtio') {
1470         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1471         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1472         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1473     } elsif ($drive->{interface} eq 'scsi') {
1474
1475         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1476         my $unit = $drive->{index} % $maxdev;
1477         my $devicetype = 'hd';
1478         my $path = '';
1479         if (drive_is_cdrom($drive)) {
1480             $devicetype = 'cd';
1481         } else {
1482             if ($drive->{file} =~ m|^/|) {
1483                 $path = $drive->{file};
1484                 if (my $info = path_is_scsi($path)) {
1485                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1486                         $devicetype = 'block';
1487                     } elsif ($info->{type} == 1) { # tape
1488                         $devicetype = 'generic';
1489                     }
1490                 }
1491             } else {
1492                  $path = PVE::Storage::path($storecfg, $drive->{file});
1493             }
1494
1495             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1496             my $version = extract_version($machine_type, kvm_user_version());
1497             if ($path =~ m/^iscsi\:\/\// &&
1498                !min_version($version, 4, 1)) {
1499                 $devicetype = 'generic';
1500             }
1501         }
1502
1503         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1504             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1505         } else {
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1507                 .",lun=$drive->{index}";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1512             $device .= ",rotation_rate=1";
1513         }
1514         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1515
1516     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1517         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype";
1523         if ($drive->{interface} eq 'ide') {
1524             $device .= ",bus=ide.$controller,unit=$unit";
1525         } else {
1526             $device .= ",bus=ahci$controller.$unit";
1527         }
1528         $device .= ",drive=drive-$drive_id,id=$drive_id";
1529
1530         if ($devicetype eq 'hd') {
1531             if (my $model = $drive->{model}) {
1532                 $model = URI::Escape::uri_unescape($model);
1533                 $device .= ",model=$model";
1534             }
1535             if ($drive->{ssd}) {
1536                 $device .= ",rotation_rate=1";
1537             }
1538         }
1539         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1540     } elsif ($drive->{interface} eq 'usb') {
1541         die "implement me";
1542         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1543     } else {
1544         die "unsupported interface type";
1545     }
1546
1547     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1548
1549     if (my $serial = $drive->{serial}) {
1550         $serial = URI::Escape::uri_unescape($serial);
1551         $device .= ",serial=$serial";
1552     }
1553
1554
1555     return $device;
1556 }
1557
1558 sub get_initiator_name {
1559     my $initiator;
1560
1561     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1562     while (defined(my $line = <$fh>)) {
1563         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1564         $initiator = $1;
1565         last;
1566     }
1567     $fh->close();
1568
1569     return $initiator;
1570 }
1571
1572 sub print_drive_commandline_full {
1573     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1574
1575     my $path;
1576     my $volid = $drive->{file};
1577     my $format = $drive->{format};
1578     my $drive_id = get_drive_id($drive);
1579
1580     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1581     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1582
1583     if (drive_is_cdrom($drive)) {
1584         $path = get_iso_path($storecfg, $vmid, $volid);
1585         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1586     } else {
1587         if ($storeid) {
1588             $path = PVE::Storage::path($storecfg, $volid);
1589             $format //= qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format //= "raw";
1593         }
1594    }
1595
1596    my $is_rbd = $path =~ m/^rbd:/;
1597
1598     my $opts = '';
1599     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1600     foreach my $o (@qemu_drive_options) {
1601         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1602     }
1603
1604     # snapshot only accepts on|off
1605     if (defined($drive->{snapshot})) {
1606         my $v = $drive->{snapshot} ? 'on' : 'off';
1607         $opts .= ",snapshot=$v";
1608     }
1609
1610     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1611         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1612     }
1613
1614     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1615         my ($dir, $qmpname) = @$type;
1616         if (my $v = $drive->{"mbps$dir"}) {
1617             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1618         }
1619         if (my $v = $drive->{"mbps${dir}_max"}) {
1620             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1621         }
1622         if (my $v = $drive->{"bps${dir}_max_length"}) {
1623             $opts .= ",throttling.bps$qmpname-max-length=$v";
1624         }
1625         if (my $v = $drive->{"iops${dir}"}) {
1626             $opts .= ",throttling.iops$qmpname=$v";
1627         }
1628         if (my $v = $drive->{"iops${dir}_max"}) {
1629             $opts .= ",throttling.iops$qmpname-max=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}_max_length"}) {
1632             $opts .= ",throttling.iops$qmpname-max-length=$v";
1633         }
1634     }
1635
1636     if ($pbs_name) {
1637         $format = "rbd" if $is_rbd;
1638         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1639             if !$format;
1640         $opts .= ",format=alloc-track,file.driver=$format";
1641     } elsif ($format) {
1642         $opts .= ",format=$format";
1643     }
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # io_uring with cache mode writeback or writethrough on krbd will hang...
1655     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1656
1657     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1658     # sometimes, just plain disable...
1659     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1660
1661     if (!$drive->{aio}) {
1662         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1663             # io_uring supports all cache modes
1664             $opts .= ",aio=io_uring";
1665         } else {
1666             # aio native works only with O_DIRECT
1667             if($cache_direct) {
1668                 $opts .= ",aio=native";
1669             } else {
1670                 $opts .= ",aio=threads";
1671             }
1672         }
1673     }
1674
1675     if (!drive_is_cdrom($drive)) {
1676         my $detectzeroes;
1677         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1678             $detectzeroes = 'off';
1679         } elsif ($drive->{discard}) {
1680             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1681         } else {
1682             # This used to be our default with discard not being specified:
1683             $detectzeroes = 'on';
1684         }
1685
1686         # note: 'detect-zeroes' works per blockdev and we want it to persist
1687         # after the alloc-track is removed, so put it on 'file' directly
1688         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1689         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1690     }
1691
1692     if ($pbs_name) {
1693         $opts .= ",backing=$pbs_name";
1694         $opts .= ",auto-remove=on";
1695     }
1696
1697     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1698     my $file_param = "file";
1699     if ($pbs_name) {
1700         # non-rbd drivers require the underlying file to be a seperate block
1701         # node, so add a second .file indirection
1702         $file_param .= ".file" if !$is_rbd;
1703         $file_param .= ".filename";
1704     }
1705     my $pathinfo = $path ? "$file_param=$path," : '';
1706
1707     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1708 }
1709
1710 sub print_pbs_blockdev {
1711     my ($pbs_conf, $pbs_name) = @_;
1712     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1713     $blockdev .= ",repository=$pbs_conf->{repository}";
1714     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1715     $blockdev .= ",archive=$pbs_conf->{archive}";
1716     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1717     return $blockdev;
1718 }
1719
1720 sub print_netdevice_full {
1721     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1722
1723     my $device = $net->{model};
1724     if ($net->{model} eq 'virtio') {
1725          $device = 'virtio-net-pci';
1726      };
1727
1728     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1729     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1730     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1731         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1732         # and out of each queue plus one config interrupt and control vector queue
1733         my $vectors = $net->{queues} * 2 + 2;
1734         $tmpstr .= ",vectors=$vectors,mq=on";
1735     }
1736     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1737
1738     if (my $mtu = $net->{mtu}) {
1739         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1740             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1741             if ($mtu == 1) {
1742                  $mtu = $bridge_mtu;
1743             } elsif ($mtu < 576) {
1744                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1745             } elsif ($mtu > $bridge_mtu) {
1746                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1747             }
1748             $tmpstr .= ",host_mtu=$mtu";
1749         } else {
1750             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1751         }
1752     }
1753
1754     if ($use_old_bios_files) {
1755         my $romfile;
1756         if ($device eq 'virtio-net-pci') {
1757             $romfile = 'pxe-virtio.rom';
1758         } elsif ($device eq 'e1000') {
1759             $romfile = 'pxe-e1000.rom';
1760         } elsif ($device eq 'e1000e') {
1761             $romfile = 'pxe-e1000e.rom';
1762         } elsif ($device eq 'ne2k') {
1763             $romfile = 'pxe-ne2k_pci.rom';
1764         } elsif ($device eq 'pcnet') {
1765             $romfile = 'pxe-pcnet.rom';
1766         } elsif ($device eq 'rtl8139') {
1767             $romfile = 'pxe-rtl8139.rom';
1768         }
1769         $tmpstr .= ",romfile=$romfile" if $romfile;
1770     }
1771
1772     return $tmpstr;
1773 }
1774
1775 sub print_netdev_full {
1776     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1777
1778     my $i = '';
1779     if ($netid =~ m/^net(\d+)$/) {
1780         $i = int($1);
1781     }
1782
1783     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1784
1785     my $ifname = "tap${vmid}i$i";
1786
1787     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1788     die "interface name '$ifname' is too long (max 15 character)\n"
1789         if length($ifname) >= 16;
1790
1791     my $vhostparam = '';
1792     if (is_native($arch)) {
1793         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1794     }
1795
1796     my $vmname = $conf->{name} || "vm$vmid";
1797
1798     my $netdev = "";
1799     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1800
1801     if ($net->{bridge}) {
1802         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1803             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1804     } else {
1805         $netdev = "type=user,id=$netid,hostname=$vmname";
1806     }
1807
1808     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1809
1810     return $netdev;
1811 }
1812
1813 my $vga_map = {
1814     'cirrus' => 'cirrus-vga',
1815     'std' => 'VGA',
1816     'vmware' => 'vmware-svga',
1817     'virtio' => 'virtio-vga',
1818 };
1819
1820 sub print_vga_device {
1821     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1822
1823     my $type = $vga_map->{$vga->{type}};
1824     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1825         $type = 'virtio-gpu';
1826     }
1827     my $vgamem_mb = $vga->{memory};
1828
1829     my $max_outputs = '';
1830     if ($qxlnum) {
1831         $type = $id ? 'qxl' : 'qxl-vga';
1832
1833         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1834             # set max outputs so linux can have up to 4 qxl displays with one device
1835             if (min_version($machine_version, 4, 1)) {
1836                 $max_outputs = ",max_outputs=4";
1837             }
1838         }
1839     }
1840
1841     die "no devicetype for $vga->{type}\n" if !$type;
1842
1843     my $memory = "";
1844     if ($vgamem_mb) {
1845         if ($vga->{type} eq 'virtio') {
1846             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1847             $memory = ",max_hostmem=$bytes";
1848         } elsif ($qxlnum) {
1849             # from https://www.spice-space.org/multiple-monitors.html
1850             $memory = ",vgamem_mb=$vga->{memory}";
1851             my $ram = $vgamem_mb * 4;
1852             my $vram = $vgamem_mb * 2;
1853             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1854         } else {
1855             $memory = ",vgamem_mb=$vga->{memory}";
1856         }
1857     } elsif ($qxlnum && $id) {
1858         $memory = ",ram_size=67108864,vram_size=33554432";
1859     }
1860
1861     my $edidoff = "";
1862     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1863         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1864     }
1865
1866     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1867     my $vgaid = "vga" . ($id // '');
1868     my $pciaddr;
1869     if ($q35 && $vgaid eq 'vga') {
1870         # the first display uses pcie.0 bus on q35 machines
1871         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1872     } else {
1873         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1874     }
1875
1876     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1877 }
1878
1879 sub parse_number_sets {
1880     my ($set) = @_;
1881     my $res = [];
1882     foreach my $part (split(/;/, $set)) {
1883         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1884             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1885             push @$res, [ $1, $2 ];
1886         } else {
1887             die "invalid range: $part\n";
1888         }
1889     }
1890     return $res;
1891 }
1892
1893 sub parse_numa {
1894     my ($data) = @_;
1895
1896     my $res = parse_property_string($numa_fmt, $data);
1897     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1898     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1899     return $res;
1900 }
1901
1902 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1903 sub parse_net {
1904     my ($data) = @_;
1905
1906     my $res = eval { parse_property_string($net_fmt, $data) };
1907     if ($@) {
1908         warn $@;
1909         return;
1910     }
1911     if (!defined($res->{macaddr})) {
1912         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1913         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1914     }
1915     return $res;
1916 }
1917
1918 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1919 sub parse_ipconfig {
1920     my ($data) = @_;
1921
1922     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1923     if ($@) {
1924         warn $@;
1925         return;
1926     }
1927
1928     if ($res->{gw} && !$res->{ip}) {
1929         warn 'gateway specified without specifying an IP address';
1930         return;
1931     }
1932     if ($res->{gw6} && !$res->{ip6}) {
1933         warn 'IPv6 gateway specified without specifying an IPv6 address';
1934         return;
1935     }
1936     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1937         warn 'gateway specified together with DHCP';
1938         return;
1939     }
1940     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1941         # gw6 + auto/dhcp
1942         warn "IPv6 gateway specified together with $res->{ip6} address";
1943         return;
1944     }
1945
1946     if (!$res->{ip} && !$res->{ip6}) {
1947         return { ip => 'dhcp', ip6 => 'dhcp' };
1948     }
1949
1950     return $res;
1951 }
1952
1953 sub print_net {
1954     my $net = shift;
1955
1956     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1957 }
1958
1959 sub add_random_macs {
1960     my ($settings) = @_;
1961
1962     foreach my $opt (keys %$settings) {
1963         next if $opt !~ m/^net(\d+)$/;
1964         my $net = parse_net($settings->{$opt});
1965         next if !$net;
1966         $settings->{$opt} = print_net($net);
1967     }
1968 }
1969
1970 sub vm_is_volid_owner {
1971     my ($storecfg, $vmid, $volid) = @_;
1972
1973     if ($volid !~  m|^/|) {
1974         my ($path, $owner);
1975         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1976         if ($owner && ($owner == $vmid)) {
1977             return 1;
1978         }
1979     }
1980
1981     return;
1982 }
1983
1984 sub vmconfig_register_unused_drive {
1985     my ($storecfg, $vmid, $conf, $drive) = @_;
1986
1987     if (drive_is_cloudinit($drive)) {
1988         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1989         warn $@ if $@;
1990     } elsif (!drive_is_cdrom($drive)) {
1991         my $volid = $drive->{file};
1992         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1993             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1994         }
1995     }
1996 }
1997
1998 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1999 my $smbios1_fmt = {
2000     uuid => {
2001         type => 'string',
2002         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2003         format_description => 'UUID',
2004         description => "Set SMBIOS1 UUID.",
2005         optional => 1,
2006     },
2007     version => {
2008         type => 'string',
2009         pattern => '[A-Za-z0-9+\/]+={0,2}',
2010         format_description => 'Base64 encoded string',
2011         description => "Set SMBIOS1 version.",
2012         optional => 1,
2013     },
2014     serial => {
2015         type => 'string',
2016         pattern => '[A-Za-z0-9+\/]+={0,2}',
2017         format_description => 'Base64 encoded string',
2018         description => "Set SMBIOS1 serial number.",
2019         optional => 1,
2020     },
2021     manufacturer => {
2022         type => 'string',
2023         pattern => '[A-Za-z0-9+\/]+={0,2}',
2024         format_description => 'Base64 encoded string',
2025         description => "Set SMBIOS1 manufacturer.",
2026         optional => 1,
2027     },
2028     product => {
2029         type => 'string',
2030         pattern => '[A-Za-z0-9+\/]+={0,2}',
2031         format_description => 'Base64 encoded string',
2032         description => "Set SMBIOS1 product ID.",
2033         optional => 1,
2034     },
2035     sku => {
2036         type => 'string',
2037         pattern => '[A-Za-z0-9+\/]+={0,2}',
2038         format_description => 'Base64 encoded string',
2039         description => "Set SMBIOS1 SKU string.",
2040         optional => 1,
2041     },
2042     family => {
2043         type => 'string',
2044         pattern => '[A-Za-z0-9+\/]+={0,2}',
2045         format_description => 'Base64 encoded string',
2046         description => "Set SMBIOS1 family string.",
2047         optional => 1,
2048     },
2049     base64 => {
2050         type => 'boolean',
2051         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2052         optional => 1,
2053     },
2054 };
2055
2056 sub parse_smbios1 {
2057     my ($data) = @_;
2058
2059     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2060     warn $@ if $@;
2061     return $res;
2062 }
2063
2064 sub print_smbios1 {
2065     my ($smbios1) = @_;
2066     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2067 }
2068
2069 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2070
2071 sub parse_watchdog {
2072     my ($value) = @_;
2073
2074     return if !$value;
2075
2076     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2077     warn $@ if $@;
2078     return $res;
2079 }
2080
2081 sub parse_guest_agent {
2082     my ($conf) = @_;
2083
2084     return {} if !defined($conf->{agent});
2085
2086     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2087     warn $@ if $@;
2088
2089     # if the agent is disabled ignore the other potentially set properties
2090     return {} if !$res->{enabled};
2091     return $res;
2092 }
2093
2094 sub get_qga_key {
2095     my ($conf, $key) = @_;
2096     return undef if !defined($conf->{agent});
2097
2098     my $agent = parse_guest_agent($conf);
2099     return $agent->{$key};
2100 }
2101
2102 sub parse_vga {
2103     my ($value) = @_;
2104
2105     return {} if !$value;
2106     my $res = eval { parse_property_string($vga_fmt, $value) };
2107     warn $@ if $@;
2108     return $res;
2109 }
2110
2111 sub parse_rng {
2112     my ($value) = @_;
2113
2114     return if !$value;
2115
2116     my $res = eval { parse_property_string($rng_fmt, $value) };
2117     warn $@ if $@;
2118     return $res;
2119 }
2120
2121 sub parse_meta_info {
2122     my ($value) = @_;
2123
2124     return if !$value;
2125
2126     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2127     warn $@ if $@;
2128     return $res;
2129 }
2130
2131 sub new_meta_info_string {
2132     my () = @_; # for now do not allow to override any value
2133
2134     return PVE::JSONSchema::print_property_string(
2135         {
2136             'creation-qemu' => kvm_user_version(),
2137             ctime => "". int(time()),
2138         },
2139         $meta_info_fmt
2140     );
2141 }
2142
2143 sub qemu_created_version_fixups {
2144     my ($conf, $forcemachine, $kvmver) = @_;
2145
2146     my $meta = parse_meta_info($conf->{meta}) // {};
2147     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2148
2149     # check if we need to apply some handling for VMs that always use the latest machine version but
2150     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2151     # would be required (we do not want to pin machine version for non-windows OS type)
2152     if (
2153         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2154         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2155         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2156         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2157     ) {
2158         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2159         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2160             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2161             # and thus with the predictable interface naming of systemd
2162             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2163         }
2164     }
2165     return;
2166 }
2167
2168 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2169 sub verify_usb_device {
2170     my ($value, $noerr) = @_;
2171
2172     return $value if parse_usb_device($value);
2173
2174     return if $noerr;
2175
2176     die "unable to parse usb device\n";
2177 }
2178
2179 # add JSON properties for create and set function
2180 sub json_config_properties {
2181     my $prop = shift;
2182
2183     my $skip_json_config_opts = {
2184         parent => 1,
2185         snaptime => 1,
2186         vmstate => 1,
2187         runningmachine => 1,
2188         runningcpu => 1,
2189         meta => 1,
2190     };
2191
2192     foreach my $opt (keys %$confdesc) {
2193         next if $skip_json_config_opts->{$opt};
2194         $prop->{$opt} = $confdesc->{$opt};
2195     }
2196
2197     return $prop;
2198 }
2199
2200 # return copy of $confdesc_cloudinit to generate documentation
2201 sub cloudinit_config_properties {
2202
2203     return dclone($confdesc_cloudinit);
2204 }
2205
2206 sub check_type {
2207     my ($key, $value) = @_;
2208
2209     die "unknown setting '$key'\n" if !$confdesc->{$key};
2210
2211     my $type = $confdesc->{$key}->{type};
2212
2213     if (!defined($value)) {
2214         die "got undefined value\n";
2215     }
2216
2217     if ($value =~ m/[\n\r]/) {
2218         die "property contains a line feed\n";
2219     }
2220
2221     if ($type eq 'boolean') {
2222         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2223         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2224         die "type check ('boolean') failed - got '$value'\n";
2225     } elsif ($type eq 'integer') {
2226         return int($1) if $value =~ m/^(\d+)$/;
2227         die "type check ('integer') failed - got '$value'\n";
2228     } elsif ($type eq 'number') {
2229         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2230         die "type check ('number') failed - got '$value'\n";
2231     } elsif ($type eq 'string') {
2232         if (my $fmt = $confdesc->{$key}->{format}) {
2233             PVE::JSONSchema::check_format($fmt, $value);
2234             return $value;
2235         }
2236         $value =~ s/^\"(.*)\"$/$1/;
2237         return $value;
2238     } else {
2239         die "internal error"
2240     }
2241 }
2242
2243 sub destroy_vm {
2244     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2245
2246     my $conf = PVE::QemuConfig->load_config($vmid);
2247
2248     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2249
2250     if ($conf->{template}) {
2251         # check if any base image is still used by a linked clone
2252         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2253                 my ($ds, $drive) = @_;
2254                 return if drive_is_cdrom($drive);
2255
2256                 my $volid = $drive->{file};
2257                 return if !$volid || $volid =~ m|^/|;
2258
2259                 die "base volume '$volid' is still in use by linked cloned\n"
2260                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2261
2262         });
2263     }
2264
2265     my $volids = {};
2266     my $remove_owned_drive = sub {
2267         my ($ds, $drive) = @_;
2268         return if drive_is_cdrom($drive, 1);
2269
2270         my $volid = $drive->{file};
2271         return if !$volid || $volid =~ m|^/|;
2272         return if $volids->{$volid};
2273
2274         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2275         return if !$path || !$owner || ($owner != $vmid);
2276
2277         $volids->{$volid} = 1;
2278         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2279         warn "Could not remove disk '$volid', check manually: $@" if $@;
2280     };
2281
2282     # only remove disks owned by this VM (referenced in the config)
2283     my $include_opts = {
2284         include_unused => 1,
2285         extra_keys => ['vmstate'],
2286     };
2287     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2288
2289     for my $snap (values %{$conf->{snapshots}}) {
2290         next if !defined($snap->{vmstate});
2291         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2292         next if !defined($drive);
2293         $remove_owned_drive->('vmstate', $drive);
2294     }
2295
2296     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2297
2298     if ($purge_unreferenced) { # also remove unreferenced disk
2299         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2300         PVE::Storage::foreach_volid($vmdisks, sub {
2301             my ($volid, $sid, $volname, $d) = @_;
2302             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2303             warn $@ if $@;
2304         });
2305     }
2306
2307     if (defined $replacement_conf) {
2308         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2309     } else {
2310         PVE::QemuConfig->destroy_config($vmid);
2311     }
2312 }
2313
2314 sub parse_vm_config {
2315     my ($filename, $raw, $strict) = @_;
2316
2317     return if !defined($raw);
2318
2319     my $res = {
2320         digest => Digest::SHA::sha1_hex($raw),
2321         snapshots => {},
2322         pending => {},
2323     };
2324
2325     my $handle_error = sub {
2326         my ($msg) = @_;
2327
2328         if ($strict) {
2329             die $msg;
2330         } else {
2331             warn $msg;
2332         }
2333     };
2334
2335     $filename =~ m|/qemu-server/(\d+)\.conf$|
2336         || die "got strange filename '$filename'";
2337
2338     my $vmid = $1;
2339
2340     my $conf = $res;
2341     my $descr;
2342     my $section = '';
2343
2344     my @lines = split(/\n/, $raw);
2345     foreach my $line (@lines) {
2346         next if $line =~ m/^\s*$/;
2347
2348         if ($line =~ m/^\[PENDING\]\s*$/i) {
2349             $section = 'pending';
2350             if (defined($descr)) {
2351                 $descr =~ s/\s+$//;
2352                 $conf->{description} = $descr;
2353             }
2354             $descr = undef;
2355             $conf = $res->{$section} = {};
2356             next;
2357
2358         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2359             $section = $1;
2360             if (defined($descr)) {
2361                 $descr =~ s/\s+$//;
2362                 $conf->{description} = $descr;
2363             }
2364             $descr = undef;
2365             $conf = $res->{snapshots}->{$section} = {};
2366             next;
2367         }
2368
2369         if ($line =~ m/^\#(.*)\s*$/) {
2370             $descr = '' if !defined($descr);
2371             $descr .= PVE::Tools::decode_text($1) . "\n";
2372             next;
2373         }
2374
2375         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2376             $descr = '' if !defined($descr);
2377             $descr .= PVE::Tools::decode_text($2);
2378         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2379             $conf->{snapstate} = $1;
2380         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2381             my $key = $1;
2382             my $value = $2;
2383             $conf->{$key} = $value;
2384         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2385             my $value = $1;
2386             if ($section eq 'pending') {
2387                 $conf->{delete} = $value; # we parse this later
2388             } else {
2389                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2390             }
2391         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2392             my $key = $1;
2393             my $value = $2;
2394             eval { $value = check_type($key, $value); };
2395             if ($@) {
2396                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2397             } else {
2398                 $key = 'ide2' if $key eq 'cdrom';
2399                 my $fmt = $confdesc->{$key}->{format};
2400                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2401                     my $v = parse_drive($key, $value);
2402                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2403                         $v->{file} = $volid;
2404                         $value = print_drive($v);
2405                     } else {
2406                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2407                         next;
2408                     }
2409                 }
2410
2411                 $conf->{$key} = $value;
2412             }
2413         } else {
2414             $handle_error->("vm $vmid - unable to parse config: $line\n");
2415         }
2416     }
2417
2418     if (defined($descr)) {
2419         $descr =~ s/\s+$//;
2420         $conf->{description} = $descr;
2421     }
2422     delete $res->{snapstate}; # just to be sure
2423
2424     return $res;
2425 }
2426
2427 sub write_vm_config {
2428     my ($filename, $conf) = @_;
2429
2430     delete $conf->{snapstate}; # just to be sure
2431
2432     if ($conf->{cdrom}) {
2433         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2434         $conf->{ide2} = $conf->{cdrom};
2435         delete $conf->{cdrom};
2436     }
2437
2438     # we do not use 'smp' any longer
2439     if ($conf->{sockets}) {
2440         delete $conf->{smp};
2441     } elsif ($conf->{smp}) {
2442         $conf->{sockets} = $conf->{smp};
2443         delete $conf->{cores};
2444         delete $conf->{smp};
2445     }
2446
2447     my $used_volids = {};
2448
2449     my $cleanup_config = sub {
2450         my ($cref, $pending, $snapname) = @_;
2451
2452         foreach my $key (keys %$cref) {
2453             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2454                 $key eq 'snapstate' || $key eq 'pending';
2455             my $value = $cref->{$key};
2456             if ($key eq 'delete') {
2457                 die "propertry 'delete' is only allowed in [PENDING]\n"
2458                     if !$pending;
2459                 # fixme: check syntax?
2460                 next;
2461             }
2462             eval { $value = check_type($key, $value); };
2463             die "unable to parse value of '$key' - $@" if $@;
2464
2465             $cref->{$key} = $value;
2466
2467             if (!$snapname && is_valid_drivename($key)) {
2468                 my $drive = parse_drive($key, $value);
2469                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2470             }
2471         }
2472     };
2473
2474     &$cleanup_config($conf);
2475
2476     &$cleanup_config($conf->{pending}, 1);
2477
2478     foreach my $snapname (keys %{$conf->{snapshots}}) {
2479         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2480         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2481     }
2482
2483     # remove 'unusedX' settings if we re-add a volume
2484     foreach my $key (keys %$conf) {
2485         my $value = $conf->{$key};
2486         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2487             delete $conf->{$key};
2488         }
2489     }
2490
2491     my $generate_raw_config = sub {
2492         my ($conf, $pending) = @_;
2493
2494         my $raw = '';
2495
2496         # add description as comment to top of file
2497         if (defined(my $descr = $conf->{description})) {
2498             if ($descr) {
2499                 foreach my $cl (split(/\n/, $descr)) {
2500                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2501                 }
2502             } else {
2503                 $raw .= "#\n" if $pending;
2504             }
2505         }
2506
2507         foreach my $key (sort keys %$conf) {
2508             next if $key =~ /^(digest|description|pending|snapshots)$/;
2509             $raw .= "$key: $conf->{$key}\n";
2510         }
2511         return $raw;
2512     };
2513
2514     my $raw = &$generate_raw_config($conf);
2515
2516     if (scalar(keys %{$conf->{pending}})){
2517         $raw .= "\n[PENDING]\n";
2518         $raw .= &$generate_raw_config($conf->{pending}, 1);
2519     }
2520
2521     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2522         $raw .= "\n[$snapname]\n";
2523         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2524     }
2525
2526     return $raw;
2527 }
2528
2529 sub load_defaults {
2530
2531     my $res = {};
2532
2533     # we use static defaults from our JSON schema configuration
2534     foreach my $key (keys %$confdesc) {
2535         if (defined(my $default = $confdesc->{$key}->{default})) {
2536             $res->{$key} = $default;
2537         }
2538     }
2539
2540     return $res;
2541 }
2542
2543 sub config_list {
2544     my $vmlist = PVE::Cluster::get_vmlist();
2545     my $res = {};
2546     return $res if !$vmlist || !$vmlist->{ids};
2547     my $ids = $vmlist->{ids};
2548     my $nodename = nodename();
2549
2550     foreach my $vmid (keys %$ids) {
2551         my $d = $ids->{$vmid};
2552         next if !$d->{node} || $d->{node} ne $nodename;
2553         next if !$d->{type} || $d->{type} ne 'qemu';
2554         $res->{$vmid}->{exists} = 1;
2555     }
2556     return $res;
2557 }
2558
2559 # test if VM uses local resources (to prevent migration)
2560 sub check_local_resources {
2561     my ($conf, $noerr) = @_;
2562
2563     my @loc_res = ();
2564
2565     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2566     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2567
2568     push @loc_res, "ivshmem" if $conf->{ivshmem};
2569
2570     foreach my $k (keys %$conf) {
2571         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2572         # sockets are safe: they will recreated be on the target side post-migrate
2573         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2574         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2575     }
2576
2577     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2578
2579     return \@loc_res;
2580 }
2581
2582 # check if used storages are available on all nodes (use by migrate)
2583 sub check_storage_availability {
2584     my ($storecfg, $conf, $node) = @_;
2585
2586     PVE::QemuConfig->foreach_volume($conf, sub {
2587         my ($ds, $drive) = @_;
2588
2589         my $volid = $drive->{file};
2590         return if !$volid;
2591
2592         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2593         return if !$sid;
2594
2595         # check if storage is available on both nodes
2596         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2597         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2598
2599         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2600
2601         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2602             if !$scfg->{content}->{$vtype};
2603    });
2604 }
2605
2606 # list nodes where all VM images are available (used by has_feature API)
2607 sub shared_nodes {
2608     my ($conf, $storecfg) = @_;
2609
2610     my $nodelist = PVE::Cluster::get_nodelist();
2611     my $nodehash = { map { $_ => 1 } @$nodelist };
2612     my $nodename = nodename();
2613
2614     PVE::QemuConfig->foreach_volume($conf, sub {
2615         my ($ds, $drive) = @_;
2616
2617         my $volid = $drive->{file};
2618         return if !$volid;
2619
2620         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2621         if ($storeid) {
2622             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2623             if ($scfg->{disable}) {
2624                 $nodehash = {};
2625             } elsif (my $avail = $scfg->{nodes}) {
2626                 foreach my $node (keys %$nodehash) {
2627                     delete $nodehash->{$node} if !$avail->{$node};
2628                 }
2629             } elsif (!$scfg->{shared}) {
2630                 foreach my $node (keys %$nodehash) {
2631                     delete $nodehash->{$node} if $node ne $nodename
2632                 }
2633             }
2634         }
2635     });
2636
2637     return $nodehash
2638 }
2639
2640 sub check_local_storage_availability {
2641     my ($conf, $storecfg) = @_;
2642
2643     my $nodelist = PVE::Cluster::get_nodelist();
2644     my $nodehash = { map { $_ => {} } @$nodelist };
2645
2646     PVE::QemuConfig->foreach_volume($conf, sub {
2647         my ($ds, $drive) = @_;
2648
2649         my $volid = $drive->{file};
2650         return if !$volid;
2651
2652         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2653         if ($storeid) {
2654             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2655
2656             if ($scfg->{disable}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2659                 }
2660             } elsif (my $avail = $scfg->{nodes}) {
2661                 foreach my $node (keys %$nodehash) {
2662                     if (!$avail->{$node}) {
2663                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2664                     }
2665                 }
2666             }
2667         }
2668     });
2669
2670     foreach my $node (values %$nodehash) {
2671         if (my $unavail = $node->{unavailable_storages}) {
2672             $node->{unavailable_storages} = [ sort keys %$unavail ];
2673         }
2674     }
2675
2676     return $nodehash
2677 }
2678
2679 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2680 sub check_running {
2681     my ($vmid, $nocheck, $node) = @_;
2682
2683     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2684     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2685 }
2686
2687 sub vzlist {
2688
2689     my $vzlist = config_list();
2690
2691     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2692
2693     while (defined(my $de = $fd->read)) {
2694         next if $de !~ m/^(\d+)\.pid$/;
2695         my $vmid = $1;
2696         next if !defined($vzlist->{$vmid});
2697         if (my $pid = check_running($vmid)) {
2698             $vzlist->{$vmid}->{pid} = $pid;
2699         }
2700     }
2701
2702     return $vzlist;
2703 }
2704
2705 our $vmstatus_return_properties = {
2706     vmid => get_standard_option('pve-vmid'),
2707     status => {
2708         description => "Qemu process status.",
2709         type => 'string',
2710         enum => ['stopped', 'running'],
2711     },
2712     maxmem => {
2713         description => "Maximum memory in bytes.",
2714         type => 'integer',
2715         optional => 1,
2716         renderer => 'bytes',
2717     },
2718     maxdisk => {
2719         description => "Root disk size in bytes.",
2720         type => 'integer',
2721         optional => 1,
2722         renderer => 'bytes',
2723     },
2724     name => {
2725         description => "VM name.",
2726         type => 'string',
2727         optional => 1,
2728     },
2729     qmpstatus => {
2730         description => "Qemu QMP agent status.",
2731         type => 'string',
2732         optional => 1,
2733     },
2734     pid => {
2735         description => "PID of running qemu process.",
2736         type => 'integer',
2737         optional => 1,
2738     },
2739     uptime => {
2740         description => "Uptime.",
2741         type => 'integer',
2742         optional => 1,
2743         renderer => 'duration',
2744     },
2745     cpus => {
2746         description => "Maximum usable CPUs.",
2747         type => 'number',
2748         optional => 1,
2749     },
2750     lock => {
2751         description => "The current config lock, if any.",
2752         type => 'string',
2753         optional => 1,
2754     },
2755     tags => {
2756         description  => "The current configured tags, if any",
2757         type => 'string',
2758         optional => 1,
2759     },
2760     'running-machine' => {
2761         description => "The currently running machine type (if running).",
2762         type => 'string',
2763         optional => 1,
2764     },
2765     'running-qemu' => {
2766         description => "The currently running QEMU version (if running).",
2767         type => 'string',
2768         optional => 1,
2769     },
2770 };
2771
2772 my $last_proc_pid_stat;
2773
2774 # get VM status information
2775 # This must be fast and should not block ($full == false)
2776 # We only query KVM using QMP if $full == true (this can be slow)
2777 sub vmstatus {
2778     my ($opt_vmid, $full) = @_;
2779
2780     my $res = {};
2781
2782     my $storecfg = PVE::Storage::config();
2783
2784     my $list = vzlist();
2785     my $defaults = load_defaults();
2786
2787     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2788
2789     my $cpucount = $cpuinfo->{cpus} || 1;
2790
2791     foreach my $vmid (keys %$list) {
2792         next if $opt_vmid && ($vmid ne $opt_vmid);
2793
2794         my $conf = PVE::QemuConfig->load_config($vmid);
2795
2796         my $d = { vmid => int($vmid) };
2797         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2798
2799         # fixme: better status?
2800         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2801
2802         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2803         if (defined($size)) {
2804             $d->{disk} = 0; # no info available
2805             $d->{maxdisk} = $size;
2806         } else {
2807             $d->{disk} = 0;
2808             $d->{maxdisk} = 0;
2809         }
2810
2811         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2812             * ($conf->{cores} || $defaults->{cores});
2813         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2814         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2815
2816         $d->{name} = $conf->{name} || "VM $vmid";
2817         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2818             : $defaults->{memory}*(1024*1024);
2819
2820         if ($conf->{balloon}) {
2821             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2822             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2823                 : $defaults->{shares};
2824         }
2825
2826         $d->{uptime} = 0;
2827         $d->{cpu} = 0;
2828         $d->{mem} = 0;
2829
2830         $d->{netout} = 0;
2831         $d->{netin} = 0;
2832
2833         $d->{diskread} = 0;
2834         $d->{diskwrite} = 0;
2835
2836         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2837
2838         $d->{serial} = 1 if conf_has_serial($conf);
2839         $d->{lock} = $conf->{lock} if $conf->{lock};
2840         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2841
2842         $res->{$vmid} = $d;
2843     }
2844
2845     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2846     foreach my $dev (keys %$netdev) {
2847         next if $dev !~ m/^tap([1-9]\d*)i/;
2848         my $vmid = $1;
2849         my $d = $res->{$vmid};
2850         next if !$d;
2851
2852         $d->{netout} += $netdev->{$dev}->{receive};
2853         $d->{netin} += $netdev->{$dev}->{transmit};
2854
2855         if ($full) {
2856             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2857             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2858         }
2859
2860     }
2861
2862     my $ctime = gettimeofday;
2863
2864     foreach my $vmid (keys %$list) {
2865
2866         my $d = $res->{$vmid};
2867         my $pid = $d->{pid};
2868         next if !$pid;
2869
2870         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2871         next if !$pstat; # not running
2872
2873         my $used = $pstat->{utime} + $pstat->{stime};
2874
2875         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2876
2877         if ($pstat->{vsize}) {
2878             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2879         }
2880
2881         my $old = $last_proc_pid_stat->{$pid};
2882         if (!$old) {
2883             $last_proc_pid_stat->{$pid} = {
2884                 time => $ctime,
2885                 used => $used,
2886                 cpu => 0,
2887             };
2888             next;
2889         }
2890
2891         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2892
2893         if ($dtime > 1000) {
2894             my $dutime = $used -  $old->{used};
2895
2896             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2897             $last_proc_pid_stat->{$pid} = {
2898                 time => $ctime,
2899                 used => $used,
2900                 cpu => $d->{cpu},
2901             };
2902         } else {
2903             $d->{cpu} = $old->{cpu};
2904         }
2905     }
2906
2907     return $res if !$full;
2908
2909     my $qmpclient = PVE::QMPClient->new();
2910
2911     my $ballooncb = sub {
2912         my ($vmid, $resp) = @_;
2913
2914         my $info = $resp->{'return'};
2915         return if !$info->{max_mem};
2916
2917         my $d = $res->{$vmid};
2918
2919         # use memory assigned to VM
2920         $d->{maxmem} = $info->{max_mem};
2921         $d->{balloon} = $info->{actual};
2922
2923         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2924             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2925             $d->{freemem} = $info->{free_mem};
2926         }
2927
2928         $d->{ballooninfo} = $info;
2929     };
2930
2931     my $blockstatscb = sub {
2932         my ($vmid, $resp) = @_;
2933         my $data = $resp->{'return'} || [];
2934         my $totalrdbytes = 0;
2935         my $totalwrbytes = 0;
2936
2937         for my $blockstat (@$data) {
2938             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2939             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2940
2941             $blockstat->{device} =~ s/drive-//;
2942             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2943         }
2944         $res->{$vmid}->{diskread} = $totalrdbytes;
2945         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2946     };
2947
2948     my $machinecb = sub {
2949         my ($vmid, $resp) = @_;
2950         my $data = $resp->{'return'} || [];
2951
2952         $res->{$vmid}->{'running-machine'} =
2953             PVE::QemuServer::Machine::current_from_query_machines($data);
2954     };
2955
2956     my $versioncb = sub {
2957         my ($vmid, $resp) = @_;
2958         my $data = $resp->{'return'} // {};
2959         my $version = 'unknown';
2960
2961         if (my $v = $data->{qemu}) {
2962             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2963         }
2964
2965         $res->{$vmid}->{'running-qemu'} = $version;
2966     };
2967
2968     my $statuscb = sub {
2969         my ($vmid, $resp) = @_;
2970
2971         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2972         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2973         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2974         # this fails if ballon driver is not loaded, so this must be
2975         # the last commnand (following command are aborted if this fails).
2976         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2977
2978         my $status = 'unknown';
2979         if (!defined($status = $resp->{'return'}->{status})) {
2980             warn "unable to get VM status\n";
2981             return;
2982         }
2983
2984         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2985     };
2986
2987     foreach my $vmid (keys %$list) {
2988         next if $opt_vmid && ($vmid ne $opt_vmid);
2989         next if !$res->{$vmid}->{pid}; # not running
2990         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2991     }
2992
2993     $qmpclient->queue_execute(undef, 2);
2994
2995     foreach my $vmid (keys %$list) {
2996         next if $opt_vmid && ($vmid ne $opt_vmid);
2997         next if !$res->{$vmid}->{pid}; #not running
2998
2999         # we can't use the $qmpclient since it might have already aborted on
3000         # 'query-balloon', but this might also fail for older versions...
3001         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3002         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3003     }
3004
3005     foreach my $vmid (keys %$list) {
3006         next if $opt_vmid && ($vmid ne $opt_vmid);
3007         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3008     }
3009
3010     return $res;
3011 }
3012
3013 sub conf_has_serial {
3014     my ($conf) = @_;
3015
3016     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3017         if ($conf->{"serial$i"}) {
3018             return 1;
3019         }
3020     }
3021
3022     return 0;
3023 }
3024
3025 sub conf_has_audio {
3026     my ($conf, $id) = @_;
3027
3028     $id //= 0;
3029     my $audio = $conf->{"audio$id"};
3030     return if !defined($audio);
3031
3032     my $audioproperties = parse_property_string($audio_fmt, $audio);
3033     my $audiodriver = $audioproperties->{driver} // 'spice';
3034
3035     return {
3036         dev => $audioproperties->{device},
3037         dev_id => "audiodev$id",
3038         backend => $audiodriver,
3039         backend_id => "$audiodriver-backend${id}",
3040     };
3041 }
3042
3043 sub audio_devs {
3044     my ($audio, $audiopciaddr, $machine_version) = @_;
3045
3046     my $devs = [];
3047
3048     my $id = $audio->{dev_id};
3049     my $audiodev = "";
3050     if (min_version($machine_version, 4, 2)) {
3051         $audiodev = ",audiodev=$audio->{backend_id}";
3052     }
3053
3054     if ($audio->{dev} eq 'AC97') {
3055         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3056     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3057         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3058         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3059         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3060     } else {
3061         die "unkown audio device '$audio->{dev}', implement me!";
3062     }
3063
3064     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3065
3066     return $devs;
3067 }
3068
3069 sub get_tpm_paths {
3070     my ($vmid) = @_;
3071     return {
3072         socket => "/var/run/qemu-server/$vmid.swtpm",
3073         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3074     };
3075 }
3076
3077 sub add_tpm_device {
3078     my ($vmid, $devices, $conf) = @_;
3079
3080     return if !$conf->{tpmstate0};
3081
3082     my $paths = get_tpm_paths($vmid);
3083
3084     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3085     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3086     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3087 }
3088
3089 sub start_swtpm {
3090     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3091
3092     return if !$tpmdrive;
3093
3094     my $state;
3095     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3096     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3097     if ($storeid) {
3098         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3099     } else {
3100         $state = $tpm->{file};
3101     }
3102
3103     my $paths = get_tpm_paths($vmid);
3104
3105     # during migration, we will get state from remote
3106     #
3107     if (!$migration) {
3108         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3109         my $setup_cmd = [
3110             "swtpm_setup",
3111             "--tpmstate",
3112             "file://$state",
3113             "--createek",
3114             "--create-ek-cert",
3115             "--create-platform-cert",
3116             "--lock-nvram",
3117             "--config",
3118             "/etc/swtpm_setup.conf", # do not use XDG configs
3119             "--runas",
3120             "0", # force creation as root, error if not possible
3121             "--not-overwrite", # ignore existing state, do not modify
3122         ];
3123
3124         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3125         # TPM 2.0 supports ECC crypto, use if possible
3126         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3127
3128         run_command($setup_cmd, outfunc => sub {
3129             print "swtpm_setup: $1\n";
3130         });
3131     }
3132
3133     my $emulator_cmd = [
3134         "swtpm",
3135         "socket",
3136         "--tpmstate",
3137         "backend-uri=file://$state,mode=0600",
3138         "--ctrl",
3139         "type=unixio,path=$paths->{socket},mode=0600",
3140         "--pid",
3141         "file=$paths->{pid}",
3142         "--terminate", # terminate on QEMU disconnect
3143         "--daemon",
3144     ];
3145     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3146     run_command($emulator_cmd, outfunc => sub { print $1; });
3147
3148     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3149     while (! -e $paths->{pid}) {
3150         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3151         usleep(50_000);
3152     }
3153
3154     # return untainted PID of swtpm daemon so it can be killed on error
3155     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3156     return $1;
3157 }
3158
3159 sub vga_conf_has_spice {
3160     my ($vga) = @_;
3161
3162     my $vgaconf = parse_vga($vga);
3163     my $vgatype = $vgaconf->{type};
3164     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3165
3166     return $1 || 1;
3167 }
3168
3169 sub is_native($) {
3170     my ($arch) = @_;
3171     return get_host_arch() eq $arch;
3172 }
3173
3174 sub get_vm_arch {
3175     my ($conf) = @_;
3176     return $conf->{arch} // get_host_arch();
3177 }
3178
3179 my $default_machines = {
3180     x86_64 => 'pc',
3181     aarch64 => 'virt',
3182 };
3183
3184 sub get_installed_machine_version {
3185     my ($kvmversion) = @_;
3186     $kvmversion = kvm_user_version() if !defined($kvmversion);
3187     $kvmversion =~ m/^(\d+\.\d+)/;
3188     return $1;
3189 }
3190
3191 sub windows_get_pinned_machine_version {
3192     my ($machine, $base_version, $kvmversion) = @_;
3193
3194     my $pin_version = $base_version;
3195     if (!defined($base_version) ||
3196         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3197     ) {
3198         $pin_version = get_installed_machine_version($kvmversion);
3199     }
3200     if (!$machine || $machine eq 'pc') {
3201         $machine = "pc-i440fx-$pin_version";
3202     } elsif ($machine eq 'q35') {
3203         $machine = "pc-q35-$pin_version";
3204     } elsif ($machine eq 'virt') {
3205         $machine = "virt-$pin_version";
3206     } else {
3207         warn "unknown machine type '$machine', not touching that!\n";
3208     }
3209
3210     return $machine;
3211 }
3212
3213 sub get_vm_machine {
3214     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3215
3216     my $machine = $forcemachine || $conf->{machine};
3217
3218     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3219         $kvmversion //= kvm_user_version();
3220         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3221         # layout which confuses windows quite a bit and may result in various regressions..
3222         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3223         if (windows_version($conf->{ostype})) {
3224             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3225         }
3226         $arch //= 'x86_64';
3227         $machine ||= $default_machines->{$arch};
3228         if ($add_pve_version) {
3229             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3230             $machine .= "+pve$pvever";
3231         }
3232     }
3233
3234     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3235         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3236         $machine = $1 if $is_pxe;
3237
3238         # for version-pinned machines that do not include a pve-version (e.g.
3239         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3240         $machine .= '+pve0';
3241
3242         $machine .= '.pxe' if $is_pxe;
3243     }
3244
3245     return $machine;
3246 }
3247
3248 sub get_ovmf_files($$$) {
3249     my ($arch, $efidisk, $smm) = @_;
3250
3251     my $types = $OVMF->{$arch}
3252         or die "no OVMF images known for architecture '$arch'\n";
3253
3254     my $type = 'default';
3255     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3256         $type = $smm ? "4m" : "4m-no-smm";
3257         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3258     }
3259
3260     return $types->{$type}->@*;
3261 }
3262
3263 my $Arch2Qemu = {
3264     aarch64 => '/usr/bin/qemu-system-aarch64',
3265     x86_64 => '/usr/bin/qemu-system-x86_64',
3266 };
3267 sub get_command_for_arch($) {
3268     my ($arch) = @_;
3269     return '/usr/bin/kvm' if is_native($arch);
3270
3271     my $cmd = $Arch2Qemu->{$arch}
3272         or die "don't know how to emulate architecture '$arch'\n";
3273     return $cmd;
3274 }
3275
3276 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3277 # to use in a QEMU command line (-cpu element), first array_intersect the result
3278 # of query_supported_ with query_understood_. This is necessary because:
3279 #
3280 # a) query_understood_ returns flags the host cannot use and
3281 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3282 #    flags, but CPU settings - with most of them being flags. Those settings
3283 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3284 #
3285 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3286 # expensive. If you need the value returned from this, you can get it much
3287 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3288 # $accel being 'kvm' or 'tcg'.
3289 #
3290 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3291 # changes, automatically populating pmxcfs.
3292 #
3293 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3294 # since kvm and tcg machines support different flags
3295 #
3296 sub query_supported_cpu_flags {
3297     my ($arch) = @_;
3298
3299     $arch //= get_host_arch();
3300     my $default_machine = $default_machines->{$arch};
3301
3302     my $flags = {};
3303
3304     # FIXME: Once this is merged, the code below should work for ARM as well:
3305     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3306     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3307         $arch eq "aarch64";
3308
3309     my $kvm_supported = defined(kvm_version());
3310     my $qemu_cmd = get_command_for_arch($arch);
3311     my $fakevmid = -1;
3312     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3313
3314     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3315     my $query_supported_run_qemu = sub {
3316         my ($kvm) = @_;
3317
3318         my $flags = {};
3319         my $cmd = [
3320             $qemu_cmd,
3321             '-machine', $default_machine,
3322             '-display', 'none',
3323             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3324             '-mon', 'chardev=qmp,mode=control',
3325             '-pidfile', $pidfile,
3326             '-S', '-daemonize'
3327         ];
3328
3329         if (!$kvm) {
3330             push @$cmd, '-accel', 'tcg';
3331         }
3332
3333         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3334         die "QEMU flag querying VM exited with code " . $rc if $rc;
3335
3336         eval {
3337             my $cmd_result = mon_cmd(
3338                 $fakevmid,
3339                 'query-cpu-model-expansion',
3340                 type => 'full',
3341                 model => { name => 'host' }
3342             );
3343
3344             my $props = $cmd_result->{model}->{props};
3345             foreach my $prop (keys %$props) {
3346                 next if $props->{$prop} ne '1';
3347                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3348                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3349                 # We only keep those with underscores, to match /proc/cpuinfo
3350                 $prop =~ s/\.|-/_/g;
3351                 $flags->{$prop} = 1;
3352             }
3353         };
3354         my $err = $@;
3355
3356         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3357         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3358
3359         die $err if $err;
3360
3361         return [ sort keys %$flags ];
3362     };
3363
3364     # We need to query QEMU twice, since KVM and TCG have different supported flags
3365     PVE::QemuConfig->lock_config($fakevmid, sub {
3366         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3367         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3368
3369         if ($kvm_supported) {
3370             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3371             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3372         }
3373     });
3374
3375     return $flags;
3376 }
3377
3378 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3379 my $understood_cpu_flag_dir = "/usr/share/kvm";
3380 sub query_understood_cpu_flags {
3381     my $arch = get_host_arch();
3382     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3383
3384     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3385         if ! -e $filepath;
3386
3387     my $raw = file_get_contents($filepath);
3388     $raw =~ s/^\s+|\s+$//g;
3389     my @flags = split(/\s+/, $raw);
3390
3391     return \@flags;
3392 }
3393
3394 my sub get_cpuunits {
3395     my ($conf) = @_;
3396     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3397
3398     my $cpuunits = $conf->{cpuunits};
3399     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3400
3401     if ($is_cgroupv2) {
3402         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3403     } else {
3404         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3405     }
3406     return $cpuunits;
3407 }
3408
3409 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3410 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3411 my sub should_disable_smm {
3412     my ($conf, $vga) = @_;
3413
3414     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3415         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3416 }
3417
3418 sub config_to_command {
3419     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3420         $pbs_backing) = @_;
3421
3422     my $cmd = [];
3423     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3424     my $devices = [];
3425     my $bridges = {};
3426     my $ostype = $conf->{ostype};
3427     my $winversion = windows_version($ostype);
3428     my $kvm = $conf->{kvm};
3429     my $nodename = nodename();
3430
3431     my $arch = get_vm_arch($conf);
3432     my $kvm_binary = get_command_for_arch($arch);
3433     my $kvmver = kvm_user_version($kvm_binary);
3434
3435     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3436         $kvmver //= "undefined";
3437         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3438     }
3439
3440     my $add_pve_version = min_version($kvmver, 4, 1);
3441
3442     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3443     my $machine_version = extract_version($machine_type, $kvmver);
3444     $kvm //= 1 if is_native($arch);
3445
3446     $machine_version =~ m/(\d+)\.(\d+)/;
3447     my ($machine_major, $machine_minor) = ($1, $2);
3448
3449     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3450         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3451     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3452         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3453             ." please upgrade node '$nodename'\n"
3454     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3455         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3456         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3457             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3458             ." node '$nodename'\n";
3459     }
3460
3461     # if a specific +pve version is required for a feature, use $version_guard
3462     # instead of min_version to allow machines to be run with the minimum
3463     # required version
3464     my $required_pve_version = 0;
3465     my $version_guard = sub {
3466         my ($major, $minor, $pve) = @_;
3467         return 0 if !min_version($machine_version, $major, $minor, $pve);
3468         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3469         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3470         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3471         return 1;
3472     };
3473
3474     if ($kvm && !defined kvm_version()) {
3475         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3476             ." or enable in BIOS.\n";
3477     }
3478
3479     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3480     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3481     my $use_old_bios_files = undef;
3482     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3483
3484     push @$cmd, $kvm_binary;
3485
3486     push @$cmd, '-id', $vmid;
3487
3488     my $vmname = $conf->{name} || "vm$vmid";
3489
3490     push @$cmd, '-name', $vmname;
3491
3492     push @$cmd, '-no-shutdown';
3493
3494     my $use_virtio = 0;
3495
3496     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3497     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3498     push @$cmd, '-mon', "chardev=qmp,mode=control";
3499
3500     if (min_version($machine_version, 2, 12)) {
3501         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3502         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3503     }
3504
3505     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3506
3507     push @$cmd, '-daemonize';
3508
3509     if ($conf->{smbios1}) {
3510         my $smbios_conf = parse_smbios1($conf->{smbios1});
3511         if ($smbios_conf->{base64}) {
3512             # Do not pass base64 flag to qemu
3513             delete $smbios_conf->{base64};
3514             my $smbios_string = "";
3515             foreach my $key (keys %$smbios_conf) {
3516                 my $value;
3517                 if ($key eq "uuid") {
3518                     $value = $smbios_conf->{uuid}
3519                 } else {
3520                     $value = decode_base64($smbios_conf->{$key});
3521                 }
3522                 # qemu accepts any binary data, only commas need escaping by double comma
3523                 $value =~ s/,/,,/g;
3524                 $smbios_string .= "," . $key . "=" . $value if $value;
3525             }
3526             push @$cmd, '-smbios', "type=1" . $smbios_string;
3527         } else {
3528             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3529         }
3530     }
3531
3532     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3533         my $d;
3534         if (my $efidisk = $conf->{efidisk0}) {
3535             $d = parse_drive('efidisk0', $efidisk);
3536         }
3537
3538         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3539         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3540
3541         my ($path, $format);
3542         my $read_only_str = '';
3543         if ($d) {
3544             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3545             $format = $d->{format};
3546             if ($storeid) {
3547                 $path = PVE::Storage::path($storecfg, $d->{file});
3548                 if (!defined($format)) {
3549                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3550                     $format = qemu_img_format($scfg, $volname);
3551                 }
3552             } else {
3553                 $path = $d->{file};
3554                 die "efidisk format must be specified\n"
3555                     if !defined($format);
3556             }
3557
3558             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3559         } else {
3560             warn "no efidisk configured! Using temporary efivars disk.\n";
3561             $path = "/tmp/$vmid-ovmf.fd";
3562             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3563             $format = 'raw';
3564         }
3565
3566         my $size_str = "";
3567
3568         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3569             $size_str = ",size=" . (-s $ovmf_vars);
3570         }
3571
3572         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3573         my $cache = "";
3574         if ($path =~ m/^rbd:/) {
3575                 $cache = ',cache=writeback';
3576                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3577         }
3578
3579         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3580         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3581     }
3582
3583     if ($q35) { # tell QEMU to load q35 config early
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3593         push @$cmd, $fixups->@*;
3594     }
3595
3596     if ($conf->{vmgenid}) {
3597         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3598     }
3599
3600     # add usb controllers
3601     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3602         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3603     push @$devices, @usbcontrollers if @usbcontrollers;
3604     my $vga = parse_vga($conf->{vga});
3605
3606     my $qxlnum = vga_conf_has_spice($conf->{vga});
3607     $vga->{type} = 'qxl' if $qxlnum;
3608
3609     if (!$vga->{type}) {
3610         if ($arch eq 'aarch64') {
3611             $vga->{type} = 'virtio';
3612         } elsif (min_version($machine_version, 2, 9)) {
3613             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3614         } else {
3615             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3616         }
3617     }
3618
3619     # enable absolute mouse coordinates (needed by vnc)
3620     my $tablet = $conf->{tablet};
3621     if (!defined($tablet)) {
3622         $tablet = $defaults->{tablet};
3623         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3624         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3625     }
3626
3627     if ($tablet) {
3628         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3629         my $kbd = print_keyboarddevice_full($conf, $arch);
3630         push @$devices, '-device', $kbd if defined($kbd);
3631     }
3632
3633     my $bootorder = device_bootorder($conf);
3634
3635     # host pci device passthrough
3636     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3637         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3638
3639     # usb devices
3640     my $usb_dev_features = {};
3641     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3642
3643     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3644         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3645     push @$devices, @usbdevices if @usbdevices;
3646
3647     # serial devices
3648     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3649         my $path = $conf->{"serial$i"} or next;
3650         if ($path eq 'socket') {
3651             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3652             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3653             # On aarch64, serial0 is the UART device. Qemu only allows
3654             # connecting UART devices via the '-serial' command line, as
3655             # the device has a fixed slot on the hardware...
3656             if ($arch eq 'aarch64' && $i == 0) {
3657                 push @$devices, '-serial', "chardev:serial$i";
3658             } else {
3659                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3660             }
3661         } else {
3662             die "no such serial device\n" if ! -c $path;
3663             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3664             push @$devices, '-device', "isa-serial,chardev=serial$i";
3665         }
3666     }
3667
3668     # parallel devices
3669     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3670         if (my $path = $conf->{"parallel$i"}) {
3671             die "no such parallel device\n" if ! -c $path;
3672             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3673             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3674             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3675         }
3676     }
3677
3678     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3679         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3680         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3681         push @$devices, @$audio_devs;
3682     }
3683
3684     add_tpm_device($vmid, $devices, $conf);
3685
3686     my $sockets = 1;
3687     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3688     $sockets = $conf->{sockets} if  $conf->{sockets};
3689
3690     my $cores = $conf->{cores} || 1;
3691
3692     my $maxcpus = $sockets * $cores;
3693
3694     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3695
3696     my $allowed_vcpus = $cpuinfo->{cpus};
3697
3698     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3699
3700     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3701         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3702         for (my $i = 2; $i <= $vcpus; $i++)  {
3703             my $cpustr = print_cpu_device($conf,$i);
3704             push @$cmd, '-device', $cpustr;
3705         }
3706
3707     } else {
3708
3709         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3710     }
3711     push @$cmd, '-nodefaults';
3712
3713     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3714
3715     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3716
3717     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3718
3719     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3720         push @$devices, '-device', print_vga_device(
3721             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3722         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3723         push @$cmd,  '-vnc', "unix:$socket,password=on";
3724     } else {
3725         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3726         push @$cmd, '-nographic';
3727     }
3728
3729     # time drift fix
3730     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3731     my $useLocaltime = $conf->{localtime};
3732
3733     if ($winversion >= 5) { # windows
3734         $useLocaltime = 1 if !defined($conf->{localtime});
3735
3736         # use time drift fix when acpi is enabled
3737         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3738             $tdf = 1 if !defined($conf->{tdf});
3739         }
3740     }
3741
3742     if ($winversion >= 6) {
3743         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3744         push @$cmd, '-no-hpet';
3745     }
3746
3747     push @$rtcFlags, 'driftfix=slew' if $tdf;
3748
3749     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3750         push @$rtcFlags, "base=$conf->{startdate}";
3751     } elsif ($useLocaltime) {
3752         push @$rtcFlags, 'base=localtime';
3753     }
3754
3755     if ($forcecpu) {
3756         push @$cmd, '-cpu', $forcecpu;
3757     } else {
3758         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3759     }
3760
3761     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3762
3763     push @$cmd, '-S' if $conf->{freeze};
3764
3765     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3766
3767     my $guest_agent = parse_guest_agent($conf);
3768
3769     if ($guest_agent->{enabled}) {
3770         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3771         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3772
3773         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3774             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3775             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3776             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3777         } elsif ($guest_agent->{type} eq 'isa') {
3778             push @$devices, '-device', "isa-serial,chardev=qga0";
3779         }
3780     }
3781
3782     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3783     if ($rng && $version_guard->(4, 1, 2)) {
3784         check_rng_source($rng->{source});
3785
3786         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3787         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3788         my $limiter_str = "";
3789         if ($max_bytes) {
3790             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3791         }
3792
3793         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3794         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3795         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3796     }
3797
3798     my $spice_port;
3799
3800     if ($qxlnum) {
3801         if ($qxlnum > 1) {
3802             if ($winversion){
3803                 for (my $i = 1; $i < $qxlnum; $i++){
3804                     push @$devices, '-device', print_vga_device(
3805                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3806                 }
3807             } else {
3808                 # assume other OS works like Linux
3809                 my ($ram, $vram) = ("134217728", "67108864");
3810                 if ($vga->{memory}) {
3811                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3812                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3813                 }
3814                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3815                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3816             }
3817         }
3818
3819         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3820
3821         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3822         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3823         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3824
3825         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3826         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3827         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3828
3829         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3830         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3831
3832         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3833         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3834         if ($spice_enhancement->{foldersharing}) {
3835             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3836             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3837         }
3838
3839         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3840         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3841             if $spice_enhancement->{videostreaming};
3842
3843         push @$devices, '-spice', "$spice_opts";
3844     }
3845
3846     # enable balloon by default, unless explicitly disabled
3847     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3848         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3849         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3850     }
3851
3852     if ($conf->{watchdog}) {
3853         my $wdopts = parse_watchdog($conf->{watchdog});
3854         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3855         my $watchdog = $wdopts->{model} || 'i6300esb';
3856         push @$devices, '-device', "$watchdog$pciaddr";
3857         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3858     }
3859
3860     my $vollist = [];
3861     my $scsicontroller = {};
3862     my $ahcicontroller = {};
3863     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3864
3865     # Add iscsi initiator name if available
3866     if (my $initiator = get_initiator_name()) {
3867         push @$devices, '-iscsi', "initiator-name=$initiator";
3868     }
3869
3870     PVE::QemuConfig->foreach_volume($conf, sub {
3871         my ($ds, $drive) = @_;
3872
3873         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3874             check_volume_storage_type($storecfg, $drive->{file});
3875             push @$vollist, $drive->{file};
3876         }
3877
3878         # ignore efidisk here, already added in bios/fw handling code above
3879         return if $drive->{interface} eq 'efidisk';
3880         # similar for TPM
3881         return if $drive->{interface} eq 'tpmstate';
3882
3883         $use_virtio = 1 if $ds =~ m/^virtio/;
3884
3885         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3886
3887         if ($drive->{interface} eq 'virtio'){
3888            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3889         }
3890
3891         if ($drive->{interface} eq 'scsi') {
3892
3893             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3894
3895             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3896                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3897
3898             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3899             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3900
3901             my $iothread = '';
3902             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3903                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3904                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3905             } elsif ($drive->{iothread}) {
3906                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3907             }
3908
3909             my $queues = '';
3910             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3911                 $queues = ",num_queues=$drive->{queues}";
3912             }
3913
3914             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3915                 if !$scsicontroller->{$controller};
3916             $scsicontroller->{$controller}=1;
3917         }
3918
3919         if ($drive->{interface} eq 'sata') {
3920             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3921             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3922             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3923                 if !$ahcicontroller->{$controller};
3924             $ahcicontroller->{$controller}=1;
3925         }
3926
3927         my $pbs_conf = $pbs_backing->{$ds};
3928         my $pbs_name = undef;
3929         if ($pbs_conf) {
3930             $pbs_name = "drive-$ds-pbs";
3931             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3932         }
3933
3934         my $drive_cmd = print_drive_commandline_full(
3935             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3936
3937         # extra protection for templates, but SATA and IDE don't support it..
3938         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3939
3940         push @$devices, '-drive',$drive_cmd;
3941         push @$devices, '-device', print_drivedevice_full(
3942             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3943     });
3944
3945     for (my $i = 0; $i < $MAX_NETS; $i++) {
3946         my $netname = "net$i";
3947
3948         next if !$conf->{$netname};
3949         my $d = parse_net($conf->{$netname});
3950         next if !$d;
3951
3952         $use_virtio = 1 if $d->{model} eq 'virtio';
3953
3954         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3955
3956         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3957         push @$devices, '-netdev', $netdevfull;
3958
3959         my $netdevicefull = print_netdevice_full(
3960             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3961
3962         push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3980             .",size=$ivshmem->{size}M";
3981     }
3982
3983     # pci.4 is nested in pci.1
3984     $bridges->{1} = 1 if $bridges->{4};
3985
3986     if (!$q35) { # add pci bridges
3987         if (min_version($machine_version, 2, 3)) {
3988            $bridges->{1} = 1;
3989            $bridges->{2} = 1;
3990         }
3991         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3992     }
3993
3994     for my $k (sort {$b cmp $a} keys %$bridges) {
3995         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3996
3997         my $k_name = $k;
3998         if ($k == 2 && $legacy_igd) {
3999             $k_name = "$k-igd";
4000         }
4001         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4002         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4003
4004         if ($q35) { # add after -readconfig pve-q35.cfg
4005             splice @$devices, 2, 0, '-device', $devstr;
4006         } else {
4007             unshift @$devices, '-device', $devstr if $k > 0;
4008         }
4009     }
4010
4011     if (!$kvm) {
4012         push @$machineFlags, 'accel=tcg';
4013     }
4014
4015     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4016
4017     my $machine_type_min = $machine_type;
4018     if ($add_pve_version) {
4019         $machine_type_min =~ s/\+pve\d+$//;
4020         $machine_type_min .= "+pve$required_pve_version";
4021     }
4022     push @$machineFlags, "type=${machine_type_min}";
4023
4024     push @$cmd, @$devices;
4025     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4026     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4027     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4028
4029     if (my $vmstate = $conf->{vmstate}) {
4030         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4031         push @$vollist, $vmstate;
4032         push @$cmd, '-loadstate', $statepath;
4033         print "activating and using '$vmstate' as vmstate\n";
4034     }
4035
4036     if (PVE::QemuConfig->is_template($conf)) {
4037         # needed to workaround base volumes being read-only
4038         push @$cmd, '-snapshot';
4039     }
4040
4041     # add custom args
4042     if ($conf->{args}) {
4043         my $aa = PVE::Tools::split_args($conf->{args});
4044         push @$cmd, @$aa;
4045     }
4046
4047     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4048 }
4049
4050 sub check_rng_source {
4051     my ($source) = @_;
4052
4053     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4054     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4055         if ! -e $source;
4056
4057     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4058     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4059         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4060         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4061         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4062             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4063             ." to the host.\n";
4064     }
4065 }
4066
4067 sub spice_port {
4068     my ($vmid) = @_;
4069
4070     my $res = mon_cmd($vmid, 'query-spice');
4071
4072     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4073 }
4074
4075 sub vm_devices_list {
4076     my ($vmid) = @_;
4077
4078     my $res = mon_cmd($vmid, 'query-pci');
4079     my $devices_to_check = [];
4080     my $devices = {};
4081     foreach my $pcibus (@$res) {
4082         push @$devices_to_check, @{$pcibus->{devices}},
4083     }
4084
4085     while (@$devices_to_check) {
4086         my $to_check = [];
4087         for my $d (@$devices_to_check) {
4088             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4089             next if !$d->{'pci_bridge'};
4090
4091             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4092             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4093         }
4094         $devices_to_check = $to_check;
4095     }
4096
4097     my $resblock = mon_cmd($vmid, 'query-block');
4098     foreach my $block (@$resblock) {
4099         if($block->{device} =~ m/^drive-(\S+)/){
4100                 $devices->{$1} = 1;
4101         }
4102     }
4103
4104     my $resmice = mon_cmd($vmid, 'query-mice');
4105     foreach my $mice (@$resmice) {
4106         if ($mice->{name} eq 'QEMU HID Tablet') {
4107             $devices->{tablet} = 1;
4108             last;
4109         }
4110     }
4111
4112     # for usb devices there is no query-usb
4113     # but we can iterate over the entries in
4114     # qom-list path=/machine/peripheral
4115     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4116     foreach my $per (@$resperipheral) {
4117         if ($per->{name} =~ m/^usb\d+$/) {
4118             $devices->{$per->{name}} = 1;
4119         }
4120     }
4121
4122     return $devices;
4123 }
4124
4125 sub vm_deviceplug {
4126     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4127
4128     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4129
4130     my $devices_list = vm_devices_list($vmid);
4131     return 1 if defined($devices_list->{$deviceid});
4132
4133     # add PCI bridge if we need it for the device
4134     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4135
4136     if ($deviceid eq 'tablet') {
4137         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4138     } elsif ($deviceid eq 'keyboard') {
4139         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4140     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4141         die "usb hotplug currently not reliable\n";
4142         # since we can't reliably hot unplug all added usb devices and usb
4143         # passthrough breaks live migration we disable usb hotplugging for now
4144         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4145     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4146         qemu_iothread_add($vmid, $deviceid, $device);
4147
4148         qemu_driveadd($storecfg, $vmid, $device);
4149         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4150
4151         qemu_deviceadd($vmid, $devicefull);
4152         eval { qemu_deviceaddverify($vmid, $deviceid); };
4153         if (my $err = $@) {
4154             eval { qemu_drivedel($vmid, $deviceid); };
4155             warn $@ if $@;
4156             die $err;
4157         }
4158     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4160         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4161         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4162
4163         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4164
4165         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4166             qemu_iothread_add($vmid, $deviceid, $device);
4167             $devicefull .= ",iothread=iothread-$deviceid";
4168         }
4169
4170         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4171             $devicefull .= ",num_queues=$device->{queues}";
4172         }
4173
4174         qemu_deviceadd($vmid, $devicefull);
4175         qemu_deviceaddverify($vmid, $deviceid);
4176     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4177         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4178         qemu_driveadd($storecfg, $vmid, $device);
4179
4180         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4181         eval { qemu_deviceadd($vmid, $devicefull); };
4182         if (my $err = $@) {
4183             eval { qemu_drivedel($vmid, $deviceid); };
4184             warn $@ if $@;
4185             die $err;
4186         }
4187     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4188         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4189
4190         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4191         my $use_old_bios_files = undef;
4192         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4193
4194         my $netdevicefull = print_netdevice_full(
4195             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4196         qemu_deviceadd($vmid, $netdevicefull);
4197         eval {
4198             qemu_deviceaddverify($vmid, $deviceid);
4199             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4200         };
4201         if (my $err = $@) {
4202             eval { qemu_netdevdel($vmid, $deviceid); };
4203             warn $@ if $@;
4204             die $err;
4205         }
4206     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4207         my $bridgeid = $2;
4208         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4209         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4210
4211         qemu_deviceadd($vmid, $devicefull);
4212         qemu_deviceaddverify($vmid, $deviceid);
4213     } else {
4214         die "can't hotplug device '$deviceid'\n";
4215     }
4216
4217     return 1;
4218 }
4219
4220 # fixme: this should raise exceptions on error!
4221 sub vm_deviceunplug {
4222     my ($vmid, $conf, $deviceid) = @_;
4223
4224     my $devices_list = vm_devices_list($vmid);
4225     return 1 if !defined($devices_list->{$deviceid});
4226
4227     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4228     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4229
4230     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4231         qemu_devicedel($vmid, $deviceid);
4232     } elsif ($deviceid =~ m/^usb\d+$/) {
4233         die "usb hotplug currently not reliable\n";
4234         # when unplugging usb devices this way, there may be remaining usb
4235         # controllers/hubs so we disable it for now
4236         #qemu_devicedel($vmid, $deviceid);
4237         #qemu_devicedelverify($vmid, $deviceid);
4238     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4239         my $device = parse_drive($deviceid, $conf->{$deviceid});
4240
4241         qemu_devicedel($vmid, $deviceid);
4242         qemu_devicedelverify($vmid, $deviceid);
4243         qemu_drivedel($vmid, $deviceid);
4244         qemu_iothread_del($vmid, $deviceid, $device);
4245     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4246         qemu_devicedel($vmid, $deviceid);
4247         qemu_devicedelverify($vmid, $deviceid);
4248     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4249         my $device = parse_drive($deviceid, $conf->{$deviceid});
4250
4251         qemu_devicedel($vmid, $deviceid);
4252         qemu_drivedel($vmid, $deviceid);
4253         qemu_deletescsihw($conf, $vmid, $deviceid);
4254
4255         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4256             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4257     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4258         qemu_devicedel($vmid, $deviceid);
4259         qemu_devicedelverify($vmid, $deviceid);
4260         qemu_netdevdel($vmid, $deviceid);
4261     } else {
4262         die "can't unplug device '$deviceid'\n";
4263     }
4264
4265     return 1;
4266 }
4267
4268 sub qemu_deviceadd {
4269     my ($vmid, $devicefull) = @_;
4270
4271     $devicefull = "driver=".$devicefull;
4272     my %options =  split(/[=,]/, $devicefull);
4273
4274     mon_cmd($vmid, "device_add" , %options);
4275 }
4276
4277 sub qemu_devicedel {
4278     my ($vmid, $deviceid) = @_;
4279
4280     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4281 }
4282
4283 sub qemu_iothread_add {
4284     my ($vmid, $deviceid, $device) = @_;
4285
4286     if ($device->{iothread}) {
4287         my $iothreads = vm_iothreads_list($vmid);
4288         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4289     }
4290 }
4291
4292 sub qemu_iothread_del {
4293     my ($vmid, $deviceid, $device) = @_;
4294
4295     if ($device->{iothread}) {
4296         my $iothreads = vm_iothreads_list($vmid);
4297         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4298     }
4299 }
4300
4301 sub qemu_objectadd {
4302     my ($vmid, $objectid, $qomtype) = @_;
4303
4304     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4305
4306     return 1;
4307 }
4308
4309 sub qemu_objectdel {
4310     my ($vmid, $objectid) = @_;
4311
4312     mon_cmd($vmid, "object-del", id => $objectid);
4313
4314     return 1;
4315 }
4316
4317 sub qemu_driveadd {
4318     my ($storecfg, $vmid, $device) = @_;
4319
4320     my $kvmver = get_running_qemu_version($vmid);
4321     my $io_uring = min_version($kvmver, 6, 0);
4322     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4323     $drive =~ s/\\/\\\\/g;
4324     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4325
4326     # If the command succeeds qemu prints: "OK"
4327     return 1 if $ret =~ m/OK/s;
4328
4329     die "adding drive failed: $ret\n";
4330 }
4331
4332 sub qemu_drivedel {
4333     my ($vmid, $deviceid) = @_;
4334
4335     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4336     $ret =~ s/^\s+//;
4337
4338     return 1 if $ret eq "";
4339
4340     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4341     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4342
4343     die "deleting drive $deviceid failed : $ret\n";
4344 }
4345
4346 sub qemu_deviceaddverify {
4347     my ($vmid, $deviceid) = @_;
4348
4349     for (my $i = 0; $i <= 5; $i++) {
4350          my $devices_list = vm_devices_list($vmid);
4351          return 1 if defined($devices_list->{$deviceid});
4352          sleep 1;
4353     }
4354
4355     die "error on hotplug device '$deviceid'\n";
4356 }
4357
4358
4359 sub qemu_devicedelverify {
4360     my ($vmid, $deviceid) = @_;
4361
4362     # need to verify that the device is correctly removed as device_del
4363     # is async and empty return is not reliable
4364
4365     for (my $i = 0; $i <= 5; $i++) {
4366          my $devices_list = vm_devices_list($vmid);
4367          return 1 if !defined($devices_list->{$deviceid});
4368          sleep 1;
4369     }
4370
4371     die "error on hot-unplugging device '$deviceid'\n";
4372 }
4373
4374 sub qemu_findorcreatescsihw {
4375     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4376
4377     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4378
4379     my $scsihwid="$controller_prefix$controller";
4380     my $devices_list = vm_devices_list($vmid);
4381
4382     if (!defined($devices_list->{$scsihwid})) {
4383         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4384     }
4385
4386     return 1;
4387 }
4388
4389 sub qemu_deletescsihw {
4390     my ($conf, $vmid, $opt) = @_;
4391
4392     my $device = parse_drive($opt, $conf->{$opt});
4393
4394     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4395         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4396         return 1;
4397     }
4398
4399     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4400
4401     my $devices_list = vm_devices_list($vmid);
4402     foreach my $opt (keys %{$devices_list}) {
4403         if (is_valid_drivename($opt)) {
4404             my $drive = parse_drive($opt, $conf->{$opt});
4405             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4406                 return 1;
4407             }
4408         }
4409     }
4410
4411     my $scsihwid="scsihw$controller";
4412
4413     vm_deviceunplug($vmid, $conf, $scsihwid);
4414
4415     return 1;
4416 }
4417
4418 sub qemu_add_pci_bridge {
4419     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4420
4421     my $bridges = {};
4422
4423     my $bridgeid;
4424
4425     print_pci_addr($device, $bridges, $arch, $machine_type);
4426
4427     while (my ($k, $v) = each %$bridges) {
4428         $bridgeid = $k;
4429     }
4430     return 1 if !defined($bridgeid) || $bridgeid < 1;
4431
4432     my $bridge = "pci.$bridgeid";
4433     my $devices_list = vm_devices_list($vmid);
4434
4435     if (!defined($devices_list->{$bridge})) {
4436         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4437     }
4438
4439     return 1;
4440 }
4441
4442 sub qemu_set_link_status {
4443     my ($vmid, $device, $up) = @_;
4444
4445     mon_cmd($vmid, "set_link", name => $device,
4446                up => $up ? JSON::true : JSON::false);
4447 }
4448
4449 sub qemu_netdevadd {
4450     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4451
4452     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4453     my %options =  split(/[=,]/, $netdev);
4454
4455     if (defined(my $vhost = $options{vhost})) {
4456         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4457     }
4458
4459     if (defined(my $queues = $options{queues})) {
4460         $options{queues} = $queues + 0;
4461     }
4462
4463     mon_cmd($vmid, "netdev_add",  %options);
4464     return 1;
4465 }
4466
4467 sub qemu_netdevdel {
4468     my ($vmid, $deviceid) = @_;
4469
4470     mon_cmd($vmid, "netdev_del", id => $deviceid);
4471 }
4472
4473 sub qemu_usb_hotplug {
4474     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4475
4476     return if !$device;
4477
4478     # remove the old one first
4479     vm_deviceunplug($vmid, $conf, $deviceid);
4480
4481     # check if xhci controller is necessary and available
4482     if ($device->{usb3}) {
4483
4484         my $devicelist = vm_devices_list($vmid);
4485
4486         if (!$devicelist->{xhci}) {
4487             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4488             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4489         }
4490     }
4491     my $d = parse_usb_device($device->{host});
4492     $d->{usb3} = $device->{usb3};
4493
4494     # add the new one
4495     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4496 }
4497
4498 sub qemu_cpu_hotplug {
4499     my ($vmid, $conf, $vcpus) = @_;
4500
4501     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4502
4503     my $sockets = 1;
4504     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4505     $sockets = $conf->{sockets} if  $conf->{sockets};
4506     my $cores = $conf->{cores} || 1;
4507     my $maxcpus = $sockets * $cores;
4508
4509     $vcpus = $maxcpus if !$vcpus;
4510
4511     die "you can't add more vcpus than maxcpus\n"
4512         if $vcpus > $maxcpus;
4513
4514     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4515
4516     if ($vcpus < $currentvcpus) {
4517
4518         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4519
4520             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4521                 qemu_devicedel($vmid, "cpu$i");
4522                 my $retry = 0;
4523                 my $currentrunningvcpus = undef;
4524                 while (1) {
4525                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4526                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4527                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4528                     $retry++;
4529                     sleep 1;
4530                 }
4531                 #update conf after each succesfull cpu unplug
4532                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4533                 PVE::QemuConfig->write_config($vmid, $conf);
4534             }
4535         } else {
4536             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4537         }
4538
4539         return;
4540     }
4541
4542     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4543     die "vcpus in running vm does not match its configuration\n"
4544         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4545
4546     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4547
4548         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4549             my $cpustr = print_cpu_device($conf, $i);
4550             qemu_deviceadd($vmid, $cpustr);
4551
4552             my $retry = 0;
4553             my $currentrunningvcpus = undef;
4554             while (1) {
4555                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4556                 last if scalar(@{$currentrunningvcpus}) == $i;
4557                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4558                 sleep 1;
4559                 $retry++;
4560             }
4561             #update conf after each succesfull cpu hotplug
4562             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4563             PVE::QemuConfig->write_config($vmid, $conf);
4564         }
4565     } else {
4566
4567         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4568             mon_cmd($vmid, "cpu-add", id => int($i));
4569         }
4570     }
4571 }
4572
4573 sub qemu_block_set_io_throttle {
4574     my ($vmid, $deviceid,
4575         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4576         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4577         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4578         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4579
4580     return if !check_running($vmid) ;
4581
4582     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4583         bps => int($bps),
4584         bps_rd => int($bps_rd),
4585         bps_wr => int($bps_wr),
4586         iops => int($iops),
4587         iops_rd => int($iops_rd),
4588         iops_wr => int($iops_wr),
4589         bps_max => int($bps_max),
4590         bps_rd_max => int($bps_rd_max),
4591         bps_wr_max => int($bps_wr_max),
4592         iops_max => int($iops_max),
4593         iops_rd_max => int($iops_rd_max),
4594         iops_wr_max => int($iops_wr_max),
4595         bps_max_length => int($bps_max_length),
4596         bps_rd_max_length => int($bps_rd_max_length),
4597         bps_wr_max_length => int($bps_wr_max_length),
4598         iops_max_length => int($iops_max_length),
4599         iops_rd_max_length => int($iops_rd_max_length),
4600         iops_wr_max_length => int($iops_wr_max_length),
4601     );
4602
4603 }
4604
4605 sub qemu_block_resize {
4606     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4607
4608     my $running = check_running($vmid);
4609
4610     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4611
4612     return if !$running;
4613
4614     my $padding = (1024 - $size % 1024) % 1024;
4615     $size = $size + $padding;
4616
4617     mon_cmd(
4618         $vmid,
4619         "block_resize",
4620         device => $deviceid,
4621         size => int($size),
4622         timeout => 60,
4623     );
4624 }
4625
4626 sub qemu_volume_snapshot {
4627     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4628
4629     my $running = check_running($vmid);
4630
4631     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4632         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4633     } else {
4634         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4635     }
4636 }
4637
4638 sub qemu_volume_snapshot_delete {
4639     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4640
4641     my $running = check_running($vmid);
4642
4643     if($running) {
4644
4645         $running = undef;
4646         my $conf = PVE::QemuConfig->load_config($vmid);
4647         PVE::QemuConfig->foreach_volume($conf, sub {
4648             my ($ds, $drive) = @_;
4649             $running = 1 if $drive->{file} eq $volid;
4650         });
4651     }
4652
4653     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4654         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4655     } else {
4656         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4657     }
4658 }
4659
4660 sub set_migration_caps {
4661     my ($vmid, $savevm) = @_;
4662
4663     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4664
4665     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4666     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4667
4668     my $cap_ref = [];
4669
4670     my $enabled_cap = {
4671         "auto-converge" => 1,
4672         "xbzrle" => 1,
4673         "x-rdma-pin-all" => 0,
4674         "zero-blocks" => 0,
4675         "compress" => 0,
4676         "dirty-bitmaps" => $dirty_bitmaps,
4677     };
4678
4679     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4680
4681     for my $supported_capability (@$supported_capabilities) {
4682         push @$cap_ref, {
4683             capability => $supported_capability->{capability},
4684             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4685         };
4686     }
4687
4688     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4689 }
4690
4691 sub foreach_volid {
4692     my ($conf, $func, @param) = @_;
4693
4694     my $volhash = {};
4695
4696     my $test_volid = sub {
4697         my ($key, $drive, $snapname) = @_;
4698
4699         my $volid = $drive->{file};
4700         return if !$volid;
4701
4702         $volhash->{$volid}->{cdrom} //= 1;
4703         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4704
4705         my $replicate = $drive->{replicate} // 1;
4706         $volhash->{$volid}->{replicate} //= 0;
4707         $volhash->{$volid}->{replicate} = 1 if $replicate;
4708
4709         $volhash->{$volid}->{shared} //= 0;
4710         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4711
4712         $volhash->{$volid}->{referenced_in_config} //= 0;
4713         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4714
4715         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4716             if defined($snapname);
4717
4718         my $size = $drive->{size};
4719         $volhash->{$volid}->{size} //= $size if $size;
4720
4721         $volhash->{$volid}->{is_vmstate} //= 0;
4722         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4723
4724         $volhash->{$volid}->{is_tpmstate} //= 0;
4725         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4726
4727         $volhash->{$volid}->{is_unused} //= 0;
4728         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4729
4730         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4731     };
4732
4733     my $include_opts = {
4734         extra_keys => ['vmstate'],
4735         include_unused => 1,
4736     };
4737
4738     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4739     foreach my $snapname (keys %{$conf->{snapshots}}) {
4740         my $snap = $conf->{snapshots}->{$snapname};
4741         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4742     }
4743
4744     foreach my $volid (keys %$volhash) {
4745         &$func($volid, $volhash->{$volid}, @param);
4746     }
4747 }
4748
4749 my $fast_plug_option = {
4750     'lock' => 1,
4751     'name' => 1,
4752     'onboot' => 1,
4753     'shares' => 1,
4754     'startup' => 1,
4755     'description' => 1,
4756     'protection' => 1,
4757     'vmstatestorage' => 1,
4758     'hookscript' => 1,
4759     'tags' => 1,
4760 };
4761
4762 # hotplug changes in [PENDING]
4763 # $selection hash can be used to only apply specified options, for
4764 # example: { cores => 1 } (only apply changed 'cores')
4765 # $errors ref is used to return error messages
4766 sub vmconfig_hotplug_pending {
4767     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4768
4769     my $defaults = load_defaults();
4770     my $arch = get_vm_arch($conf);
4771     my $machine_type = get_vm_machine($conf, undef, $arch);
4772
4773     # commit values which do not have any impact on running VM first
4774     # Note: those option cannot raise errors, we we do not care about
4775     # $selection and always apply them.
4776
4777     my $add_error = sub {
4778         my ($opt, $msg) = @_;
4779         $errors->{$opt} = "hotplug problem - $msg";
4780     };
4781
4782     my $changes = 0;
4783     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4784         if ($fast_plug_option->{$opt}) {
4785             $conf->{$opt} = $conf->{pending}->{$opt};
4786             delete $conf->{pending}->{$opt};
4787             $changes = 1;
4788         }
4789     }
4790
4791     if ($changes) {
4792         PVE::QemuConfig->write_config($vmid, $conf);
4793     }
4794
4795     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4796
4797     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4798     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4799     foreach my $opt (sort keys %$pending_delete_hash) {
4800         next if $selection && !$selection->{$opt};
4801         my $force = $pending_delete_hash->{$opt}->{force};
4802         eval {
4803             if ($opt eq 'hotplug') {
4804                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4805             } elsif ($opt eq 'tablet') {
4806                 die "skip\n" if !$hotplug_features->{usb};
4807                 if ($defaults->{tablet}) {
4808                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4809                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4810                         if $arch eq 'aarch64';
4811                 } else {
4812                     vm_deviceunplug($vmid, $conf, 'tablet');
4813                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4814                 }
4815             } elsif ($opt =~ m/^usb\d+/) {
4816                 die "skip\n";
4817                 # since we cannot reliably hot unplug usb devices we are disabling it
4818                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4819                 #vm_deviceunplug($vmid, $conf, $opt);
4820             } elsif ($opt eq 'vcpus') {
4821                 die "skip\n" if !$hotplug_features->{cpu};
4822                 qemu_cpu_hotplug($vmid, $conf, undef);
4823             } elsif ($opt eq 'balloon') {
4824                 # enable balloon device is not hotpluggable
4825                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4826                 # here we reset the ballooning value to memory
4827                 my $balloon = $conf->{memory} || $defaults->{memory};
4828                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4829             } elsif ($fast_plug_option->{$opt}) {
4830                 # do nothing
4831             } elsif ($opt =~ m/^net(\d+)$/) {
4832                 die "skip\n" if !$hotplug_features->{network};
4833                 vm_deviceunplug($vmid, $conf, $opt);
4834             } elsif (is_valid_drivename($opt)) {
4835                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4836                 vm_deviceunplug($vmid, $conf, $opt);
4837                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4838             } elsif ($opt =~ m/^memory$/) {
4839                 die "skip\n" if !$hotplug_features->{memory};
4840                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4841             } elsif ($opt eq 'cpuunits') {
4842                 $cgroup->change_cpu_shares(undef, 1024);
4843             } elsif ($opt eq 'cpulimit') {
4844                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4845             } else {
4846                 die "skip\n";
4847             }
4848         };
4849         if (my $err = $@) {
4850             &$add_error($opt, $err) if $err ne "skip\n";
4851         } else {
4852             delete $conf->{$opt};
4853             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4854         }
4855     }
4856
4857     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4858     $apply_pending_cloudinit = sub {
4859         return if $apply_pending_cloudinit_done; # once is enough
4860         $apply_pending_cloudinit_done = 1; # once is enough
4861
4862         my ($key, $value) = @_;
4863
4864         my @cloudinit_opts = keys %$confdesc_cloudinit;
4865         foreach my $opt (keys %{$conf->{pending}}) {
4866             next if !grep { $_ eq $opt } @cloudinit_opts;
4867             $conf->{$opt} = delete $conf->{pending}->{$opt};
4868         }
4869
4870         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4871         foreach my $opt (sort keys %$pending_delete_hash) {
4872             next if !grep { $_ eq $opt } @cloudinit_opts;
4873             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4874             delete $conf->{$opt};
4875         }
4876
4877         my $new_conf = { %$conf };
4878         $new_conf->{$key} = $value;
4879         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4880     };
4881
4882     foreach my $opt (keys %{$conf->{pending}}) {
4883         next if $selection && !$selection->{$opt};
4884         my $value = $conf->{pending}->{$opt};
4885         eval {
4886             if ($opt eq 'hotplug') {
4887                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4888             } elsif ($opt eq 'tablet') {
4889                 die "skip\n" if !$hotplug_features->{usb};
4890                 if ($value == 1) {
4891                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4892                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4893                         if $arch eq 'aarch64';
4894                 } elsif ($value == 0) {
4895                     vm_deviceunplug($vmid, $conf, 'tablet');
4896                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4897                 }
4898             } elsif ($opt =~ m/^usb\d+$/) {
4899                 die "skip\n";
4900                 # since we cannot reliably hot unplug usb devices we disable it for now
4901                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4902                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4903                 #die "skip\n" if !$d;
4904                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4905             } elsif ($opt eq 'vcpus') {
4906                 die "skip\n" if !$hotplug_features->{cpu};
4907                 qemu_cpu_hotplug($vmid, $conf, $value);
4908             } elsif ($opt eq 'balloon') {
4909                 # enable/disable balloning device is not hotpluggable
4910                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4911                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4912                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4913
4914                 # allow manual ballooning if shares is set to zero
4915                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4916                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4917                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4918                 }
4919             } elsif ($opt =~ m/^net(\d+)$/) {
4920                 # some changes can be done without hotplug
4921                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4922                                     $vmid, $opt, $value, $arch, $machine_type);
4923             } elsif (is_valid_drivename($opt)) {
4924                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4925                 # some changes can be done without hotplug
4926                 my $drive = parse_drive($opt, $value);
4927                 if (drive_is_cloudinit($drive)) {
4928                     &$apply_pending_cloudinit($opt, $value);
4929                 }
4930                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4931                                      $vmid, $opt, $value, $arch, $machine_type);
4932             } elsif ($opt =~ m/^memory$/) { #dimms
4933                 die "skip\n" if !$hotplug_features->{memory};
4934                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4935             } elsif ($opt eq 'cpuunits') {
4936                 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4937                 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4938             } elsif ($opt eq 'cpulimit') {
4939                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4940                 $cgroup->change_cpu_quota($cpulimit, 100000);
4941             } elsif ($opt eq 'agent') {
4942                 vmconfig_update_agent($conf, $opt, $value);
4943             } else {
4944                 die "skip\n";  # skip non-hot-pluggable options
4945             }
4946         };
4947         if (my $err = $@) {
4948             &$add_error($opt, $err) if $err ne "skip\n";
4949         } else {
4950             $conf->{$opt} = $value;
4951             delete $conf->{pending}->{$opt};
4952         }
4953     }
4954
4955     PVE::QemuConfig->write_config($vmid, $conf);
4956 }
4957
4958 sub try_deallocate_drive {
4959     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4960
4961     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4962         my $volid = $drive->{file};
4963         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4964             my $sid = PVE::Storage::parse_volume_id($volid);
4965             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4966
4967             # check if the disk is really unused
4968             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4969                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4970             PVE::Storage::vdisk_free($storecfg, $volid);
4971             return 1;
4972         } else {
4973             # If vm is not owner of this disk remove from config
4974             return 1;
4975         }
4976     }
4977
4978     return;
4979 }
4980
4981 sub vmconfig_delete_or_detach_drive {
4982     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4983
4984     my $drive = parse_drive($opt, $conf->{$opt});
4985
4986     my $rpcenv = PVE::RPCEnvironment::get();
4987     my $authuser = $rpcenv->get_user();
4988
4989     if ($force) {
4990         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4991         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4992     } else {
4993         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4994     }
4995 }
4996
4997
4998
4999 sub vmconfig_apply_pending {
5000     my ($vmid, $conf, $storecfg, $errors) = @_;
5001
5002     return if !scalar(keys %{$conf->{pending}});
5003
5004     my $add_apply_error = sub {
5005         my ($opt, $msg) = @_;
5006         my $err_msg = "unable to apply pending change $opt : $msg";
5007         $errors->{$opt} = $err_msg;
5008         warn $err_msg;
5009     };
5010
5011     # cold plug
5012
5013     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5014     foreach my $opt (sort keys %$pending_delete_hash) {
5015         my $force = $pending_delete_hash->{$opt}->{force};
5016         eval {
5017             if ($opt =~ m/^unused/) {
5018                 die "internal error";
5019             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5020                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5021             }
5022         };
5023         if (my $err = $@) {
5024             $add_apply_error->($opt, $err);
5025         } else {
5026             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5027             delete $conf->{$opt};
5028         }
5029     }
5030
5031     PVE::QemuConfig->cleanup_pending($conf);
5032
5033     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5034         next if $opt eq 'delete'; # just to be sure
5035         eval {
5036             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5037                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5038             }
5039         };
5040         if (my $err = $@) {
5041             $add_apply_error->($opt, $err);
5042         } else {
5043             $conf->{$opt} = delete $conf->{pending}->{$opt};
5044         }
5045     }
5046
5047     # write all changes at once to avoid unnecessary i/o
5048     PVE::QemuConfig->write_config($vmid, $conf);
5049 }
5050
5051 sub vmconfig_update_net {
5052     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5053
5054     my $newnet = parse_net($value);
5055
5056     if ($conf->{$opt}) {
5057         my $oldnet = parse_net($conf->{$opt});
5058
5059         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5060             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5061             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5062             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5063
5064             # for non online change, we try to hot-unplug
5065             die "skip\n" if !$hotplug;
5066             vm_deviceunplug($vmid, $conf, $opt);
5067         } else {
5068
5069             die "internal error" if $opt !~ m/net(\d+)/;
5070             my $iface = "tap${vmid}i$1";
5071
5072             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5073                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5074                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5075                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5076                 PVE::Network::tap_unplug($iface);
5077
5078                 if ($have_sdn) {
5079                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5080                 } else {
5081                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5082                 }
5083             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5084                 # Rate can be applied on its own but any change above needs to
5085                 # include the rate in tap_plug since OVS resets everything.
5086                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5087             }
5088
5089             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5090                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5091             }
5092
5093             return 1;
5094         }
5095     }
5096
5097     if ($hotplug) {
5098         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5099     } else {
5100         die "skip\n";
5101     }
5102 }
5103
5104 sub vmconfig_update_agent {
5105     my ($conf, $opt, $value) = @_;
5106
5107     die "skip\n" if !$conf->{$opt};
5108
5109     my $hotplug_options = { fstrim_cloned_disks => 1 };
5110
5111     my $old_agent = parse_guest_agent($conf);
5112     my $agent = parse_guest_agent({$opt => $value});
5113
5114     for my $option (keys %$agent) { # added/changed options
5115         next if defined($hotplug_options->{$option});
5116         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5117     }
5118
5119     for my $option (keys %$old_agent) { # removed options
5120         next if defined($hotplug_options->{$option});
5121         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5122     }
5123
5124     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5125 }
5126
5127 sub vmconfig_update_disk {
5128     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5129
5130     my $drive = parse_drive($opt, $value);
5131
5132     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5133         my $media = $drive->{media} || 'disk';
5134         my $oldmedia = $old_drive->{media} || 'disk';
5135         die "unable to change media type\n" if $media ne $oldmedia;
5136
5137         if (!drive_is_cdrom($old_drive)) {
5138
5139             if ($drive->{file} ne $old_drive->{file}) {
5140
5141                 die "skip\n" if !$hotplug;
5142
5143                 # unplug and register as unused
5144                 vm_deviceunplug($vmid, $conf, $opt);
5145                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5146
5147             } else {
5148                 # update existing disk
5149
5150                 # skip non hotpluggable value
5151                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5152                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5153                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5154                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5155                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5156                     die "skip\n";
5157                 }
5158
5159                 # apply throttle
5160                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5161                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5162                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5163                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5164                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5165                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5166                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5167                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5168                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5169                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5170                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5171                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5172                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5173                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5174                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5175                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5176                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5177                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5178
5179                     qemu_block_set_io_throttle(
5180                         $vmid,"drive-$opt",
5181                         ($drive->{mbps} || 0)*1024*1024,
5182                         ($drive->{mbps_rd} || 0)*1024*1024,
5183                         ($drive->{mbps_wr} || 0)*1024*1024,
5184                          $drive->{iops} || 0,
5185                          $drive->{iops_rd} || 0,
5186                          $drive->{iops_wr} || 0,
5187                         ($drive->{mbps_max} || 0)*1024*1024,
5188                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5189                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5190                          $drive->{iops_max} || 0,
5191                          $drive->{iops_rd_max} || 0,
5192                          $drive->{iops_wr_max} || 0,
5193                          $drive->{bps_max_length} || 1,
5194                          $drive->{bps_rd_max_length} || 1,
5195                          $drive->{bps_wr_max_length} || 1,
5196                          $drive->{iops_max_length} || 1,
5197                          $drive->{iops_rd_max_length} || 1,
5198                          $drive->{iops_wr_max_length} || 1,
5199                     );
5200
5201                 }
5202
5203                 return 1;
5204             }
5205
5206         } else { # cdrom
5207
5208             if ($drive->{file} eq 'none') {
5209                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5210                 if (drive_is_cloudinit($old_drive)) {
5211                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5212                 }
5213             } else {
5214                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5215
5216                 # force eject if locked
5217                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5218
5219                 if ($path) {
5220                     mon_cmd($vmid, "blockdev-change-medium",
5221                         id => "$opt", filename => "$path");
5222                 }
5223             }
5224
5225             return 1;
5226         }
5227     }
5228
5229     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5230     # hotplug new disks
5231     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5232     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5233 }
5234
5235 # called in locked context by incoming migration
5236 sub vm_migrate_get_nbd_disks {
5237     my ($storecfg, $conf, $replicated_volumes) = @_;
5238
5239     my $local_volumes = {};
5240     PVE::QemuConfig->foreach_volume($conf, sub {
5241         my ($ds, $drive) = @_;
5242
5243         return if drive_is_cdrom($drive);
5244         return if $ds eq 'tpmstate0';
5245
5246         my $volid = $drive->{file};
5247
5248         return if !$volid;
5249
5250         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5251
5252         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5253         return if $scfg->{shared};
5254
5255         # replicated disks re-use existing state via bitmap
5256         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5257         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5258     });
5259     return $local_volumes;
5260 }
5261
5262 # called in locked context by incoming migration
5263 sub vm_migrate_alloc_nbd_disks {
5264     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5265
5266     my $nbd = {};
5267     foreach my $opt (sort keys %$source_volumes) {
5268         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5269
5270         if ($use_existing) {
5271             $nbd->{$opt}->{drivestr} = print_drive($drive);
5272             $nbd->{$opt}->{volid} = $volid;
5273             $nbd->{$opt}->{replicated} = 1;
5274             next;
5275         }
5276
5277         # storage mapping + volname = regular migration
5278         # storage mapping + format = remote migration
5279         # order of precedence, filtered by whether storage supports it:
5280         # 1. explicit requested format
5281         # 2. format of current volume
5282         # 3. default format of storage
5283         if (!$storagemap->{identity}) {
5284             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5285             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5286             if (!$format || !grep { $format eq $_ } @$validFormats) {
5287                 if ($volname) {
5288                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5289                     my $fileFormat = qemu_img_format($scfg, $volname);
5290                     $format = $fileFormat
5291                         if grep { $fileFormat eq $_ } @$validFormats;
5292                 }
5293                 $format //= $defFormat;
5294             }
5295         } else {
5296             # can't happen for remote migration, so $volname is always defined
5297             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5298             $format = qemu_img_format($scfg, $volname);
5299         }
5300
5301         my $size = $drive->{size} / 1024;
5302         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5303         my $newdrive = $drive;
5304         $newdrive->{format} = $format;
5305         $newdrive->{file} = $newvolid;
5306         my $drivestr = print_drive($newdrive);
5307         $nbd->{$opt}->{drivestr} = $drivestr;
5308         $nbd->{$opt}->{volid} = $newvolid;
5309     }
5310
5311     return $nbd;
5312 }
5313
5314 # see vm_start_nolock for parameters, additionally:
5315 # migrate_opts:
5316 #   storagemap = parsed storage map for allocating NBD disks
5317 sub vm_start {
5318     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5319
5320     return PVE::QemuConfig->lock_config($vmid, sub {
5321         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5322
5323         die "you can't start a vm if it's a template\n"
5324             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5325
5326         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5327         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5328
5329         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5330
5331         if ($has_backup_lock && $running) {
5332             # a backup is currently running, attempt to start the guest in the
5333             # existing QEMU instance
5334             return vm_resume($vmid);
5335         }
5336
5337         PVE::QemuConfig->check_lock($conf)
5338             if !($params->{skiplock} || $has_suspended_lock);
5339
5340         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5341
5342         die "VM $vmid already running\n" if $running;
5343
5344         if (my $storagemap = $migrate_opts->{storagemap}) {
5345             my $replicated = $migrate_opts->{replicated_volumes};
5346             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5347             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5348
5349             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5350                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5351             }
5352         }
5353
5354         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5355     });
5356 }
5357
5358
5359 # params:
5360 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5361 #   skiplock => 0/1, skip checking for config lock
5362 #   skiptemplate => 0/1, skip checking whether VM is template
5363 #   forcemachine => to force Qemu machine (rollback/migration)
5364 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5365 #   timeout => in seconds
5366 #   paused => start VM in paused state (backup)
5367 #   resume => resume from hibernation
5368 #   pbs-backing => {
5369 #      sata0 => {
5370 #         repository
5371 #         snapshot
5372 #         keyfile
5373 #         archive
5374 #      },
5375 #      virtio2 => ...
5376 #   }
5377 # migrate_opts:
5378 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5379 #   migratedfrom => source node
5380 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5381 #   network => CIDR of migration network
5382 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5383 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5384 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5385 #   tpmstate_vol => new volid of tpmstate0, not yet contained in config
5386 sub vm_start_nolock {
5387     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5388
5389     my $statefile = $params->{statefile};
5390     my $resume = $params->{resume};
5391
5392     my $migratedfrom = $migrate_opts->{migratedfrom};
5393     my $migration_type = $migrate_opts->{type};
5394
5395     my $res = {};
5396
5397     # clean up leftover reboot request files
5398     eval { clear_reboot_request($vmid); };
5399     warn $@ if $@;
5400
5401     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5402         vmconfig_apply_pending($vmid, $conf, $storecfg);
5403         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5404     }
5405
5406     # don't regenerate the ISO if the VM is started as part of a live migration
5407     # this way we can reuse the old ISO with the correct config
5408     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5409
5410     # override TPM state vol if migrated, conf is out of date still
5411     if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
5412         my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
5413         $parsed->{file} = $tpmvol;
5414         $conf->{tpmstate0} = print_drive($parsed);
5415     }
5416
5417     my $defaults = load_defaults();
5418
5419     # set environment variable useful inside network script
5420     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5421
5422     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5423
5424     my $forcemachine = $params->{forcemachine};
5425     my $forcecpu = $params->{forcecpu};
5426     if ($resume) {
5427         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5428         $forcemachine = $conf->{runningmachine};
5429         $forcecpu = $conf->{runningcpu};
5430         print "Resuming suspended VM\n";
5431     }
5432
5433     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5434         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5435
5436     my $migration_ip;
5437     my $get_migration_ip = sub {
5438         my ($nodename) = @_;
5439
5440         return $migration_ip if defined($migration_ip);
5441
5442         my $cidr = $migrate_opts->{network};
5443
5444         if (!defined($cidr)) {
5445             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5446             $cidr = $dc_conf->{migration}->{network};
5447         }
5448
5449         if (defined($cidr)) {
5450             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5451
5452             die "could not get IP: no address configured on local " .
5453                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5454
5455             die "could not get IP: multiple addresses configured on local " .
5456                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5457
5458             $migration_ip = @$ips[0];
5459         }
5460
5461         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5462             if !defined($migration_ip);
5463
5464         return $migration_ip;
5465     };
5466
5467     my $migrate_uri;
5468     if ($statefile) {
5469         if ($statefile eq 'tcp') {
5470             my $localip = "localhost";
5471             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5472             my $nodename = nodename();
5473
5474             if (!defined($migration_type)) {
5475                 if (defined($datacenterconf->{migration}->{type})) {
5476                     $migration_type = $datacenterconf->{migration}->{type};
5477                 } else {
5478                     $migration_type = 'secure';
5479                 }
5480             }
5481
5482             if ($migration_type eq 'insecure') {
5483                 $localip = $get_migration_ip->($nodename);
5484                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5485             }
5486
5487             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5488             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5489             $migrate_uri = "tcp:${localip}:${migrate_port}";
5490             push @$cmd, '-incoming', $migrate_uri;
5491             push @$cmd, '-S';
5492
5493         } elsif ($statefile eq 'unix') {
5494             # should be default for secure migrations as a ssh TCP forward
5495             # tunnel is not deterministic reliable ready and fails regurarly
5496             # to set up in time, so use UNIX socket forwards
5497             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5498             unlink $socket_addr;
5499
5500             $migrate_uri = "unix:$socket_addr";
5501
5502             push @$cmd, '-incoming', $migrate_uri;
5503             push @$cmd, '-S';
5504
5505         } elsif (-e $statefile) {
5506             push @$cmd, '-loadstate', $statefile;
5507         } else {
5508             my $statepath = PVE::Storage::path($storecfg, $statefile);
5509             push @$vollist, $statefile;
5510             push @$cmd, '-loadstate', $statepath;
5511         }
5512     } elsif ($params->{paused}) {
5513         push @$cmd, '-S';
5514     }
5515
5516     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5517
5518     my $pci_devices = {}; # host pci devices
5519     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5520         my $dev = $conf->{"hostpci$i"} or next;
5521         $pci_devices->{$i} = parse_hostpci($dev);
5522     }
5523
5524     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5525     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5526
5527     # map to a flat list of pci ids
5528     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5529
5530     # reserve all PCI IDs before actually doing anything with them
5531     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5532
5533     eval {
5534         for my $id (sort keys %$pci_devices) {
5535             my $d = $pci_devices->{$id};
5536             for my $dev ($d->{pciid}->@*) {
5537                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5538             }
5539         }
5540     };
5541     if (my $err = $@) {
5542         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5543         warn $@ if $@;
5544         die $err;
5545     }
5546
5547     PVE::Storage::activate_volumes($storecfg, $vollist);
5548
5549     eval {
5550         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5551     };
5552     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5553     # timeout should be more than enough here...
5554     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5555
5556     my $cpuunits = get_cpuunits($conf);
5557
5558     my %run_params = (
5559         timeout => $statefile ? undef : $start_timeout,
5560         umask => 0077,
5561         noerr => 1,
5562     );
5563
5564     # when migrating, prefix QEMU output so other side can pick up any
5565     # errors that might occur and show the user
5566     if ($migratedfrom) {
5567         $run_params{quiet} = 1;
5568         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5569     }
5570
5571     my %systemd_properties = (
5572         Slice => 'qemu.slice',
5573         KillMode => 'process',
5574         SendSIGKILL => 0,
5575         TimeoutStopUSec => ULONG_MAX, # infinity
5576     );
5577
5578     if (PVE::CGroup::cgroup_mode() == 2) {
5579         $systemd_properties{CPUWeight} = $cpuunits;
5580     } else {
5581         $systemd_properties{CPUShares} = $cpuunits;
5582     }
5583
5584     if (my $cpulimit = $conf->{cpulimit}) {
5585         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5586     }
5587     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5588
5589     my $run_qemu = sub {
5590         PVE::Tools::run_fork sub {
5591             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5592
5593             my $tpmpid;
5594             if (my $tpm = $conf->{tpmstate0}) {
5595                 # start the TPM emulator so QEMU can connect on start
5596                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5597             }
5598
5599             my $exitcode = run_command($cmd, %run_params);
5600             if ($exitcode) {
5601                 if ($tpmpid) {
5602                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5603                     kill 'TERM', $tpmpid;
5604                 }
5605                 die "QEMU exited with code $exitcode\n";
5606             }
5607         };
5608     };
5609
5610     if ($conf->{hugepages}) {
5611
5612         my $code = sub {
5613             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5614             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5615
5616             PVE::QemuServer::Memory::hugepages_mount();
5617             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5618
5619             eval { $run_qemu->() };
5620             if (my $err = $@) {
5621                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5622                     if !$conf->{keephugepages};
5623                 die $err;
5624             }
5625
5626             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5627                 if !$conf->{keephugepages};
5628         };
5629         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5630
5631     } else {
5632         eval { $run_qemu->() };
5633     }
5634
5635     if (my $err = $@) {
5636         # deactivate volumes if start fails
5637         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5638         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5639
5640         die "start failed: $err";
5641     }
5642
5643     # re-reserve all PCI IDs now that we can know the actual VM PID
5644     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5645     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5646     warn $@ if $@;
5647
5648     print "migration listens on $migrate_uri\n" if $migrate_uri;
5649     $res->{migrate_uri} = $migrate_uri;
5650
5651     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5652         eval { mon_cmd($vmid, "cont"); };
5653         warn $@ if $@;
5654     }
5655
5656     #start nbd server for storage migration
5657     if (my $nbd = $migrate_opts->{nbd}) {
5658         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5659
5660         my $migrate_storage_uri;
5661         # nbd_protocol_version > 0 for unix socket support
5662         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5663             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5664             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5665             $migrate_storage_uri = "nbd:unix:$socket_path";
5666         } else {
5667             my $nodename = nodename();
5668             my $localip = $get_migration_ip->($nodename);
5669             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5670             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5671
5672             mon_cmd($vmid, "nbd-server-start", addr => {
5673                 type => 'inet',
5674                 data => {
5675                     host => "${localip}",
5676                     port => "${storage_migrate_port}",
5677                 },
5678             });
5679             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5680             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5681         }
5682
5683         $res->{migrate_storage_uri} = $migrate_storage_uri;
5684
5685         foreach my $opt (sort keys %$nbd) {
5686             my $drivestr = $nbd->{$opt}->{drivestr};
5687             my $volid = $nbd->{$opt}->{volid};
5688             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5689             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5690             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5691             print "re-using replicated volume: $opt - $volid\n"
5692                 if $nbd->{$opt}->{replicated};
5693
5694             $res->{drives}->{$opt} = $nbd->{$opt};
5695             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5696         }
5697     }
5698
5699     if ($migratedfrom) {
5700         eval {
5701             set_migration_caps($vmid);
5702         };
5703         warn $@ if $@;
5704
5705         if ($spice_port) {
5706             print "spice listens on port $spice_port\n";
5707             $res->{spice_port} = $spice_port;
5708             if ($migrate_opts->{spice_ticket}) {
5709                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5710                     $migrate_opts->{spice_ticket});
5711                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5712             }
5713         }
5714
5715     } else {
5716         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5717             if !$statefile && $conf->{balloon};
5718
5719         foreach my $opt (keys %$conf) {
5720             next if $opt !~  m/^net\d+$/;
5721             my $nicconf = parse_net($conf->{$opt});
5722             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5723         }
5724     }
5725
5726     mon_cmd($vmid, 'qom-set',
5727                 path => "machine/peripheral/balloon0",
5728                 property => "guest-stats-polling-interval",
5729                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5730
5731     if ($resume) {
5732         print "Resumed VM, removing state\n";
5733         if (my $vmstate = $conf->{vmstate}) {
5734             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5735             PVE::Storage::vdisk_free($storecfg, $vmstate);
5736         }
5737         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5738         PVE::QemuConfig->write_config($vmid, $conf);
5739     }
5740
5741     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5742
5743     return $res;
5744 }
5745
5746 sub vm_commandline {
5747     my ($storecfg, $vmid, $snapname) = @_;
5748
5749     my $conf = PVE::QemuConfig->load_config($vmid);
5750
5751     my ($forcemachine, $forcecpu);
5752     if ($snapname) {
5753         my $snapshot = $conf->{snapshots}->{$snapname};
5754         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5755
5756         # check for machine or CPU overrides in snapshot
5757         $forcemachine = $snapshot->{runningmachine};
5758         $forcecpu = $snapshot->{runningcpu};
5759
5760         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5761
5762         $conf = $snapshot;
5763     }
5764
5765     my $defaults = load_defaults();
5766
5767     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5768
5769     return PVE::Tools::cmd2string($cmd);
5770 }
5771
5772 sub vm_reset {
5773     my ($vmid, $skiplock) = @_;
5774
5775     PVE::QemuConfig->lock_config($vmid, sub {
5776
5777         my $conf = PVE::QemuConfig->load_config($vmid);
5778
5779         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5780
5781         mon_cmd($vmid, "system_reset");
5782     });
5783 }
5784
5785 sub get_vm_volumes {
5786     my ($conf) = @_;
5787
5788     my $vollist = [];
5789     foreach_volid($conf, sub {
5790         my ($volid, $attr) = @_;
5791
5792         return if $volid =~ m|^/|;
5793
5794         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5795         return if !$sid;
5796
5797         push @$vollist, $volid;
5798     });
5799
5800     return $vollist;
5801 }
5802
5803 sub vm_stop_cleanup {
5804     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5805
5806     eval {
5807
5808         if (!$keepActive) {
5809             my $vollist = get_vm_volumes($conf);
5810             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5811
5812             if (my $tpmdrive = $conf->{tpmstate0}) {
5813                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5814                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5815                 if ($storeid) {
5816                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5817                 }
5818             }
5819         }
5820
5821         foreach my $ext (qw(mon qmp pid vnc qga)) {
5822             unlink "/var/run/qemu-server/${vmid}.$ext";
5823         }
5824
5825         if ($conf->{ivshmem}) {
5826             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5827             # just delete it for now, VMs which have this already open do not
5828             # are affected, but new VMs will get a separated one. If this
5829             # becomes an issue we either add some sort of ref-counting or just
5830             # add a "don't delete on stop" flag to the ivshmem format.
5831             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5832         }
5833
5834         my $ids = [];
5835         foreach my $key (keys %$conf) {
5836             next if $key !~ m/^hostpci(\d+)$/;
5837             my $hostpciindex = $1;
5838             my $d = parse_hostpci($conf->{$key});
5839             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5840
5841             foreach my $pci (@{$d->{pciid}}) {
5842                 my $pciid = $pci->{id};
5843                 push @$ids, $pci->{id};
5844                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5845             }
5846         }
5847         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5848
5849         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5850     };
5851     warn $@ if $@; # avoid errors - just warn
5852 }
5853
5854 # call only in locked context
5855 sub _do_vm_stop {
5856     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5857
5858     my $pid = check_running($vmid, $nocheck);
5859     return if !$pid;
5860
5861     my $conf;
5862     if (!$nocheck) {
5863         $conf = PVE::QemuConfig->load_config($vmid);
5864         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5865         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5866             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5867             $timeout = $opts->{down} if $opts->{down};
5868         }
5869         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5870     }
5871
5872     eval {
5873         if ($shutdown) {
5874             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5875                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5876             } else {
5877                 mon_cmd($vmid, "system_powerdown");
5878             }
5879         } else {
5880             mon_cmd($vmid, "quit");
5881         }
5882     };
5883     my $err = $@;
5884
5885     if (!$err) {
5886         $timeout = 60 if !defined($timeout);
5887
5888         my $count = 0;
5889         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5890             $count++;
5891             sleep 1;
5892         }
5893
5894         if ($count >= $timeout) {
5895             if ($force) {
5896                 warn "VM still running - terminating now with SIGTERM\n";
5897                 kill 15, $pid;
5898             } else {
5899                 die "VM quit/powerdown failed - got timeout\n";
5900             }
5901         } else {
5902             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5903             return;
5904         }
5905     } else {
5906        if (!check_running($vmid, $nocheck)) {
5907             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5908             return;
5909        }
5910        if ($force) {
5911             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5912             kill 15, $pid;
5913         } else {
5914             die "VM quit/powerdown failed\n";
5915         }
5916     }
5917
5918     # wait again
5919     $timeout = 10;
5920
5921     my $count = 0;
5922     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5923         $count++;
5924         sleep 1;
5925     }
5926
5927     if ($count >= $timeout) {
5928         warn "VM still running - terminating now with SIGKILL\n";
5929         kill 9, $pid;
5930         sleep 1;
5931     }
5932
5933     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5934 }
5935
5936 # Note: use $nocheck to skip tests if VM configuration file exists.
5937 # We need that when migration VMs to other nodes (files already moved)
5938 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5939 sub vm_stop {
5940     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5941
5942     $force = 1 if !defined($force) && !$shutdown;
5943
5944     if ($migratedfrom){
5945         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5946         kill 15, $pid if $pid;
5947         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5948         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5949         return;
5950     }
5951
5952     PVE::QemuConfig->lock_config($vmid, sub {
5953         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5954    });
5955 }
5956
5957 sub vm_reboot {
5958     my ($vmid, $timeout) = @_;
5959
5960     PVE::QemuConfig->lock_config($vmid, sub {
5961         eval {
5962
5963             # only reboot if running, as qmeventd starts it again on a stop event
5964             return if !check_running($vmid);
5965
5966             create_reboot_request($vmid);
5967
5968             my $storecfg = PVE::Storage::config();
5969             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5970
5971         };
5972         if (my $err = $@) {
5973             # avoid that the next normal shutdown will be confused for a reboot
5974             clear_reboot_request($vmid);
5975             die $err;
5976         }
5977    });
5978 }
5979
5980 # note: if using the statestorage parameter, the caller has to check privileges
5981 sub vm_suspend {
5982     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5983
5984     my $conf;
5985     my $path;
5986     my $storecfg;
5987     my $vmstate;
5988
5989     PVE::QemuConfig->lock_config($vmid, sub {
5990
5991         $conf = PVE::QemuConfig->load_config($vmid);
5992
5993         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5994         PVE::QemuConfig->check_lock($conf)
5995             if !($skiplock || $is_backing_up);
5996
5997         die "cannot suspend to disk during backup\n"
5998             if $is_backing_up && $includestate;
5999
6000         if ($includestate) {
6001             $conf->{lock} = 'suspending';
6002             my $date = strftime("%Y-%m-%d", localtime(time()));
6003             $storecfg = PVE::Storage::config();
6004             if (!$statestorage) {
6005                 $statestorage = find_vmstate_storage($conf, $storecfg);
6006                 # check permissions for the storage
6007                 my $rpcenv = PVE::RPCEnvironment::get();
6008                 if ($rpcenv->{type} ne 'cli') {
6009                     my $authuser = $rpcenv->get_user();
6010                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6011                 }
6012             }
6013
6014
6015             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6016                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6017             $path = PVE::Storage::path($storecfg, $vmstate);
6018             PVE::QemuConfig->write_config($vmid, $conf);
6019         } else {
6020             mon_cmd($vmid, "stop");
6021         }
6022     });
6023
6024     if ($includestate) {
6025         # save vm state
6026         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6027
6028         eval {
6029             set_migration_caps($vmid, 1);
6030             mon_cmd($vmid, "savevm-start", statefile => $path);
6031             for(;;) {
6032                 my $state = mon_cmd($vmid, "query-savevm");
6033                 if (!$state->{status}) {
6034                     die "savevm not active\n";
6035                 } elsif ($state->{status} eq 'active') {
6036                     sleep(1);
6037                     next;
6038                 } elsif ($state->{status} eq 'completed') {
6039                     print "State saved, quitting\n";
6040                     last;
6041                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6042                     die "query-savevm failed with error '$state->{error}'\n"
6043                 } else {
6044                     die "query-savevm returned status '$state->{status}'\n";
6045                 }
6046             }
6047         };
6048         my $err = $@;
6049
6050         PVE::QemuConfig->lock_config($vmid, sub {
6051             $conf = PVE::QemuConfig->load_config($vmid);
6052             if ($err) {
6053                 # cleanup, but leave suspending lock, to indicate something went wrong
6054                 eval {
6055                     mon_cmd($vmid, "savevm-end");
6056                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6057                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6058                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6059                     PVE::QemuConfig->write_config($vmid, $conf);
6060                 };
6061                 warn $@ if $@;
6062                 die $err;
6063             }
6064
6065             die "lock changed unexpectedly\n"
6066                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6067
6068             mon_cmd($vmid, "quit");
6069             $conf->{lock} = 'suspended';
6070             PVE::QemuConfig->write_config($vmid, $conf);
6071         });
6072     }
6073 }
6074
6075 sub vm_resume {
6076     my ($vmid, $skiplock, $nocheck) = @_;
6077
6078     PVE::QemuConfig->lock_config($vmid, sub {
6079         my $res = mon_cmd($vmid, 'query-status');
6080         my $resume_cmd = 'cont';
6081         my $reset = 0;
6082
6083         if ($res->{status}) {
6084             return if $res->{status} eq 'running'; # job done, go home
6085             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6086             $reset = 1 if $res->{status} eq 'shutdown';
6087         }
6088
6089         if (!$nocheck) {
6090
6091             my $conf = PVE::QemuConfig->load_config($vmid);
6092
6093             PVE::QemuConfig->check_lock($conf)
6094                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6095         }
6096
6097         if ($reset) {
6098             # required if a VM shuts down during a backup and we get a resume
6099             # request before the backup finishes for example
6100             mon_cmd($vmid, "system_reset");
6101         }
6102         mon_cmd($vmid, $resume_cmd);
6103     });
6104 }
6105
6106 sub vm_sendkey {
6107     my ($vmid, $skiplock, $key) = @_;
6108
6109     PVE::QemuConfig->lock_config($vmid, sub {
6110
6111         my $conf = PVE::QemuConfig->load_config($vmid);
6112
6113         # there is no qmp command, so we use the human monitor command
6114         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6115         die $res if $res ne '';
6116     });
6117 }
6118
6119 # vzdump restore implementaion
6120
6121 sub tar_archive_read_firstfile {
6122     my $archive = shift;
6123
6124     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6125
6126     # try to detect archive type first
6127     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6128         die "unable to open file '$archive'\n";
6129     my $firstfile = <$fh>;
6130     kill 15, $pid;
6131     close $fh;
6132
6133     die "ERROR: archive contaions no data\n" if !$firstfile;
6134     chomp $firstfile;
6135
6136     return $firstfile;
6137 }
6138
6139 sub tar_restore_cleanup {
6140     my ($storecfg, $statfile) = @_;
6141
6142     print STDERR "starting cleanup\n";
6143
6144     if (my $fd = IO::File->new($statfile, "r")) {
6145         while (defined(my $line = <$fd>)) {
6146             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6147                 my $volid = $2;
6148                 eval {
6149                     if ($volid =~ m|^/|) {
6150                         unlink $volid || die 'unlink failed\n';
6151                     } else {
6152                         PVE::Storage::vdisk_free($storecfg, $volid);
6153                     }
6154                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6155                 };
6156                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6157             } else {
6158                 print STDERR "unable to parse line in statfile - $line";
6159             }
6160         }
6161         $fd->close();
6162     }
6163 }
6164
6165 sub restore_file_archive {
6166     my ($archive, $vmid, $user, $opts) = @_;
6167
6168     return restore_vma_archive($archive, $vmid, $user, $opts)
6169         if $archive eq '-';
6170
6171     my $info = PVE::Storage::archive_info($archive);
6172     my $format = $opts->{format} // $info->{format};
6173     my $comp = $info->{compression};
6174
6175     # try to detect archive format
6176     if ($format eq 'tar') {
6177         return restore_tar_archive($archive, $vmid, $user, $opts);
6178     } else {
6179         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6180     }
6181 }
6182
6183 # hepler to remove disks that will not be used after restore
6184 my $restore_cleanup_oldconf = sub {
6185     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6186
6187     PVE::QemuConfig->foreach_volume($oldconf, sub {
6188         my ($ds, $drive) = @_;
6189
6190         return if drive_is_cdrom($drive, 1);
6191
6192         my $volid = $drive->{file};
6193         return if !$volid || $volid =~ m|^/|;
6194
6195         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6196         return if !$path || !$owner || ($owner != $vmid);
6197
6198         # Note: only delete disk we want to restore
6199         # other volumes will become unused
6200         if ($virtdev_hash->{$ds}) {
6201             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6202             if (my $err = $@) {
6203                 warn $err;
6204             }
6205         }
6206     });
6207
6208     # delete vmstate files, after the restore we have no snapshots anymore
6209     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6210         my $snap = $oldconf->{snapshots}->{$snapname};
6211         if ($snap->{vmstate}) {
6212             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6213             if (my $err = $@) {
6214                 warn $err;
6215             }
6216         }
6217     }
6218 };
6219
6220 # Helper to parse vzdump backup device hints
6221 #
6222 # $rpcenv: Environment, used to ckeck storage permissions
6223 # $user: User ID, to check storage permissions
6224 # $storecfg: Storage configuration
6225 # $fh: the file handle for reading the configuration
6226 # $devinfo: should contain device sizes for all backu-up'ed devices
6227 # $options: backup options (pool, default storage)
6228 #
6229 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6230 my $parse_backup_hints = sub {
6231     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6232
6233     my $virtdev_hash = {};
6234
6235     while (defined(my $line = <$fh>)) {
6236         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6237             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6238             die "archive does not contain data for drive '$virtdev'\n"
6239                 if !$devinfo->{$devname};
6240
6241             if (defined($options->{storage})) {
6242                 $storeid = $options->{storage} || 'local';
6243             } elsif (!$storeid) {
6244                 $storeid = 'local';
6245             }
6246             $format = 'raw' if !$format;
6247             $devinfo->{$devname}->{devname} = $devname;
6248             $devinfo->{$devname}->{virtdev} = $virtdev;
6249             $devinfo->{$devname}->{format} = $format;
6250             $devinfo->{$devname}->{storeid} = $storeid;
6251
6252             # check permission on storage
6253             my $pool = $options->{pool}; # todo: do we need that?
6254             if ($user ne 'root@pam') {
6255                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6256             }
6257
6258             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6259         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6260             my $virtdev = $1;
6261             my $drive = parse_drive($virtdev, $2);
6262             if (drive_is_cloudinit($drive)) {
6263                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6264                 $storeid = $options->{storage} if defined ($options->{storage});
6265                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6266                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6267
6268                 $virtdev_hash->{$virtdev} = {
6269                     format => $format,
6270                     storeid => $storeid,
6271                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6272                     is_cloudinit => 1,
6273                 };
6274             }
6275         }
6276     }
6277
6278     return $virtdev_hash;
6279 };
6280
6281 # Helper to allocate and activate all volumes required for a restore
6282 #
6283 # $storecfg: Storage configuration
6284 # $virtdev_hash: as returned by parse_backup_hints()
6285 #
6286 # Returns: { $virtdev => $volid }
6287 my $restore_allocate_devices = sub {
6288     my ($storecfg, $virtdev_hash, $vmid) = @_;
6289
6290     my $map = {};
6291     foreach my $virtdev (sort keys %$virtdev_hash) {
6292         my $d = $virtdev_hash->{$virtdev};
6293         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6294         my $storeid = $d->{storeid};
6295         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6296
6297         # test if requested format is supported
6298         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6299         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6300         $d->{format} = $defFormat if !$supported;
6301
6302         my $name;
6303         if ($d->{is_cloudinit}) {
6304             $name = "vm-$vmid-cloudinit";
6305             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6306             if ($scfg->{path}) {
6307                 $name .= ".$d->{format}";
6308             }
6309         }
6310
6311         my $volid = PVE::Storage::vdisk_alloc(
6312             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6313
6314         print STDERR "new volume ID is '$volid'\n";
6315         $d->{volid} = $volid;
6316
6317         PVE::Storage::activate_volumes($storecfg, [$volid]);
6318
6319         $map->{$virtdev} = $volid;
6320     }
6321
6322     return $map;
6323 };
6324
6325 sub restore_update_config_line {
6326     my ($cookie, $map, $line, $unique) = @_;
6327
6328     return '' if $line =~ m/^\#qmdump\#/;
6329     return '' if $line =~ m/^\#vzdump\#/;
6330     return '' if $line =~ m/^lock:/;
6331     return '' if $line =~ m/^unused\d+:/;
6332     return '' if $line =~ m/^parent:/;
6333
6334     my $res = '';
6335
6336     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6337     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6338         # try to convert old 1.X settings
6339         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6340         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6341             my ($model, $macaddr) = split(/\=/, $devconfig);
6342             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6343             my $net = {
6344                 model => $model,
6345                 bridge => "vmbr$ind",
6346                 macaddr => $macaddr,
6347             };
6348             my $netstr = print_net($net);
6349
6350             $res .= "net$cookie->{netcount}: $netstr\n";
6351             $cookie->{netcount}++;
6352         }
6353     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6354         my ($id, $netstr) = ($1, $2);
6355         my $net = parse_net($netstr);
6356         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6357         $netstr = print_net($net);
6358         $res .= "$id: $netstr\n";
6359     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6360         my $virtdev = $1;
6361         my $value = $3;
6362         my $di = parse_drive($virtdev, $value);
6363         if (defined($di->{backup}) && !$di->{backup}) {
6364             $res .= "#$line";
6365         } elsif ($map->{$virtdev}) {
6366             delete $di->{format}; # format can change on restore
6367             $di->{file} = $map->{$virtdev};
6368             $value = print_drive($di);
6369             $res .= "$virtdev: $value\n";
6370         } else {
6371             $res .= $line;
6372         }
6373     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6374         my $vmgenid = $1;
6375         if ($vmgenid ne '0') {
6376             # always generate a new vmgenid if there was a valid one setup
6377             $vmgenid = generate_uuid();
6378         }
6379         $res .= "vmgenid: $vmgenid\n";
6380     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6381         my ($uuid, $uuid_str);
6382         UUID::generate($uuid);
6383         UUID::unparse($uuid, $uuid_str);
6384         my $smbios1 = parse_smbios1($2);
6385         $smbios1->{uuid} = $uuid_str;
6386         $res .= $1.print_smbios1($smbios1)."\n";
6387     } else {
6388         $res .= $line;
6389     }
6390
6391     return $res;
6392 }
6393
6394 my $restore_deactivate_volumes = sub {
6395     my ($storecfg, $devinfo) = @_;
6396
6397     my $vollist = [];
6398     foreach my $devname (keys %$devinfo) {
6399         my $volid = $devinfo->{$devname}->{volid};
6400         push @$vollist, $volid if $volid;
6401     }
6402
6403     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6404 };
6405
6406 my $restore_destroy_volumes = sub {
6407     my ($storecfg, $devinfo) = @_;
6408
6409     foreach my $devname (keys %$devinfo) {
6410         my $volid = $devinfo->{$devname}->{volid};
6411         next if !$volid;
6412         eval {
6413             if ($volid =~ m|^/|) {
6414                 unlink $volid || die 'unlink failed\n';
6415             } else {
6416                 PVE::Storage::vdisk_free($storecfg, $volid);
6417             }
6418             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6419         };
6420         print STDERR "unable to cleanup '$volid' - $@" if $@;
6421     }
6422 };
6423
6424 sub scan_volids {
6425     my ($cfg, $vmid) = @_;
6426
6427     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6428
6429     my $volid_hash = {};
6430     foreach my $storeid (keys %$info) {
6431         foreach my $item (@{$info->{$storeid}}) {
6432             next if !($item->{volid} && $item->{size});
6433             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6434             $volid_hash->{$item->{volid}} = $item;
6435         }
6436     }
6437
6438     return $volid_hash;
6439 }
6440
6441 sub update_disk_config {
6442     my ($vmid, $conf, $volid_hash) = @_;
6443
6444     my $changes;
6445     my $prefix = "VM $vmid";
6446
6447     # used and unused disks
6448     my $referenced = {};
6449
6450     # Note: it is allowed to define multiple storages with same path (alias), so
6451     # we need to check both 'volid' and real 'path' (two different volid can point
6452     # to the same path).
6453
6454     my $referencedpath = {};
6455
6456     # update size info
6457     PVE::QemuConfig->foreach_volume($conf, sub {
6458         my ($opt, $drive) = @_;
6459
6460         my $volid = $drive->{file};
6461         return if !$volid;
6462         my $volume = $volid_hash->{$volid};
6463
6464         # mark volid as "in-use" for next step
6465         $referenced->{$volid} = 1;
6466         if ($volume && (my $path = $volume->{path})) {
6467             $referencedpath->{$path} = 1;
6468         }
6469
6470         return if drive_is_cdrom($drive);
6471         return if !$volume;
6472
6473         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6474         if (defined($updated)) {
6475             $changes = 1;
6476             $conf->{$opt} = print_drive($updated);
6477             print "$prefix ($opt): $msg\n";
6478         }
6479     });
6480
6481     # remove 'unusedX' entry if volume is used
6482     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6483         my ($opt, $drive) = @_;
6484
6485         my $volid = $drive->{file};
6486         return if !$volid;
6487
6488         my $path;
6489         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6490         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6491             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6492             $changes = 1;
6493             delete $conf->{$opt};
6494         }
6495
6496         $referenced->{$volid} = 1;
6497         $referencedpath->{$path} = 1 if $path;
6498     });
6499
6500     foreach my $volid (sort keys %$volid_hash) {
6501         next if $volid =~ m/vm-$vmid-state-/;
6502         next if $referenced->{$volid};
6503         my $path = $volid_hash->{$volid}->{path};
6504         next if !$path; # just to be sure
6505         next if $referencedpath->{$path};
6506         $changes = 1;
6507         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6508         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6509         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6510     }
6511
6512     return $changes;
6513 }
6514
6515 sub rescan {
6516     my ($vmid, $nolock, $dryrun) = @_;
6517
6518     my $cfg = PVE::Storage::config();
6519
6520     print "rescan volumes...\n";
6521     my $volid_hash = scan_volids($cfg, $vmid);
6522
6523     my $updatefn =  sub {
6524         my ($vmid) = @_;
6525
6526         my $conf = PVE::QemuConfig->load_config($vmid);
6527
6528         PVE::QemuConfig->check_lock($conf);
6529
6530         my $vm_volids = {};
6531         foreach my $volid (keys %$volid_hash) {
6532             my $info = $volid_hash->{$volid};
6533             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6534         }
6535
6536         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6537
6538         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6539     };
6540
6541     if (defined($vmid)) {
6542         if ($nolock) {
6543             &$updatefn($vmid);
6544         } else {
6545             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6546         }
6547     } else {
6548         my $vmlist = config_list();
6549         foreach my $vmid (keys %$vmlist) {
6550             if ($nolock) {
6551                 &$updatefn($vmid);
6552             } else {
6553                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6554             }
6555         }
6556     }
6557 }
6558
6559 sub restore_proxmox_backup_archive {
6560     my ($archive, $vmid, $user, $options) = @_;
6561
6562     my $storecfg = PVE::Storage::config();
6563
6564     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6565     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6566
6567     my $fingerprint = $scfg->{fingerprint};
6568     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6569
6570     my $repo = PVE::PBSClient::get_repository($scfg);
6571
6572     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6573     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6574     local $ENV{PBS_PASSWORD} = $password;
6575     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6576
6577     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6578         PVE::Storage::parse_volname($storecfg, $archive);
6579
6580     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6581
6582     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6583
6584     my $tmpdir = "/var/tmp/vzdumptmp$$";
6585     rmtree $tmpdir;
6586     mkpath $tmpdir;
6587
6588     my $conffile = PVE::QemuConfig->config_file($vmid);
6589      # disable interrupts (always do cleanups)
6590     local $SIG{INT} =
6591         local $SIG{TERM} =
6592         local $SIG{QUIT} =
6593         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6594
6595     # Note: $oldconf is undef if VM does not exists
6596     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6597     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6598     my $new_conf_raw = '';
6599
6600     my $rpcenv = PVE::RPCEnvironment::get();
6601     my $devinfo = {};
6602
6603     eval {
6604         # enable interrupts
6605         local $SIG{INT} =
6606             local $SIG{TERM} =
6607             local $SIG{QUIT} =
6608             local $SIG{HUP} =
6609             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6610
6611         my $cfgfn = "$tmpdir/qemu-server.conf";
6612         my $firewall_config_fn = "$tmpdir/fw.conf";
6613         my $index_fn = "$tmpdir/index.json";
6614
6615         my $cmd = "restore";
6616
6617         my $param = [$pbs_backup_name, "index.json", $index_fn];
6618         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6619         my $index = PVE::Tools::file_get_contents($index_fn);
6620         $index = decode_json($index);
6621
6622         # print Dumper($index);
6623         foreach my $info (@{$index->{files}}) {
6624             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6625                 my $devname = $1;
6626                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6627                     $devinfo->{$devname}->{size} = $1;
6628                 } else {
6629                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6630                 }
6631             }
6632         }
6633
6634         my $is_qemu_server_backup = scalar(
6635             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6636         );
6637         if (!$is_qemu_server_backup) {
6638             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6639         }
6640         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6641
6642         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6643         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6644
6645         if ($has_firewall_config) {
6646             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6647             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6648
6649             my $pve_firewall_dir = '/etc/pve/firewall';
6650             mkdir $pve_firewall_dir; # make sure the dir exists
6651             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6652         }
6653
6654         my $fh = IO::File->new($cfgfn, "r") ||
6655             die "unable to read qemu-server.conf - $!\n";
6656
6657         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6658
6659         # fixme: rate limit?
6660
6661         # create empty/temp config
6662         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6663
6664         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6665
6666         # allocate volumes
6667         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6668
6669         foreach my $virtdev (sort keys %$virtdev_hash) {
6670             my $d = $virtdev_hash->{$virtdev};
6671             next if $d->{is_cloudinit}; # no need to restore cloudinit
6672
6673             # this fails if storage is unavailable
6674             my $volid = $d->{volid};
6675             my $path = PVE::Storage::path($storecfg, $volid);
6676
6677             # for live-restore we only want to preload the efidisk and TPM state
6678             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6679
6680             my $pbs_restore_cmd = [
6681                 '/usr/bin/pbs-restore',
6682                 '--repository', $repo,
6683                 $pbs_backup_name,
6684                 "$d->{devname}.img.fidx",
6685                 $path,
6686                 '--verbose',
6687                 ];
6688
6689             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6690             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6691
6692             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6693                 push @$pbs_restore_cmd, '--skip-zero';
6694             }
6695
6696             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6697             print "restore proxmox backup image: $dbg_cmdstring\n";
6698             run_command($pbs_restore_cmd);
6699         }
6700
6701         $fh->seek(0, 0) || die "seek failed - $!\n";
6702
6703         my $cookie = { netcount => 0 };
6704         while (defined(my $line = <$fh>)) {
6705             $new_conf_raw .= restore_update_config_line(
6706                 $cookie,
6707                 $map,
6708                 $line,
6709                 $options->{unique},
6710             );
6711         }
6712
6713         $fh->close();
6714     };
6715     my $err = $@;
6716
6717     if ($err || !$options->{live}) {
6718         $restore_deactivate_volumes->($storecfg, $devinfo);
6719     }
6720
6721     rmtree $tmpdir;
6722
6723     if ($err) {
6724         $restore_destroy_volumes->($storecfg, $devinfo);
6725         die $err;
6726     }
6727
6728     if ($options->{live}) {
6729         # keep lock during live-restore
6730         $new_conf_raw .= "\nlock: create";
6731     }
6732
6733     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6734
6735     PVE::Cluster::cfs_update(); # make sure we read new file
6736
6737     eval { rescan($vmid, 1); };
6738     warn $@ if $@;
6739
6740     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6741
6742     if ($options->{live}) {
6743         # enable interrupts
6744         local $SIG{INT} =
6745             local $SIG{TERM} =
6746             local $SIG{QUIT} =
6747             local $SIG{HUP} =
6748             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6749
6750         my $conf = PVE::QemuConfig->load_config($vmid);
6751         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6752
6753         # these special drives are already restored before start
6754         delete $devinfo->{'drive-efidisk0'};
6755         delete $devinfo->{'drive-tpmstate0-backup'};
6756         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6757
6758         PVE::QemuConfig->remove_lock($vmid, "create");
6759     }
6760 }
6761
6762 sub pbs_live_restore {
6763     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6764
6765     print "starting VM for live-restore\n";
6766     print "repository: '$repo', snapshot: '$snap'\n";
6767
6768     my $pbs_backing = {};
6769     for my $ds (keys %$restored_disks) {
6770         $ds =~ m/^drive-(.*)$/;
6771         my $confname = $1;
6772         $pbs_backing->{$confname} = {
6773             repository => $repo,
6774             snapshot => $snap,
6775             archive => "$ds.img.fidx",
6776         };
6777         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6778
6779         my $drive = parse_drive($confname, $conf->{$confname});
6780         print "restoring '$ds' to '$drive->{file}'\n";
6781     }
6782
6783     my $drives_streamed = 0;
6784     eval {
6785         # make sure HA doesn't interrupt our restore by stopping the VM
6786         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6787             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6788         }
6789
6790         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6791         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6792         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6793
6794         my $qmeventd_fd = register_qmeventd_handle($vmid);
6795
6796         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6797         # this will effectively collapse the backing image chain consisting of
6798         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6799         # removes itself once all backing images vanish with 'auto-remove=on')
6800         my $jobs = {};
6801         for my $ds (sort keys %$restored_disks) {
6802             my $job_id = "restore-$ds";
6803             mon_cmd($vmid, 'block-stream',
6804                 'job-id' => $job_id,
6805                 device => "$ds",
6806             );
6807             $jobs->{$job_id} = {};
6808         }
6809
6810         mon_cmd($vmid, 'cont');
6811         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6812
6813         print "restore-drive jobs finished successfully, removing all tracking block devices"
6814             ." to disconnect from Proxmox Backup Server\n";
6815
6816         for my $ds (sort keys %$restored_disks) {
6817             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6818         }
6819
6820         close($qmeventd_fd);
6821     };
6822
6823     my $err = $@;
6824
6825     if ($err) {
6826         warn "An error occured during live-restore: $err\n";
6827         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6828         die "live-restore failed\n";
6829     }
6830 }
6831
6832 sub restore_vma_archive {
6833     my ($archive, $vmid, $user, $opts, $comp) = @_;
6834
6835     my $readfrom = $archive;
6836
6837     my $cfg = PVE::Storage::config();
6838     my $commands = [];
6839     my $bwlimit = $opts->{bwlimit};
6840
6841     my $dbg_cmdstring = '';
6842     my $add_pipe = sub {
6843         my ($cmd) = @_;
6844         push @$commands, $cmd;
6845         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6846         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6847         $readfrom = '-';
6848     };
6849
6850     my $input = undef;
6851     if ($archive eq '-') {
6852         $input = '<&STDIN';
6853     } else {
6854         # If we use a backup from a PVE defined storage we also consider that
6855         # storage's rate limit:
6856         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6857         if (defined($volid)) {
6858             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6859             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6860             if ($readlimit) {
6861                 print STDERR "applying read rate limit: $readlimit\n";
6862                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6863                 $add_pipe->($cstream);
6864             }
6865         }
6866     }
6867
6868     if ($comp) {
6869         my $info = PVE::Storage::decompressor_info('vma', $comp);
6870         my $cmd = $info->{decompressor};
6871         push @$cmd, $readfrom;
6872         $add_pipe->($cmd);
6873     }
6874
6875     my $tmpdir = "/var/tmp/vzdumptmp$$";
6876     rmtree $tmpdir;
6877
6878     # disable interrupts (always do cleanups)
6879     local $SIG{INT} =
6880         local $SIG{TERM} =
6881         local $SIG{QUIT} =
6882         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6883
6884     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6885     POSIX::mkfifo($mapfifo, 0600);
6886     my $fifofh;
6887     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6888
6889     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6890
6891     my $oldtimeout;
6892     my $timeout = 5;
6893
6894     my $devinfo = {};
6895
6896     my $rpcenv = PVE::RPCEnvironment::get();
6897
6898     my $conffile = PVE::QemuConfig->config_file($vmid);
6899
6900     # Note: $oldconf is undef if VM does not exist
6901     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6902     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6903     my $new_conf_raw = '';
6904
6905     my %storage_limits;
6906
6907     my $print_devmap = sub {
6908         my $cfgfn = "$tmpdir/qemu-server.conf";
6909
6910         # we can read the config - that is already extracted
6911         my $fh = IO::File->new($cfgfn, "r") ||
6912             die "unable to read qemu-server.conf - $!\n";
6913
6914         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6915         if (-f $fwcfgfn) {
6916             my $pve_firewall_dir = '/etc/pve/firewall';
6917             mkdir $pve_firewall_dir; # make sure the dir exists
6918             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6919         }
6920
6921         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6922
6923         foreach my $info (values %{$virtdev_hash}) {
6924             my $storeid = $info->{storeid};
6925             next if defined($storage_limits{$storeid});
6926
6927             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6928             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6929             $storage_limits{$storeid} = $limit * 1024;
6930         }
6931
6932         foreach my $devname (keys %$devinfo) {
6933             die "found no device mapping information for device '$devname'\n"
6934                 if !$devinfo->{$devname}->{virtdev};
6935         }
6936
6937         # create empty/temp config
6938         if ($oldconf) {
6939             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6940             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6941         }
6942
6943         # allocate volumes
6944         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6945
6946         # print restore information to $fifofh
6947         foreach my $virtdev (sort keys %$virtdev_hash) {
6948             my $d = $virtdev_hash->{$virtdev};
6949             next if $d->{is_cloudinit}; # no need to restore cloudinit
6950
6951             my $storeid = $d->{storeid};
6952             my $volid = $d->{volid};
6953
6954             my $map_opts = '';
6955             if (my $limit = $storage_limits{$storeid}) {
6956                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6957             }
6958
6959             my $write_zeros = 1;
6960             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6961                 $write_zeros = 0;
6962             }
6963
6964             my $path = PVE::Storage::path($cfg, $volid);
6965
6966             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6967
6968             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6969         }
6970
6971         $fh->seek(0, 0) || die "seek failed - $!\n";
6972
6973         my $cookie = { netcount => 0 };
6974         while (defined(my $line = <$fh>)) {
6975             $new_conf_raw .= restore_update_config_line(
6976                 $cookie,
6977                 $map,
6978                 $line,
6979                 $opts->{unique},
6980             );
6981         }
6982
6983         $fh->close();
6984     };
6985
6986     eval {
6987         # enable interrupts
6988         local $SIG{INT} =
6989             local $SIG{TERM} =
6990             local $SIG{QUIT} =
6991             local $SIG{HUP} =
6992             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6993         local $SIG{ALRM} = sub { die "got timeout\n"; };
6994
6995         $oldtimeout = alarm($timeout);
6996
6997         my $parser = sub {
6998             my $line = shift;
6999
7000             print "$line\n";
7001
7002             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7003                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7004                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7005             } elsif ($line =~ m/^CTIME: /) {
7006                 # we correctly received the vma config, so we can disable
7007                 # the timeout now for disk allocation (set to 10 minutes, so
7008                 # that we always timeout if something goes wrong)
7009                 alarm(600);
7010                 &$print_devmap();
7011                 print $fifofh "done\n";
7012                 my $tmp = $oldtimeout || 0;
7013                 $oldtimeout = undef;
7014                 alarm($tmp);
7015                 close($fifofh);
7016                 $fifofh = undef;
7017             }
7018         };
7019
7020         print "restore vma archive: $dbg_cmdstring\n";
7021         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7022     };
7023     my $err = $@;
7024
7025     alarm($oldtimeout) if $oldtimeout;
7026
7027     $restore_deactivate_volumes->($cfg, $devinfo);
7028
7029     close($fifofh) if $fifofh;
7030     unlink $mapfifo;
7031     rmtree $tmpdir;
7032
7033     if ($err) {
7034         $restore_destroy_volumes->($cfg, $devinfo);
7035         die $err;
7036     }
7037
7038     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7039
7040     PVE::Cluster::cfs_update(); # make sure we read new file
7041
7042     eval { rescan($vmid, 1); };
7043     warn $@ if $@;
7044
7045     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7046 }
7047
7048 sub restore_tar_archive {
7049     my ($archive, $vmid, $user, $opts) = @_;
7050
7051     if ($archive ne '-') {
7052         my $firstfile = tar_archive_read_firstfile($archive);
7053         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7054             if $firstfile ne 'qemu-server.conf';
7055     }
7056
7057     my $storecfg = PVE::Storage::config();
7058
7059     # avoid zombie disks when restoring over an existing VM -> cleanup first
7060     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7061     # skiplock=1 because qmrestore has set the 'create' lock itself already
7062     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7063     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7064
7065     my $tocmd = "/usr/lib/qemu-server/qmextract";
7066
7067     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7068     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7069     $tocmd .= ' --prealloc' if $opts->{prealloc};
7070     $tocmd .= ' --info' if $opts->{info};
7071
7072     # tar option "xf" does not autodetect compression when read from STDIN,
7073     # so we pipe to zcat
7074     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7075         PVE::Tools::shellquote("--to-command=$tocmd");
7076
7077     my $tmpdir = "/var/tmp/vzdumptmp$$";
7078     mkpath $tmpdir;
7079
7080     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7081     local $ENV{VZDUMP_VMID} = $vmid;
7082     local $ENV{VZDUMP_USER} = $user;
7083
7084     my $conffile = PVE::QemuConfig->config_file($vmid);
7085     my $new_conf_raw = '';
7086
7087     # disable interrupts (always do cleanups)
7088     local $SIG{INT} =
7089         local $SIG{TERM} =
7090         local $SIG{QUIT} =
7091         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7092
7093     eval {
7094         # enable interrupts
7095         local $SIG{INT} =
7096             local $SIG{TERM} =
7097             local $SIG{QUIT} =
7098             local $SIG{HUP} =
7099             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7100
7101         if ($archive eq '-') {
7102             print "extracting archive from STDIN\n";
7103             run_command($cmd, input => "<&STDIN");
7104         } else {
7105             print "extracting archive '$archive'\n";
7106             run_command($cmd);
7107         }
7108
7109         return if $opts->{info};
7110
7111         # read new mapping
7112         my $map = {};
7113         my $statfile = "$tmpdir/qmrestore.stat";
7114         if (my $fd = IO::File->new($statfile, "r")) {
7115             while (defined (my $line = <$fd>)) {
7116                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7117                     $map->{$1} = $2 if $1;
7118                 } else {
7119                     print STDERR "unable to parse line in statfile - $line\n";
7120                 }
7121             }
7122             $fd->close();
7123         }
7124
7125         my $confsrc = "$tmpdir/qemu-server.conf";
7126
7127         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7128
7129         my $cookie = { netcount => 0 };
7130         while (defined (my $line = <$srcfd>)) {
7131             $new_conf_raw .= restore_update_config_line(
7132                 $cookie,
7133                 $map,
7134                 $line,
7135                 $opts->{unique},
7136             );
7137         }
7138
7139         $srcfd->close();
7140     };
7141     if (my $err = $@) {
7142         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7143         die $err;
7144     }
7145
7146     rmtree $tmpdir;
7147
7148     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7149
7150     PVE::Cluster::cfs_update(); # make sure we read new file
7151
7152     eval { rescan($vmid, 1); };
7153     warn $@ if $@;
7154 };
7155
7156 sub foreach_storage_used_by_vm {
7157     my ($conf, $func) = @_;
7158
7159     my $sidhash = {};
7160
7161     PVE::QemuConfig->foreach_volume($conf, sub {
7162         my ($ds, $drive) = @_;
7163         return if drive_is_cdrom($drive);
7164
7165         my $volid = $drive->{file};
7166
7167         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7168         $sidhash->{$sid} = $sid if $sid;
7169     });
7170
7171     foreach my $sid (sort keys %$sidhash) {
7172         &$func($sid);
7173     }
7174 }
7175
7176 my $qemu_snap_storage = {
7177     rbd => 1,
7178 };
7179 sub do_snapshots_with_qemu {
7180     my ($storecfg, $volid, $deviceid) = @_;
7181
7182     return if $deviceid =~ m/tpmstate0/;
7183
7184     my $storage_name = PVE::Storage::parse_volume_id($volid);
7185     my $scfg = $storecfg->{ids}->{$storage_name};
7186     die "could not find storage '$storage_name'\n" if !defined($scfg);
7187
7188     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7189         return 1;
7190     }
7191
7192     if ($volid =~ m/\.(qcow2|qed)$/){
7193         return 1;
7194     }
7195
7196     return;
7197 }
7198
7199 sub qga_check_running {
7200     my ($vmid, $nowarn) = @_;
7201
7202     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7203     if ($@) {
7204         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7205         return 0;
7206     }
7207     return 1;
7208 }
7209
7210 sub template_create {
7211     my ($vmid, $conf, $disk) = @_;
7212
7213     my $storecfg = PVE::Storage::config();
7214
7215     PVE::QemuConfig->foreach_volume($conf, sub {
7216         my ($ds, $drive) = @_;
7217
7218         return if drive_is_cdrom($drive);
7219         return if $disk && $ds ne $disk;
7220
7221         my $volid = $drive->{file};
7222         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7223
7224         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7225         $drive->{file} = $voliddst;
7226         $conf->{$ds} = print_drive($drive);
7227         PVE::QemuConfig->write_config($vmid, $conf);
7228     });
7229 }
7230
7231 sub convert_iscsi_path {
7232     my ($path) = @_;
7233
7234     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7235         my $portal = $1;
7236         my $target = $2;
7237         my $lun = $3;
7238
7239         my $initiator_name = get_initiator_name();
7240
7241         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7242                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7243     }
7244
7245     die "cannot convert iscsi path '$path', unkown format\n";
7246 }
7247
7248 sub qemu_img_convert {
7249     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7250
7251     my $storecfg = PVE::Storage::config();
7252     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7253     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7254
7255     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7256
7257     my $cachemode;
7258     my $src_path;
7259     my $src_is_iscsi = 0;
7260     my $src_format;
7261
7262     if ($src_storeid) {
7263         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7264         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7265         $src_format = qemu_img_format($src_scfg, $src_volname);
7266         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7267         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7268         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7269     } elsif (-f $src_volid) {
7270         $src_path = $src_volid;
7271         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7272             $src_format = $1;
7273         }
7274     }
7275
7276     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7277
7278     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7279     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7280     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7281     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7282
7283     my $cmd = [];
7284     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7285     push @$cmd, '-l', "snapshot.name=$snapname"
7286         if $snapname && $src_format && $src_format eq "qcow2";
7287     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7288     push @$cmd, '-T', $cachemode if defined($cachemode);
7289
7290     if ($src_is_iscsi) {
7291         push @$cmd, '--image-opts';
7292         $src_path = convert_iscsi_path($src_path);
7293     } elsif ($src_format) {
7294         push @$cmd, '-f', $src_format;
7295     }
7296
7297     if ($dst_is_iscsi) {
7298         push @$cmd, '--target-image-opts';
7299         $dst_path = convert_iscsi_path($dst_path);
7300     } else {
7301         push @$cmd, '-O', $dst_format;
7302     }
7303
7304     push @$cmd, $src_path;
7305
7306     if (!$dst_is_iscsi && $is_zero_initialized) {
7307         push @$cmd, "zeroinit:$dst_path";
7308     } else {
7309         push @$cmd, $dst_path;
7310     }
7311
7312     my $parser = sub {
7313         my $line = shift;
7314         if($line =~ m/\((\S+)\/100\%\)/){
7315             my $percent = $1;
7316             my $transferred = int($size * $percent / 100);
7317             my $total_h = render_bytes($size, 1);
7318             my $transferred_h = render_bytes($transferred, 1);
7319
7320             print "transferred $transferred_h of $total_h ($percent%)\n";
7321         }
7322
7323     };
7324
7325     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7326     my $err = $@;
7327     die "copy failed: $err" if $err;
7328 }
7329
7330 sub qemu_img_format {
7331     my ($scfg, $volname) = @_;
7332
7333     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7334         return $1;
7335     } else {
7336         return "raw";
7337     }
7338 }
7339
7340 sub qemu_drive_mirror {
7341     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7342
7343     $jobs = {} if !$jobs;
7344
7345     my $qemu_target;
7346     my $format;
7347     $jobs->{"drive-$drive"} = {};
7348
7349     if ($dst_volid =~ /^nbd:/) {
7350         $qemu_target = $dst_volid;
7351         $format = "nbd";
7352     } else {
7353         my $storecfg = PVE::Storage::config();
7354         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7355
7356         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7357
7358         $format = qemu_img_format($dst_scfg, $dst_volname);
7359
7360         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7361
7362         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7363     }
7364
7365     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7366     $opts->{format} = $format if $format;
7367
7368     if (defined($src_bitmap)) {
7369         $opts->{sync} = 'incremental';
7370         $opts->{bitmap} = $src_bitmap;
7371         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7372     }
7373
7374     if (defined($bwlimit)) {
7375         $opts->{speed} = $bwlimit * 1024;
7376         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7377     } else {
7378         print "drive mirror is starting for drive-$drive\n";
7379     }
7380
7381     # if a job already runs for this device we get an error, catch it for cleanup
7382     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7383     if (my $err = $@) {
7384         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7385         warn "$@\n" if $@;
7386         die "mirroring error: $err\n";
7387     }
7388
7389     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7390 }
7391
7392 # $completion can be either
7393 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7394 # 'cancel': wait until all jobs are ready, block-job-cancel them
7395 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7396 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7397 sub qemu_drive_mirror_monitor {
7398     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7399
7400     $completion //= 'complete';
7401     $op //= "mirror";
7402
7403     eval {
7404         my $err_complete = 0;
7405
7406         my $starttime = time ();
7407         while (1) {
7408             die "block job ('$op') timed out\n" if $err_complete > 300;
7409
7410             my $stats = mon_cmd($vmid, "query-block-jobs");
7411             my $ctime = time();
7412
7413             my $running_jobs = {};
7414             for my $stat (@$stats) {
7415                 next if $stat->{type} ne $op;
7416                 $running_jobs->{$stat->{device}} = $stat;
7417             }
7418
7419             my $readycounter = 0;
7420
7421             for my $job_id (sort keys %$jobs) {
7422                 my $job = $running_jobs->{$job_id};
7423
7424                 my $vanished = !defined($job);
7425                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7426                 if($complete || ($vanished && $completion eq 'auto')) {
7427                     print "$job_id: $op-job finished\n";
7428                     delete $jobs->{$job_id};
7429                     next;
7430                 }
7431
7432                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7433
7434                 my $busy = $job->{busy};
7435                 my $ready = $job->{ready};
7436                 if (my $total = $job->{len}) {
7437                     my $transferred = $job->{offset} || 0;
7438                     my $remaining = $total - $transferred;
7439                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7440
7441                     my $duration = $ctime - $starttime;
7442                     my $total_h = render_bytes($total, 1);
7443                     my $transferred_h = render_bytes($transferred, 1);
7444
7445                     my $status = sprintf(
7446                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7447                         render_duration($duration),
7448                     );
7449
7450                     if ($ready) {
7451                         if ($busy) {
7452                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7453                         } else {
7454                             $status .= ", ready";
7455                         }
7456                     }
7457                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7458                     $jobs->{$job_id}->{ready} = $ready;
7459                 }
7460
7461                 $readycounter++ if $job->{ready};
7462             }
7463
7464             last if scalar(keys %$jobs) == 0;
7465
7466             if ($readycounter == scalar(keys %$jobs)) {
7467                 print "all '$op' jobs are ready\n";
7468
7469                 # do the complete later (or has already been done)
7470                 last if $completion eq 'skip' || $completion eq 'auto';
7471
7472                 if ($vmiddst && $vmiddst != $vmid) {
7473                     my $agent_running = $qga && qga_check_running($vmid);
7474                     if ($agent_running) {
7475                         print "freeze filesystem\n";
7476                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7477                         warn $@ if $@;
7478                     } else {
7479                         print "suspend vm\n";
7480                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7481                         warn $@ if $@;
7482                     }
7483
7484                     # if we clone a disk for a new target vm, we don't switch the disk
7485                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7486
7487                     if ($agent_running) {
7488                         print "unfreeze filesystem\n";
7489                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7490                         warn $@ if $@;
7491                     } else {
7492                         print "resume vm\n";
7493                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7494                         warn $@ if $@;
7495                     }
7496
7497                     last;
7498                 } else {
7499
7500                     for my $job_id (sort keys %$jobs) {
7501                         # try to switch the disk if source and destination are on the same guest
7502                         print "$job_id: Completing block job_id...\n";
7503
7504                         my $op;
7505                         if ($completion eq 'complete') {
7506                             $op = 'block-job-complete';
7507                         } elsif ($completion eq 'cancel') {
7508                             $op = 'block-job-cancel';
7509                         } else {
7510                             die "invalid completion value: $completion\n";
7511                         }
7512                         eval { mon_cmd($vmid, $op, device => $job_id) };
7513                         if ($@ =~ m/cannot be completed/) {
7514                             print "$job_id: block job cannot be completed, trying again.\n";
7515                             $err_complete++;
7516                         }else {
7517                             print "$job_id: Completed successfully.\n";
7518                             $jobs->{$job_id}->{complete} = 1;
7519                         }
7520                     }
7521                 }
7522             }
7523             sleep 1;
7524         }
7525     };
7526     my $err = $@;
7527
7528     if ($err) {
7529         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7530         die "block job ($op) error: $err";
7531     }
7532 }
7533
7534 sub qemu_blockjobs_cancel {
7535     my ($vmid, $jobs) = @_;
7536
7537     foreach my $job (keys %$jobs) {
7538         print "$job: Cancelling block job\n";
7539         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7540         $jobs->{$job}->{cancel} = 1;
7541     }
7542
7543     while (1) {
7544         my $stats = mon_cmd($vmid, "query-block-jobs");
7545
7546         my $running_jobs = {};
7547         foreach my $stat (@$stats) {
7548             $running_jobs->{$stat->{device}} = $stat;
7549         }
7550
7551         foreach my $job (keys %$jobs) {
7552
7553             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7554                 print "$job: Done.\n";
7555                 delete $jobs->{$job};
7556             }
7557         }
7558
7559         last if scalar(keys %$jobs) == 0;
7560
7561         sleep 1;
7562     }
7563 }
7564
7565 sub clone_disk {
7566     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7567         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7568
7569     my $newvolid;
7570
7571     if (!$full) {
7572         print "create linked clone of drive $drivename ($drive->{file})\n";
7573         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7574         push @$newvollist, $newvolid;
7575     } else {
7576
7577         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7578         $storeid = $storage if $storage;
7579
7580         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7581
7582         print "create full clone of drive $drivename ($drive->{file})\n";
7583         my $name = undef;
7584         my $size = undef;
7585         if (drive_is_cloudinit($drive)) {
7586             $name = "vm-$newvmid-cloudinit";
7587             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7588             if ($scfg->{path}) {
7589                 $name .= ".$dst_format";
7590             }
7591             $snapname = undef;
7592             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7593         } elsif ($drivename eq 'efidisk0') {
7594             $size = get_efivars_size($conf);
7595         } elsif ($drivename eq 'tpmstate0') {
7596             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7597         } else {
7598             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7599         }
7600         $newvolid = PVE::Storage::vdisk_alloc(
7601             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7602         );
7603         push @$newvollist, $newvolid;
7604
7605         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7606
7607         if (drive_is_cloudinit($drive)) {
7608             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7609             # if this is the case, we have to complete any block-jobs still there from
7610             # previous drive-mirrors
7611             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7612                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7613             }
7614             goto no_data_clone;
7615         }
7616
7617         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7618         if (!$running || $snapname) {
7619             # TODO: handle bwlimits
7620             if ($drivename eq 'efidisk0') {
7621                 # the relevant data on the efidisk may be smaller than the source
7622                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7623                 # that is given by the OVMF_VARS.fd
7624                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7625                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7626
7627                 # better for Ceph if block size is not too small, see bug #3324
7628                 my $bs = 1024*1024;
7629
7630                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7631                     "if=$src_path", "of=$dst_path"]);
7632             } else {
7633                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7634             }
7635         } else {
7636
7637             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7638
7639             my $kvmver = get_running_qemu_version ($vmid);
7640             if (!min_version($kvmver, 2, 7)) {
7641                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7642                     if $drive->{iothread};
7643             }
7644
7645             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7646                 $completion, $qga, $bwlimit);
7647         }
7648     }
7649
7650 no_data_clone:
7651     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7652
7653     my $disk = dclone($drive);
7654     delete $disk->{format};
7655     $disk->{file} = $newvolid;
7656     $disk->{size} = $size if defined($size);
7657
7658     return $disk;
7659 }
7660
7661 sub get_running_qemu_version {
7662     my ($vmid) = @_;
7663     my $res = mon_cmd($vmid, "query-version");
7664     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7665 }
7666
7667 sub qemu_use_old_bios_files {
7668     my ($machine_type) = @_;
7669
7670     return if !$machine_type;
7671
7672     my $use_old_bios_files = undef;
7673
7674     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7675         $machine_type = $1;
7676         $use_old_bios_files = 1;
7677     } else {
7678         my $version = extract_version($machine_type, kvm_user_version());
7679         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7680         # load new efi bios files on migration. So this hack is required to allow
7681         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7682         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7683         $use_old_bios_files = !min_version($version, 2, 4);
7684     }
7685
7686     return ($use_old_bios_files, $machine_type);
7687 }
7688
7689 sub get_efivars_size {
7690     my ($conf) = @_;
7691     my $arch = get_vm_arch($conf);
7692     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7693     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7694     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7695     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7696     return -s $ovmf_vars;
7697 }
7698
7699 sub update_efidisk_size {
7700     my ($conf) = @_;
7701
7702     return if !defined($conf->{efidisk0});
7703
7704     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7705     $disk->{size} = get_efivars_size($conf);
7706     $conf->{efidisk0} = print_drive($disk);
7707
7708     return;
7709 }
7710
7711 sub update_tpmstate_size {
7712     my ($conf) = @_;
7713
7714     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7715     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7716     $conf->{tpmstate0} = print_drive($disk);
7717 }
7718
7719 sub create_efidisk($$$$$$$) {
7720     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7721
7722     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7723     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7724
7725     my $vars_size_b = -s $ovmf_vars;
7726     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7727     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7728     PVE::Storage::activate_volumes($storecfg, [$volid]);
7729
7730     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7731     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7732
7733     return ($volid, $size/1024);
7734 }
7735
7736 sub vm_iothreads_list {
7737     my ($vmid) = @_;
7738
7739     my $res = mon_cmd($vmid, 'query-iothreads');
7740
7741     my $iothreads = {};
7742     foreach my $iothread (@$res) {
7743         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7744     }
7745
7746     return $iothreads;
7747 }
7748
7749 sub scsihw_infos {
7750     my ($conf, $drive) = @_;
7751
7752     my $maxdev = 0;
7753
7754     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7755         $maxdev = 7;
7756     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7757         $maxdev = 1;
7758     } else {
7759         $maxdev = 256;
7760     }
7761
7762     my $controller = int($drive->{index} / $maxdev);
7763     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7764         ? "virtioscsi"
7765         : "scsihw";
7766
7767     return ($maxdev, $controller, $controller_prefix);
7768 }
7769
7770 sub windows_version {
7771     my ($ostype) = @_;
7772
7773     return 0 if !$ostype;
7774
7775     my $winversion = 0;
7776
7777     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7778         $winversion = 5;
7779     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7780         $winversion = 6;
7781     } elsif ($ostype =~ m/^win(\d+)$/) {
7782         $winversion = $1;
7783     }
7784
7785     return $winversion;
7786 }
7787
7788 sub resolve_dst_disk_format {
7789         my ($storecfg, $storeid, $src_volname, $format) = @_;
7790         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7791
7792         if (!$format) {
7793             # if no target format is specified, use the source disk format as hint
7794             if ($src_volname) {
7795                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7796                 $format = qemu_img_format($scfg, $src_volname);
7797             } else {
7798                 return $defFormat;
7799             }
7800         }
7801
7802         # test if requested format is supported - else use default
7803         my $supported = grep { $_ eq $format } @$validFormats;
7804         $format = $defFormat if !$supported;
7805         return $format;
7806 }
7807
7808 # NOTE: if this logic changes, please update docs & possibly gui logic
7809 sub find_vmstate_storage {
7810     my ($conf, $storecfg) = @_;
7811
7812     # first, return storage from conf if set
7813     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7814
7815     my ($target, $shared, $local);
7816
7817     foreach_storage_used_by_vm($conf, sub {
7818         my ($sid) = @_;
7819         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7820         my $dst = $scfg->{shared} ? \$shared : \$local;
7821         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7822     });
7823
7824     # second, use shared storage where VM has at least one disk
7825     # third, use local storage where VM has at least one disk
7826     # fall back to local storage
7827     $target = $shared // $local // 'local';
7828
7829     return $target;
7830 }
7831
7832 sub generate_uuid {
7833     my ($uuid, $uuid_str);
7834     UUID::generate($uuid);
7835     UUID::unparse($uuid, $uuid_str);
7836     return $uuid_str;
7837 }
7838
7839 sub generate_smbios1_uuid {
7840     return "uuid=".generate_uuid();
7841 }
7842
7843 sub nbd_stop {
7844     my ($vmid) = @_;
7845
7846     mon_cmd($vmid, 'nbd-server-stop');
7847 }
7848
7849 sub create_reboot_request {
7850     my ($vmid) = @_;
7851     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7852         or die "failed to create reboot trigger file: $!\n";
7853     close($fh);
7854 }
7855
7856 sub clear_reboot_request {
7857     my ($vmid) = @_;
7858     my $path = "/run/qemu-server/$vmid.reboot";
7859     my $res = 0;
7860
7861     $res = unlink($path);
7862     die "could not remove reboot request for $vmid: $!"
7863         if !$res && $! != POSIX::ENOENT;
7864
7865     return $res;
7866 }
7867
7868 sub bootorder_from_legacy {
7869     my ($conf, $bootcfg) = @_;
7870
7871     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7872     my $bootindex_hash = {};
7873     my $i = 1;
7874     foreach my $o (split(//, $boot)) {
7875         $bootindex_hash->{$o} = $i*100;
7876         $i++;
7877     }
7878
7879     my $bootorder = {};
7880
7881     PVE::QemuConfig->foreach_volume($conf, sub {
7882         my ($ds, $drive) = @_;
7883
7884         if (drive_is_cdrom ($drive, 1)) {
7885             if ($bootindex_hash->{d}) {
7886                 $bootorder->{$ds} = $bootindex_hash->{d};
7887                 $bootindex_hash->{d} += 1;
7888             }
7889         } elsif ($bootindex_hash->{c}) {
7890             $bootorder->{$ds} = $bootindex_hash->{c}
7891                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7892             $bootindex_hash->{c} += 1;
7893         }
7894     });
7895
7896     if ($bootindex_hash->{n}) {
7897         for (my $i = 0; $i < $MAX_NETS; $i++) {
7898             my $netname = "net$i";
7899             next if !$conf->{$netname};
7900             $bootorder->{$netname} = $bootindex_hash->{n};
7901             $bootindex_hash->{n} += 1;
7902         }
7903     }
7904
7905     return $bootorder;
7906 }
7907
7908 # Generate default device list for 'boot: order=' property. Matches legacy
7909 # default boot order, but with explicit device names. This is important, since
7910 # the fallback for when neither 'order' nor the old format is specified relies
7911 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7912 sub get_default_bootdevices {
7913     my ($conf) = @_;
7914
7915     my @ret = ();
7916
7917     # harddisk
7918     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7919     push @ret, $first if $first;
7920
7921     # cdrom
7922     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7923     push @ret, $first if $first;
7924
7925     # network
7926     for (my $i = 0; $i < $MAX_NETS; $i++) {
7927         my $netname = "net$i";
7928         next if !$conf->{$netname};
7929         push @ret, $netname;
7930         last;
7931     }
7932
7933     return \@ret;
7934 }
7935
7936 sub device_bootorder {
7937     my ($conf) = @_;
7938
7939     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7940
7941     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7942
7943     my $bootorder = {};
7944     if (!defined($boot) || $boot->{legacy}) {
7945         $bootorder = bootorder_from_legacy($conf, $boot);
7946     } elsif ($boot->{order}) {
7947         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7948         for my $dev (PVE::Tools::split_list($boot->{order})) {
7949             $bootorder->{$dev} = $i++;
7950         }
7951     }
7952
7953     return $bootorder;
7954 }
7955
7956 sub register_qmeventd_handle {
7957     my ($vmid) = @_;
7958
7959     my $fh;
7960     my $peer = "/var/run/qmeventd.sock";
7961     my $count = 0;
7962
7963     for (;;) {
7964         $count++;
7965         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7966         last if $fh;
7967         if ($! != EINTR && $! != EAGAIN) {
7968             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7969         }
7970         if ($count > 4) {
7971             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7972               . "after $count retries\n";
7973         }
7974         usleep(25000);
7975     }
7976
7977     # send handshake to mark VM as backing up
7978     print $fh to_json({vzdump => {vmid => "$vmid"}});
7979
7980     # return handle to be closed later when inhibit is no longer required
7981     return $fh;
7982 }
7983
7984 # bash completion helper
7985
7986 sub complete_backup_archives {
7987     my ($cmdname, $pname, $cvalue) = @_;
7988
7989     my $cfg = PVE::Storage::config();
7990
7991     my $storeid;
7992
7993     if ($cvalue =~ m/^([^:]+):/) {
7994         $storeid = $1;
7995     }
7996
7997     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7998
7999     my $res = [];
8000     foreach my $id (keys %$data) {
8001         foreach my $item (@{$data->{$id}}) {
8002             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8003             push @$res, $item->{volid} if defined($item->{volid});
8004         }
8005     }
8006
8007     return $res;
8008 }
8009
8010 my $complete_vmid_full = sub {
8011     my ($running) = @_;
8012
8013     my $idlist = vmstatus();
8014
8015     my $res = [];
8016
8017     foreach my $id (keys %$idlist) {
8018         my $d = $idlist->{$id};
8019         if (defined($running)) {
8020             next if $d->{template};
8021             next if $running && $d->{status} ne 'running';
8022             next if !$running && $d->{status} eq 'running';
8023         }
8024         push @$res, $id;
8025
8026     }
8027     return $res;
8028 };
8029
8030 sub complete_vmid {
8031     return &$complete_vmid_full();
8032 }
8033
8034 sub complete_vmid_stopped {
8035     return &$complete_vmid_full(0);
8036 }
8037
8038 sub complete_vmid_running {
8039     return &$complete_vmid_full(1);
8040 }
8041
8042 sub complete_storage {
8043
8044     my $cfg = PVE::Storage::config();
8045     my $ids = $cfg->{ids};
8046
8047     my $res = [];
8048     foreach my $sid (keys %$ids) {
8049         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8050         next if !$ids->{$sid}->{content}->{images};
8051         push @$res, $sid;
8052     }
8053
8054     return $res;
8055 }
8056
8057 sub complete_migration_storage {
8058     my ($cmd, $param, $current_value, $all_args) = @_;
8059
8060     my $targetnode = @$all_args[1];
8061
8062     my $cfg = PVE::Storage::config();
8063     my $ids = $cfg->{ids};
8064
8065     my $res = [];
8066     foreach my $sid (keys %$ids) {
8067         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8068         next if !$ids->{$sid}->{content}->{images};
8069         push @$res, $sid;
8070     }
8071
8072     return $res;
8073 }
8074
8075 sub vm_is_paused {
8076     my ($vmid) = @_;
8077     my $qmpstatus = eval {
8078         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8079         mon_cmd($vmid, "query-status");
8080     };
8081     warn "$@\n" if $@;
8082     return $qmpstatus && $qmpstatus->{status} eq "paused";
8083 }
8084
8085 sub check_volume_storage_type {
8086     my ($storecfg, $vol) = @_;
8087
8088     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8089     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8090     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8091
8092     die "storage '$storeid' does not support content-type '$vtype'\n"
8093         if !$scfg->{content}->{$vtype};
8094
8095     return 1;
8096 }
8097
8098 1;