PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. The default is read from the"
 371             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.",
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.",
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1040
1041     return verify_volume_id_or_absolute_path($volid, $noerr);
1042 }
1043
1044 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1045 sub verify_volume_id_or_absolute_path {
1046     my ($volid, $noerr) = @_;
1047
1048     return $volid if $volid =~ m|^/|;
1049
1050     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1051     if ($@) {
1052         return if $noerr;
1053         die $@;
1054     }
1055     return $volid;
1056 }
1057
1058 my $usb_fmt = {
1059     host => {
1060         default_key => 1,
1061         type => 'string', format => 'pve-qm-usb-device',
1062         format_description => 'HOSTUSBDEVICE|spice',
1063         description => <<EODESCR,
1064 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1065
1066  'bus-port(.port)*' (decimal numbers) or
1067  'vendor_id:product_id' (hexadeciaml numbers) or
1068  'spice'
1069
1070 You can use the 'lsusb -t' command to list existing usb devices.
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1073 machines - use with special care.
1074
1075 The value 'spice' can be used to add a usb redirection devices for spice.
1076 EODESCR
1077     },
1078     usb3 => {
1079         optional => 1,
1080         type => 'boolean',
1081         description => "Specifies whether if given host option is a USB3 device or port.",
1082         default => 0,
1083     },
1084 };
1085
1086 my $usbdesc = {
1087     optional => 1,
1088     type => 'string', format => $usb_fmt,
1089     description => "Configure an USB device (n is 0 to 4).",
1090 };
1091 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1092
1093 my $serialdesc = {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '(/dev/.+|socket)',
1097         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1098         verbose_description =>  <<EODESCR,
1099 Create a serial device inside the VM (n is 0 to 3), and pass through a
1100 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1101 host side (use 'qm terminal' to open a terminal connection).
1102
1103 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1104 use with special care.
1105
1106 CAUTION: Experimental! User reported problems with this option.
1107 EODESCR
1108 };
1109
1110 my $paralleldesc= {
1111         optional => 1,
1112         type => 'string',
1113         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1114         description =>  "Map host parallel devices (n is 0 to 2).",
1115         verbose_description =>  <<EODESCR,
1116 Map host parallel devices (n is 0 to 2).
1117
1118 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1119 machines - use with special care.
1120
1121 CAUTION: Experimental! User reported problems with this option.
1122 EODESCR
1123 };
1124
1125 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1126     $confdesc->{"parallel$i"} = $paralleldesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1130     $confdesc->{"serial$i"} = $serialdesc;
1131 }
1132
1133 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1134     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1135 }
1136
1137 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1138     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1139 }
1140
1141 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1142     $confdesc->{"usb$i"} = $usbdesc;
1143 }
1144
1145 my $boot_fmt = {
1146     legacy => {
1147         optional => 1,
1148         default_key => 1,
1149         type => 'string',
1150         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1151                      . " Deprecated, use 'order=' instead.",
1152         pattern => '[acdn]{1,4}',
1153         format_description => "[acdn]{1,4}",
1154
1155         # note: this is also the fallback if boot: is not given at all
1156         default => 'cdn',
1157     },
1158     order => {
1159         optional => 1,
1160         type => 'string',
1161         format => 'pve-qm-bootdev-list',
1162         format_description => "device[;device...]",
1163         description => <<EODESC,
1164 The guest will attempt to boot from devices in the order they appear here.
1165
1166 Disks, optical drives and passed-through storage USB devices will be directly
1167 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1168 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1169
1170 Note that only devices in this list will be marked as bootable and thus loaded
1171 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1172 (e.g. software-raid), you need to specify all of them here.
1173
1174 Overrides the deprecated 'legacy=[acdn]*' value when given.
1175 EODESC
1176     },
1177 };
1178 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1179
1180 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1181 sub verify_bootdev {
1182     my ($dev, $noerr) = @_;
1183
1184     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1185     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1186
1187     my $check = sub {
1188         my ($base) = @_;
1189         return 0 if $dev !~ m/^$base\d+$/;
1190         return 0 if !$confdesc->{$dev};
1191         return 1;
1192     };
1193
1194     return $dev if $check->("net");
1195     return $dev if $check->("usb");
1196     return $dev if $check->("hostpci");
1197
1198     return if $noerr;
1199     die "invalid boot device '$dev'\n";
1200 }
1201
1202 sub print_bootorder {
1203     my ($devs) = @_;
1204     return "" if !@$devs;
1205     my $data = { order => join(';', @$devs) };
1206     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1207 }
1208
1209 my $kvm_api_version = 0;
1210
1211 sub kvm_version {
1212     return $kvm_api_version if $kvm_api_version;
1213
1214     open my $fh, '<', '/dev/kvm' or return;
1215
1216     # 0xae00 => KVM_GET_API_VERSION
1217     $kvm_api_version = ioctl($fh, 0xae00, 0);
1218     close($fh);
1219
1220     return $kvm_api_version;
1221 }
1222
1223 my $kvm_user_version = {};
1224 my $kvm_mtime = {};
1225
1226 sub kvm_user_version {
1227     my ($binary) = @_;
1228
1229     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1230     my $st = stat($binary);
1231
1232     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1233     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1234         $cachedmtime == $st->mtime;
1235
1236     $kvm_user_version->{$binary} = 'unknown';
1237     $kvm_mtime->{$binary} = $st->mtime;
1238
1239     my $code = sub {
1240         my $line = shift;
1241         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1242             $kvm_user_version->{$binary} = $2;
1243         }
1244     };
1245
1246     eval { run_command([$binary, '--version'], outfunc => $code); };
1247     warn $@ if $@;
1248
1249     return $kvm_user_version->{$binary};
1250
1251 }
1252 my sub extract_version {
1253     my ($machine_type, $version) = @_;
1254     $version = kvm_user_version() if !defined($version);
1255     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1256 }
1257
1258 sub kernel_has_vhost_net {
1259     return -c '/dev/vhost-net';
1260 }
1261
1262 sub option_exists {
1263     my $key = shift;
1264     return defined($confdesc->{$key});
1265 }
1266
1267 my $cdrom_path;
1268 sub get_cdrom_path {
1269
1270     return  $cdrom_path if $cdrom_path;
1271
1272     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1273     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1274     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1275 }
1276
1277 sub get_iso_path {
1278     my ($storecfg, $vmid, $cdrom) = @_;
1279
1280     if ($cdrom eq 'cdrom') {
1281         return get_cdrom_path();
1282     } elsif ($cdrom eq 'none') {
1283         return '';
1284     } elsif ($cdrom =~ m|^/|) {
1285         return $cdrom;
1286     } else {
1287         return PVE::Storage::path($storecfg, $cdrom);
1288     }
1289 }
1290
1291 # try to convert old style file names to volume IDs
1292 sub filename_to_volume_id {
1293     my ($vmid, $file, $media) = @_;
1294
1295      if (!($file eq 'none' || $file eq 'cdrom' ||
1296           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1297
1298         return if $file =~ m|/|;
1299
1300         if ($media && $media eq 'cdrom') {
1301             $file = "local:iso/$file";
1302         } else {
1303             $file = "local:$vmid/$file";
1304         }
1305     }
1306
1307     return $file;
1308 }
1309
1310 sub verify_media_type {
1311     my ($opt, $vtype, $media) = @_;
1312
1313     return if !$media;
1314
1315     my $etype;
1316     if ($media eq 'disk') {
1317         $etype = 'images';
1318     } elsif ($media eq 'cdrom') {
1319         $etype = 'iso';
1320     } else {
1321         die "internal error";
1322     }
1323
1324     return if ($vtype eq $etype);
1325
1326     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1327 }
1328
1329 sub cleanup_drive_path {
1330     my ($opt, $storecfg, $drive) = @_;
1331
1332     # try to convert filesystem paths to volume IDs
1333
1334     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1335         ($drive->{file} !~ m|^/dev/.+|) &&
1336         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1337         ($drive->{file} !~ m/^\d+$/)) {
1338         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1339         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1340             if !$vtype;
1341         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1342         verify_media_type($opt, $vtype, $drive->{media});
1343         $drive->{file} = $volid;
1344     }
1345
1346     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1347 }
1348
1349 sub parse_hotplug_features {
1350     my ($data) = @_;
1351
1352     my $res = {};
1353
1354     return $res if $data eq '0';
1355
1356     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1357
1358     foreach my $feature (PVE::Tools::split_list($data)) {
1359         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1360             $res->{$1} = 1;
1361         } else {
1362             die "invalid hotplug feature '$feature'\n";
1363         }
1364     }
1365     return $res;
1366 }
1367
1368 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1369 sub pve_verify_hotplug_features {
1370     my ($value, $noerr) = @_;
1371
1372     return $value if parse_hotplug_features($value);
1373
1374     return if $noerr;
1375
1376     die "unable to parse hotplug option\n";
1377 }
1378
1379 sub scsi_inquiry {
1380     my($fh, $noerr) = @_;
1381
1382     my $SG_IO = 0x2285;
1383     my $SG_GET_VERSION_NUM = 0x2282;
1384
1385     my $versionbuf = "\x00" x 8;
1386     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1387     if (!$ret) {
1388         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1389         return;
1390     }
1391     my $version = unpack("I", $versionbuf);
1392     if ($version < 30000) {
1393         die "scsi generic interface too old\n"  if !$noerr;
1394         return;
1395     }
1396
1397     my $buf = "\x00" x 36;
1398     my $sensebuf = "\x00" x 8;
1399     my $cmd = pack("C x3 C x1", 0x12, 36);
1400
1401     # see /usr/include/scsi/sg.h
1402     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1403
1404     my $packet = pack(
1405         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1406     );
1407
1408     $ret = ioctl($fh, $SG_IO, $packet);
1409     if (!$ret) {
1410         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my @res = unpack($sg_io_hdr_t, $packet);
1415     if ($res[17] || $res[18]) {
1416         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1417         return;
1418     }
1419
1420     my $res = {};
1421     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1422
1423     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1424     $res->{type} &= 0x1F;
1425
1426     return $res;
1427 }
1428
1429 sub path_is_scsi {
1430     my ($path) = @_;
1431
1432     my $fh = IO::File->new("+<$path") || return;
1433     my $res = scsi_inquiry($fh, 1);
1434     close($fh);
1435
1436     return $res;
1437 }
1438
1439 sub print_tabletdevice_full {
1440     my ($conf, $arch) = @_;
1441
1442     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1443
1444     # we use uhci for old VMs because tablet driver was buggy in older qemu
1445     my $usbbus;
1446     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1447         $usbbus = 'ehci';
1448     } else {
1449         $usbbus = 'uhci';
1450     }
1451
1452     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1453 }
1454
1455 sub print_keyboarddevice_full {
1456     my ($conf, $arch) = @_;
1457
1458     return if $arch ne 'aarch64';
1459
1460     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1461 }
1462
1463 my sub get_drive_id {
1464     my ($drive) = @_;
1465     return "$drive->{interface}$drive->{index}";
1466 }
1467
1468 sub print_drivedevice_full {
1469     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1470
1471     my $device = '';
1472     my $maxdev = 0;
1473
1474     my $drive_id = get_drive_id($drive);
1475     if ($drive->{interface} eq 'virtio') {
1476         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1477         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1478         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1479     } elsif ($drive->{interface} eq 'scsi') {
1480
1481         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1482         my $unit = $drive->{index} % $maxdev;
1483         my $devicetype = 'hd';
1484         my $path = '';
1485         if (drive_is_cdrom($drive)) {
1486             $devicetype = 'cd';
1487         } else {
1488             if ($drive->{file} =~ m|^/|) {
1489                 $path = $drive->{file};
1490                 if (my $info = path_is_scsi($path)) {
1491                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1492                         $devicetype = 'block';
1493                     } elsif ($info->{type} == 1) { # tape
1494                         $devicetype = 'generic';
1495                     }
1496                 }
1497             } else {
1498                  $path = PVE::Storage::path($storecfg, $drive->{file});
1499             }
1500
1501             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1502             my $version = extract_version($machine_type, kvm_user_version());
1503             if ($path =~ m/^iscsi\:\/\// &&
1504                !min_version($version, 4, 1)) {
1505                 $devicetype = 'generic';
1506             }
1507         }
1508
1509         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1510             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1511         } else {
1512             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1513                 .",lun=$drive->{index}";
1514         }
1515         $device .= ",drive=drive-$drive_id,id=$drive_id";
1516
1517         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1518             $device .= ",rotation_rate=1";
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521
1522     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1523         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1524         my $controller = int($drive->{index} / $maxdev);
1525         my $unit = $drive->{index} % $maxdev;
1526         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1527
1528         $device = "ide-$devicetype";
1529         if ($drive->{interface} eq 'ide') {
1530             $device .= ",bus=ide.$controller,unit=$unit";
1531         } else {
1532             $device .= ",bus=ahci$controller.$unit";
1533         }
1534         $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536         if ($devicetype eq 'hd') {
1537             if (my $model = $drive->{model}) {
1538                 $model = URI::Escape::uri_unescape($model);
1539                 $device .= ",model=$model";
1540             }
1541             if ($drive->{ssd}) {
1542                 $device .= ",rotation_rate=1";
1543             }
1544         }
1545         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1546     } elsif ($drive->{interface} eq 'usb') {
1547         die "implement me";
1548         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1549     } else {
1550         die "unsupported interface type";
1551     }
1552
1553     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1554
1555     if (my $serial = $drive->{serial}) {
1556         $serial = URI::Escape::uri_unescape($serial);
1557         $device .= ",serial=$serial";
1558     }
1559
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_commandline_full {
1579     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format = $drive->{format};
1584     my $drive_id = get_drive_id($drive);
1585
1586     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1587     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1588
1589     if (drive_is_cdrom($drive)) {
1590         $path = get_iso_path($storecfg, $vmid, $volid);
1591         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1592     } else {
1593         if ($storeid) {
1594             $path = PVE::Storage::path($storecfg, $volid);
1595             $format //= qemu_img_format($scfg, $volname);
1596         } else {
1597             $path = $volid;
1598             $format //= "raw";
1599         }
1600    }
1601
1602    my $is_rbd = $path =~ m/^rbd:/;
1603
1604     my $opts = '';
1605     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1606     foreach my $o (@qemu_drive_options) {
1607         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1608     }
1609
1610     # snapshot only accepts on|off
1611     if (defined($drive->{snapshot})) {
1612         my $v = $drive->{snapshot} ? 'on' : 'off';
1613         $opts .= ",snapshot=$v";
1614     }
1615
1616     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1617         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1618     }
1619
1620     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1621         my ($dir, $qmpname) = @$type;
1622         if (my $v = $drive->{"mbps$dir"}) {
1623             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1624         }
1625         if (my $v = $drive->{"mbps${dir}_max"}) {
1626             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1627         }
1628         if (my $v = $drive->{"bps${dir}_max_length"}) {
1629             $opts .= ",throttling.bps$qmpname-max-length=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}"}) {
1632             $opts .= ",throttling.iops$qmpname=$v";
1633         }
1634         if (my $v = $drive->{"iops${dir}_max"}) {
1635             $opts .= ",throttling.iops$qmpname-max=$v";
1636         }
1637         if (my $v = $drive->{"iops${dir}_max_length"}) {
1638             $opts .= ",throttling.iops$qmpname-max-length=$v";
1639         }
1640     }
1641
1642     if ($pbs_name) {
1643         $format = "rbd" if $is_rbd;
1644         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1645             if !$format;
1646         $opts .= ",format=alloc-track,file.driver=$format";
1647     } elsif ($format) {
1648         $opts .= ",format=$format";
1649     }
1650
1651     my $cache_direct = 0;
1652
1653     if (my $cache = $drive->{cache}) {
1654         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1655     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1656         $opts .= ",cache=none";
1657         $cache_direct = 1;
1658     }
1659
1660     # io_uring with cache mode writeback or writethrough on krbd will hang...
1661     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1662
1663     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1664     # sometimes, just plain disable...
1665     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1666
1667     if (!$drive->{aio}) {
1668         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1669             # io_uring supports all cache modes
1670             $opts .= ",aio=io_uring";
1671         } else {
1672             # aio native works only with O_DIRECT
1673             if($cache_direct) {
1674                 $opts .= ",aio=native";
1675             } else {
1676                 $opts .= ",aio=threads";
1677             }
1678         }
1679     }
1680
1681     if (!drive_is_cdrom($drive)) {
1682         my $detectzeroes;
1683         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1684             $detectzeroes = 'off';
1685         } elsif ($drive->{discard}) {
1686             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1687         } else {
1688             # This used to be our default with discard not being specified:
1689             $detectzeroes = 'on';
1690         }
1691
1692         # note: 'detect-zeroes' works per blockdev and we want it to persist
1693         # after the alloc-track is removed, so put it on 'file' directly
1694         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1695         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1696     }
1697
1698     if ($pbs_name) {
1699         $opts .= ",backing=$pbs_name";
1700         $opts .= ",auto-remove=on";
1701     }
1702
1703     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1704     my $file_param = "file";
1705     if ($pbs_name) {
1706         # non-rbd drivers require the underlying file to be a seperate block
1707         # node, so add a second .file indirection
1708         $file_param .= ".file" if !$is_rbd;
1709         $file_param .= ".filename";
1710     }
1711     my $pathinfo = $path ? "$file_param=$path," : '';
1712
1713     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1714 }
1715
1716 sub print_pbs_blockdev {
1717     my ($pbs_conf, $pbs_name) = @_;
1718     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1719     $blockdev .= ",repository=$pbs_conf->{repository}";
1720     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1721     $blockdev .= ",archive=$pbs_conf->{archive}";
1722     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1723     return $blockdev;
1724 }
1725
1726 sub print_netdevice_full {
1727     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1728
1729     my $device = $net->{model};
1730     if ($net->{model} eq 'virtio') {
1731          $device = 'virtio-net-pci';
1732      };
1733
1734     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1735     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1736     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1737         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1738         # and out of each queue plus one config interrupt and control vector queue
1739         my $vectors = $net->{queues} * 2 + 2;
1740         $tmpstr .= ",vectors=$vectors,mq=on";
1741     }
1742     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1743
1744     if (my $mtu = $net->{mtu}) {
1745         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1746             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1747             if ($mtu == 1) {
1748                  $mtu = $bridge_mtu;
1749             } elsif ($mtu < 576) {
1750                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1751             } elsif ($mtu > $bridge_mtu) {
1752                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1753             }
1754             $tmpstr .= ",host_mtu=$mtu";
1755         } else {
1756             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1757         }
1758     }
1759
1760     if ($use_old_bios_files) {
1761         my $romfile;
1762         if ($device eq 'virtio-net-pci') {
1763             $romfile = 'pxe-virtio.rom';
1764         } elsif ($device eq 'e1000') {
1765             $romfile = 'pxe-e1000.rom';
1766         } elsif ($device eq 'e1000e') {
1767             $romfile = 'pxe-e1000e.rom';
1768         } elsif ($device eq 'ne2k') {
1769             $romfile = 'pxe-ne2k_pci.rom';
1770         } elsif ($device eq 'pcnet') {
1771             $romfile = 'pxe-pcnet.rom';
1772         } elsif ($device eq 'rtl8139') {
1773             $romfile = 'pxe-rtl8139.rom';
1774         }
1775         $tmpstr .= ",romfile=$romfile" if $romfile;
1776     }
1777
1778     return $tmpstr;
1779 }
1780
1781 sub print_netdev_full {
1782     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1783
1784     my $i = '';
1785     if ($netid =~ m/^net(\d+)$/) {
1786         $i = int($1);
1787     }
1788
1789     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1790
1791     my $ifname = "tap${vmid}i$i";
1792
1793     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1794     die "interface name '$ifname' is too long (max 15 character)\n"
1795         if length($ifname) >= 16;
1796
1797     my $vhostparam = '';
1798     if (is_native($arch)) {
1799         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1800     }
1801
1802     my $vmname = $conf->{name} || "vm$vmid";
1803
1804     my $netdev = "";
1805     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1806
1807     if ($net->{bridge}) {
1808         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1809             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1810     } else {
1811         $netdev = "type=user,id=$netid,hostname=$vmname";
1812     }
1813
1814     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1815
1816     return $netdev;
1817 }
1818
1819 my $vga_map = {
1820     'cirrus' => 'cirrus-vga',
1821     'std' => 'VGA',
1822     'vmware' => 'vmware-svga',
1823     'virtio' => 'virtio-vga',
1824 };
1825
1826 sub print_vga_device {
1827     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1828
1829     my $type = $vga_map->{$vga->{type}};
1830     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1831         $type = 'virtio-gpu';
1832     }
1833     my $vgamem_mb = $vga->{memory};
1834
1835     my $max_outputs = '';
1836     if ($qxlnum) {
1837         $type = $id ? 'qxl' : 'qxl-vga';
1838
1839         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1840             # set max outputs so linux can have up to 4 qxl displays with one device
1841             if (min_version($machine_version, 4, 1)) {
1842                 $max_outputs = ",max_outputs=4";
1843             }
1844         }
1845     }
1846
1847     die "no devicetype for $vga->{type}\n" if !$type;
1848
1849     my $memory = "";
1850     if ($vgamem_mb) {
1851         if ($vga->{type} eq 'virtio') {
1852             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1853             $memory = ",max_hostmem=$bytes";
1854         } elsif ($qxlnum) {
1855             # from https://www.spice-space.org/multiple-monitors.html
1856             $memory = ",vgamem_mb=$vga->{memory}";
1857             my $ram = $vgamem_mb * 4;
1858             my $vram = $vgamem_mb * 2;
1859             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1860         } else {
1861             $memory = ",vgamem_mb=$vga->{memory}";
1862         }
1863     } elsif ($qxlnum && $id) {
1864         $memory = ",ram_size=67108864,vram_size=33554432";
1865     }
1866
1867     my $edidoff = "";
1868     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1869         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1870     }
1871
1872     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1873     my $vgaid = "vga" . ($id // '');
1874     my $pciaddr;
1875     if ($q35 && $vgaid eq 'vga') {
1876         # the first display uses pcie.0 bus on q35 machines
1877         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1878     } else {
1879         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1880     }
1881
1882     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1883 }
1884
1885 sub parse_number_sets {
1886     my ($set) = @_;
1887     my $res = [];
1888     foreach my $part (split(/;/, $set)) {
1889         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1890             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1891             push @$res, [ $1, $2 ];
1892         } else {
1893             die "invalid range: $part\n";
1894         }
1895     }
1896     return $res;
1897 }
1898
1899 sub parse_numa {
1900     my ($data) = @_;
1901
1902     my $res = parse_property_string($numa_fmt, $data);
1903     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1904     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1905     return $res;
1906 }
1907
1908 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1909 sub parse_net {
1910     my ($data) = @_;
1911
1912     my $res = eval { parse_property_string($net_fmt, $data) };
1913     if ($@) {
1914         warn $@;
1915         return;
1916     }
1917     if (!defined($res->{macaddr})) {
1918         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1919         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1920     }
1921     return $res;
1922 }
1923
1924 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1925 sub parse_ipconfig {
1926     my ($data) = @_;
1927
1928     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1929     if ($@) {
1930         warn $@;
1931         return;
1932     }
1933
1934     if ($res->{gw} && !$res->{ip}) {
1935         warn 'gateway specified without specifying an IP address';
1936         return;
1937     }
1938     if ($res->{gw6} && !$res->{ip6}) {
1939         warn 'IPv6 gateway specified without specifying an IPv6 address';
1940         return;
1941     }
1942     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1943         warn 'gateway specified together with DHCP';
1944         return;
1945     }
1946     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1947         # gw6 + auto/dhcp
1948         warn "IPv6 gateway specified together with $res->{ip6} address";
1949         return;
1950     }
1951
1952     if (!$res->{ip} && !$res->{ip6}) {
1953         return { ip => 'dhcp', ip6 => 'dhcp' };
1954     }
1955
1956     return $res;
1957 }
1958
1959 sub print_net {
1960     my $net = shift;
1961
1962     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1963 }
1964
1965 sub add_random_macs {
1966     my ($settings) = @_;
1967
1968     foreach my $opt (keys %$settings) {
1969         next if $opt !~ m/^net(\d+)$/;
1970         my $net = parse_net($settings->{$opt});
1971         next if !$net;
1972         $settings->{$opt} = print_net($net);
1973     }
1974 }
1975
1976 sub vm_is_volid_owner {
1977     my ($storecfg, $vmid, $volid) = @_;
1978
1979     if ($volid !~  m|^/|) {
1980         my ($path, $owner);
1981         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1982         if ($owner && ($owner == $vmid)) {
1983             return 1;
1984         }
1985     }
1986
1987     return;
1988 }
1989
1990 sub vmconfig_register_unused_drive {
1991     my ($storecfg, $vmid, $conf, $drive) = @_;
1992
1993     if (drive_is_cloudinit($drive)) {
1994         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1995         warn $@ if $@;
1996     } elsif (!drive_is_cdrom($drive)) {
1997         my $volid = $drive->{file};
1998         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1999             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2000         }
2001     }
2002 }
2003
2004 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2005 my $smbios1_fmt = {
2006     uuid => {
2007         type => 'string',
2008         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2009         format_description => 'UUID',
2010         description => "Set SMBIOS1 UUID.",
2011         optional => 1,
2012     },
2013     version => {
2014         type => 'string',
2015         pattern => '[A-Za-z0-9+\/]+={0,2}',
2016         format_description => 'Base64 encoded string',
2017         description => "Set SMBIOS1 version.",
2018         optional => 1,
2019     },
2020     serial => {
2021         type => 'string',
2022         pattern => '[A-Za-z0-9+\/]+={0,2}',
2023         format_description => 'Base64 encoded string',
2024         description => "Set SMBIOS1 serial number.",
2025         optional => 1,
2026     },
2027     manufacturer => {
2028         type => 'string',
2029         pattern => '[A-Za-z0-9+\/]+={0,2}',
2030         format_description => 'Base64 encoded string',
2031         description => "Set SMBIOS1 manufacturer.",
2032         optional => 1,
2033     },
2034     product => {
2035         type => 'string',
2036         pattern => '[A-Za-z0-9+\/]+={0,2}',
2037         format_description => 'Base64 encoded string',
2038         description => "Set SMBIOS1 product ID.",
2039         optional => 1,
2040     },
2041     sku => {
2042         type => 'string',
2043         pattern => '[A-Za-z0-9+\/]+={0,2}',
2044         format_description => 'Base64 encoded string',
2045         description => "Set SMBIOS1 SKU string.",
2046         optional => 1,
2047     },
2048     family => {
2049         type => 'string',
2050         pattern => '[A-Za-z0-9+\/]+={0,2}',
2051         format_description => 'Base64 encoded string',
2052         description => "Set SMBIOS1 family string.",
2053         optional => 1,
2054     },
2055     base64 => {
2056         type => 'boolean',
2057         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2058         optional => 1,
2059     },
2060 };
2061
2062 sub parse_smbios1 {
2063     my ($data) = @_;
2064
2065     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2066     warn $@ if $@;
2067     return $res;
2068 }
2069
2070 sub print_smbios1 {
2071     my ($smbios1) = @_;
2072     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2073 }
2074
2075 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2076
2077 sub parse_watchdog {
2078     my ($value) = @_;
2079
2080     return if !$value;
2081
2082     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2083     warn $@ if $@;
2084     return $res;
2085 }
2086
2087 sub parse_guest_agent {
2088     my ($conf) = @_;
2089
2090     return {} if !defined($conf->{agent});
2091
2092     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2093     warn $@ if $@;
2094
2095     # if the agent is disabled ignore the other potentially set properties
2096     return {} if !$res->{enabled};
2097     return $res;
2098 }
2099
2100 sub get_qga_key {
2101     my ($conf, $key) = @_;
2102     return undef if !defined($conf->{agent});
2103
2104     my $agent = parse_guest_agent($conf);
2105     return $agent->{$key};
2106 }
2107
2108 sub parse_vga {
2109     my ($value) = @_;
2110
2111     return {} if !$value;
2112     my $res = eval { parse_property_string($vga_fmt, $value) };
2113     warn $@ if $@;
2114     return $res;
2115 }
2116
2117 sub parse_rng {
2118     my ($value) = @_;
2119
2120     return if !$value;
2121
2122     my $res = eval { parse_property_string($rng_fmt, $value) };
2123     warn $@ if $@;
2124     return $res;
2125 }
2126
2127 sub parse_meta_info {
2128     my ($value) = @_;
2129
2130     return if !$value;
2131
2132     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2133     warn $@ if $@;
2134     return $res;
2135 }
2136
2137 sub new_meta_info_string {
2138     my () = @_; # for now do not allow to override any value
2139
2140     return PVE::JSONSchema::print_property_string(
2141         {
2142             'creation-qemu' => kvm_user_version(),
2143             ctime => "". int(time()),
2144         },
2145         $meta_info_fmt
2146     );
2147 }
2148
2149 sub qemu_created_version_fixups {
2150     my ($conf, $forcemachine, $kvmver) = @_;
2151
2152     my $meta = parse_meta_info($conf->{meta}) // {};
2153     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2154
2155     # check if we need to apply some handling for VMs that always use the latest machine version but
2156     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2157     # would be required (we do not want to pin machine version for non-windows OS type)
2158     if (
2159         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2160         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2161         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2162         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2163     ) {
2164         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2165         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2166             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2167             # and thus with the predictable interface naming of systemd
2168             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2169         }
2170     }
2171     return;
2172 }
2173
2174 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2175 sub verify_usb_device {
2176     my ($value, $noerr) = @_;
2177
2178     return $value if parse_usb_device($value);
2179
2180     return if $noerr;
2181
2182     die "unable to parse usb device\n";
2183 }
2184
2185 # add JSON properties for create and set function
2186 sub json_config_properties {
2187     my $prop = shift;
2188
2189     my $skip_json_config_opts = {
2190         parent => 1,
2191         snaptime => 1,
2192         vmstate => 1,
2193         runningmachine => 1,
2194         runningcpu => 1,
2195         meta => 1,
2196     };
2197
2198     foreach my $opt (keys %$confdesc) {
2199         next if $skip_json_config_opts->{$opt};
2200         $prop->{$opt} = $confdesc->{$opt};
2201     }
2202
2203     return $prop;
2204 }
2205
2206 # return copy of $confdesc_cloudinit to generate documentation
2207 sub cloudinit_config_properties {
2208
2209     return dclone($confdesc_cloudinit);
2210 }
2211
2212 sub check_type {
2213     my ($key, $value) = @_;
2214
2215     die "unknown setting '$key'\n" if !$confdesc->{$key};
2216
2217     my $type = $confdesc->{$key}->{type};
2218
2219     if (!defined($value)) {
2220         die "got undefined value\n";
2221     }
2222
2223     if ($value =~ m/[\n\r]/) {
2224         die "property contains a line feed\n";
2225     }
2226
2227     if ($type eq 'boolean') {
2228         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2229         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2230         die "type check ('boolean') failed - got '$value'\n";
2231     } elsif ($type eq 'integer') {
2232         return int($1) if $value =~ m/^(\d+)$/;
2233         die "type check ('integer') failed - got '$value'\n";
2234     } elsif ($type eq 'number') {
2235         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2236         die "type check ('number') failed - got '$value'\n";
2237     } elsif ($type eq 'string') {
2238         if (my $fmt = $confdesc->{$key}->{format}) {
2239             PVE::JSONSchema::check_format($fmt, $value);
2240             return $value;
2241         }
2242         $value =~ s/^\"(.*)\"$/$1/;
2243         return $value;
2244     } else {
2245         die "internal error"
2246     }
2247 }
2248
2249 sub destroy_vm {
2250     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2251
2252     my $conf = PVE::QemuConfig->load_config($vmid);
2253
2254     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2255
2256     if ($conf->{template}) {
2257         # check if any base image is still used by a linked clone
2258         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2259                 my ($ds, $drive) = @_;
2260                 return if drive_is_cdrom($drive);
2261
2262                 my $volid = $drive->{file};
2263                 return if !$volid || $volid =~ m|^/|;
2264
2265                 die "base volume '$volid' is still in use by linked cloned\n"
2266                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2267
2268         });
2269     }
2270
2271     my $volids = {};
2272     my $remove_owned_drive = sub {
2273         my ($ds, $drive) = @_;
2274         return if drive_is_cdrom($drive, 1);
2275
2276         my $volid = $drive->{file};
2277         return if !$volid || $volid =~ m|^/|;
2278         return if $volids->{$volid};
2279
2280         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2281         return if !$path || !$owner || ($owner != $vmid);
2282
2283         $volids->{$volid} = 1;
2284         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2285         warn "Could not remove disk '$volid', check manually: $@" if $@;
2286     };
2287
2288     # only remove disks owned by this VM (referenced in the config)
2289     my $include_opts = {
2290         include_unused => 1,
2291         extra_keys => ['vmstate'],
2292     };
2293     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2294
2295     for my $snap (values %{$conf->{snapshots}}) {
2296         next if !defined($snap->{vmstate});
2297         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2298         next if !defined($drive);
2299         $remove_owned_drive->('vmstate', $drive);
2300     }
2301
2302     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2303
2304     if ($purge_unreferenced) { # also remove unreferenced disk
2305         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2306         PVE::Storage::foreach_volid($vmdisks, sub {
2307             my ($volid, $sid, $volname, $d) = @_;
2308             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2309             warn $@ if $@;
2310         });
2311     }
2312
2313     if (defined $replacement_conf) {
2314         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2315     } else {
2316         PVE::QemuConfig->destroy_config($vmid);
2317     }
2318 }
2319
2320 sub parse_vm_config {
2321     my ($filename, $raw, $strict) = @_;
2322
2323     return if !defined($raw);
2324
2325     my $res = {
2326         digest => Digest::SHA::sha1_hex($raw),
2327         snapshots => {},
2328         pending => {},
2329     };
2330
2331     my $handle_error = sub {
2332         my ($msg) = @_;
2333
2334         if ($strict) {
2335             die $msg;
2336         } else {
2337             warn $msg;
2338         }
2339     };
2340
2341     $filename =~ m|/qemu-server/(\d+)\.conf$|
2342         || die "got strange filename '$filename'";
2343
2344     my $vmid = $1;
2345
2346     my $conf = $res;
2347     my $descr;
2348     my $section = '';
2349
2350     my @lines = split(/\n/, $raw);
2351     foreach my $line (@lines) {
2352         next if $line =~ m/^\s*$/;
2353
2354         if ($line =~ m/^\[PENDING\]\s*$/i) {
2355             $section = 'pending';
2356             if (defined($descr)) {
2357                 $descr =~ s/\s+$//;
2358                 $conf->{description} = $descr;
2359             }
2360             $descr = undef;
2361             $conf = $res->{$section} = {};
2362             next;
2363
2364         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2365             $section = $1;
2366             if (defined($descr)) {
2367                 $descr =~ s/\s+$//;
2368                 $conf->{description} = $descr;
2369             }
2370             $descr = undef;
2371             $conf = $res->{snapshots}->{$section} = {};
2372             next;
2373         }
2374
2375         if ($line =~ m/^\#(.*)\s*$/) {
2376             $descr = '' if !defined($descr);
2377             $descr .= PVE::Tools::decode_text($1) . "\n";
2378             next;
2379         }
2380
2381         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2382             $descr = '' if !defined($descr);
2383             $descr .= PVE::Tools::decode_text($2);
2384         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2385             $conf->{snapstate} = $1;
2386         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2387             my $key = $1;
2388             my $value = $2;
2389             $conf->{$key} = $value;
2390         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2391             my $value = $1;
2392             if ($section eq 'pending') {
2393                 $conf->{delete} = $value; # we parse this later
2394             } else {
2395                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2396             }
2397         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2398             my $key = $1;
2399             my $value = $2;
2400             eval { $value = check_type($key, $value); };
2401             if ($@) {
2402                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2403             } else {
2404                 $key = 'ide2' if $key eq 'cdrom';
2405                 my $fmt = $confdesc->{$key}->{format};
2406                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2407                     my $v = parse_drive($key, $value);
2408                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2409                         $v->{file} = $volid;
2410                         $value = print_drive($v);
2411                     } else {
2412                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2413                         next;
2414                     }
2415                 }
2416
2417                 $conf->{$key} = $value;
2418             }
2419         } else {
2420             $handle_error->("vm $vmid - unable to parse config: $line\n");
2421         }
2422     }
2423
2424     if (defined($descr)) {
2425         $descr =~ s/\s+$//;
2426         $conf->{description} = $descr;
2427     }
2428     delete $res->{snapstate}; # just to be sure
2429
2430     return $res;
2431 }
2432
2433 sub write_vm_config {
2434     my ($filename, $conf) = @_;
2435
2436     delete $conf->{snapstate}; # just to be sure
2437
2438     if ($conf->{cdrom}) {
2439         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2440         $conf->{ide2} = $conf->{cdrom};
2441         delete $conf->{cdrom};
2442     }
2443
2444     # we do not use 'smp' any longer
2445     if ($conf->{sockets}) {
2446         delete $conf->{smp};
2447     } elsif ($conf->{smp}) {
2448         $conf->{sockets} = $conf->{smp};
2449         delete $conf->{cores};
2450         delete $conf->{smp};
2451     }
2452
2453     my $used_volids = {};
2454
2455     my $cleanup_config = sub {
2456         my ($cref, $pending, $snapname) = @_;
2457
2458         foreach my $key (keys %$cref) {
2459             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2460                 $key eq 'snapstate' || $key eq 'pending';
2461             my $value = $cref->{$key};
2462             if ($key eq 'delete') {
2463                 die "propertry 'delete' is only allowed in [PENDING]\n"
2464                     if !$pending;
2465                 # fixme: check syntax?
2466                 next;
2467             }
2468             eval { $value = check_type($key, $value); };
2469             die "unable to parse value of '$key' - $@" if $@;
2470
2471             $cref->{$key} = $value;
2472
2473             if (!$snapname && is_valid_drivename($key)) {
2474                 my $drive = parse_drive($key, $value);
2475                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2476             }
2477         }
2478     };
2479
2480     &$cleanup_config($conf);
2481
2482     &$cleanup_config($conf->{pending}, 1);
2483
2484     foreach my $snapname (keys %{$conf->{snapshots}}) {
2485         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2486         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2487     }
2488
2489     # remove 'unusedX' settings if we re-add a volume
2490     foreach my $key (keys %$conf) {
2491         my $value = $conf->{$key};
2492         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2493             delete $conf->{$key};
2494         }
2495     }
2496
2497     my $generate_raw_config = sub {
2498         my ($conf, $pending) = @_;
2499
2500         my $raw = '';
2501
2502         # add description as comment to top of file
2503         if (defined(my $descr = $conf->{description})) {
2504             if ($descr) {
2505                 foreach my $cl (split(/\n/, $descr)) {
2506                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2507                 }
2508             } else {
2509                 $raw .= "#\n" if $pending;
2510             }
2511         }
2512
2513         foreach my $key (sort keys %$conf) {
2514             next if $key =~ /^(digest|description|pending|snapshots)$/;
2515             $raw .= "$key: $conf->{$key}\n";
2516         }
2517         return $raw;
2518     };
2519
2520     my $raw = &$generate_raw_config($conf);
2521
2522     if (scalar(keys %{$conf->{pending}})){
2523         $raw .= "\n[PENDING]\n";
2524         $raw .= &$generate_raw_config($conf->{pending}, 1);
2525     }
2526
2527     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2528         $raw .= "\n[$snapname]\n";
2529         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2530     }
2531
2532     return $raw;
2533 }
2534
2535 sub load_defaults {
2536
2537     my $res = {};
2538
2539     # we use static defaults from our JSON schema configuration
2540     foreach my $key (keys %$confdesc) {
2541         if (defined(my $default = $confdesc->{$key}->{default})) {
2542             $res->{$key} = $default;
2543         }
2544     }
2545
2546     return $res;
2547 }
2548
2549 sub config_list {
2550     my $vmlist = PVE::Cluster::get_vmlist();
2551     my $res = {};
2552     return $res if !$vmlist || !$vmlist->{ids};
2553     my $ids = $vmlist->{ids};
2554     my $nodename = nodename();
2555
2556     foreach my $vmid (keys %$ids) {
2557         my $d = $ids->{$vmid};
2558         next if !$d->{node} || $d->{node} ne $nodename;
2559         next if !$d->{type} || $d->{type} ne 'qemu';
2560         $res->{$vmid}->{exists} = 1;
2561     }
2562     return $res;
2563 }
2564
2565 # test if VM uses local resources (to prevent migration)
2566 sub check_local_resources {
2567     my ($conf, $noerr) = @_;
2568
2569     my @loc_res = ();
2570
2571     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2572     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2573
2574     push @loc_res, "ivshmem" if $conf->{ivshmem};
2575
2576     foreach my $k (keys %$conf) {
2577         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2578         # sockets are safe: they will recreated be on the target side post-migrate
2579         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2580         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2581     }
2582
2583     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2584
2585     return \@loc_res;
2586 }
2587
2588 # check if used storages are available on all nodes (use by migrate)
2589 sub check_storage_availability {
2590     my ($storecfg, $conf, $node) = @_;
2591
2592     PVE::QemuConfig->foreach_volume($conf, sub {
2593         my ($ds, $drive) = @_;
2594
2595         my $volid = $drive->{file};
2596         return if !$volid;
2597
2598         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2599         return if !$sid;
2600
2601         # check if storage is available on both nodes
2602         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2603         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2604
2605         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2606
2607         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2608             if !$scfg->{content}->{$vtype};
2609    });
2610 }
2611
2612 # list nodes where all VM images are available (used by has_feature API)
2613 sub shared_nodes {
2614     my ($conf, $storecfg) = @_;
2615
2616     my $nodelist = PVE::Cluster::get_nodelist();
2617     my $nodehash = { map { $_ => 1 } @$nodelist };
2618     my $nodename = nodename();
2619
2620     PVE::QemuConfig->foreach_volume($conf, sub {
2621         my ($ds, $drive) = @_;
2622
2623         my $volid = $drive->{file};
2624         return if !$volid;
2625
2626         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2627         if ($storeid) {
2628             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2629             if ($scfg->{disable}) {
2630                 $nodehash = {};
2631             } elsif (my $avail = $scfg->{nodes}) {
2632                 foreach my $node (keys %$nodehash) {
2633                     delete $nodehash->{$node} if !$avail->{$node};
2634                 }
2635             } elsif (!$scfg->{shared}) {
2636                 foreach my $node (keys %$nodehash) {
2637                     delete $nodehash->{$node} if $node ne $nodename
2638                 }
2639             }
2640         }
2641     });
2642
2643     return $nodehash
2644 }
2645
2646 sub check_local_storage_availability {
2647     my ($conf, $storecfg) = @_;
2648
2649     my $nodelist = PVE::Cluster::get_nodelist();
2650     my $nodehash = { map { $_ => {} } @$nodelist };
2651
2652     PVE::QemuConfig->foreach_volume($conf, sub {
2653         my ($ds, $drive) = @_;
2654
2655         my $volid = $drive->{file};
2656         return if !$volid;
2657
2658         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2659         if ($storeid) {
2660             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2661
2662             if ($scfg->{disable}) {
2663                 foreach my $node (keys %$nodehash) {
2664                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2665                 }
2666             } elsif (my $avail = $scfg->{nodes}) {
2667                 foreach my $node (keys %$nodehash) {
2668                     if (!$avail->{$node}) {
2669                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2670                     }
2671                 }
2672             }
2673         }
2674     });
2675
2676     foreach my $node (values %$nodehash) {
2677         if (my $unavail = $node->{unavailable_storages}) {
2678             $node->{unavailable_storages} = [ sort keys %$unavail ];
2679         }
2680     }
2681
2682     return $nodehash
2683 }
2684
2685 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2686 sub check_running {
2687     my ($vmid, $nocheck, $node) = @_;
2688
2689     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2690     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2691 }
2692
2693 sub vzlist {
2694
2695     my $vzlist = config_list();
2696
2697     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2698
2699     while (defined(my $de = $fd->read)) {
2700         next if $de !~ m/^(\d+)\.pid$/;
2701         my $vmid = $1;
2702         next if !defined($vzlist->{$vmid});
2703         if (my $pid = check_running($vmid)) {
2704             $vzlist->{$vmid}->{pid} = $pid;
2705         }
2706     }
2707
2708     return $vzlist;
2709 }
2710
2711 our $vmstatus_return_properties = {
2712     vmid => get_standard_option('pve-vmid'),
2713     status => {
2714         description => "Qemu process status.",
2715         type => 'string',
2716         enum => ['stopped', 'running'],
2717     },
2718     maxmem => {
2719         description => "Maximum memory in bytes.",
2720         type => 'integer',
2721         optional => 1,
2722         renderer => 'bytes',
2723     },
2724     maxdisk => {
2725         description => "Root disk size in bytes.",
2726         type => 'integer',
2727         optional => 1,
2728         renderer => 'bytes',
2729     },
2730     name => {
2731         description => "VM name.",
2732         type => 'string',
2733         optional => 1,
2734     },
2735     qmpstatus => {
2736         description => "Qemu QMP agent status.",
2737         type => 'string',
2738         optional => 1,
2739     },
2740     pid => {
2741         description => "PID of running qemu process.",
2742         type => 'integer',
2743         optional => 1,
2744     },
2745     uptime => {
2746         description => "Uptime.",
2747         type => 'integer',
2748         optional => 1,
2749         renderer => 'duration',
2750     },
2751     cpus => {
2752         description => "Maximum usable CPUs.",
2753         type => 'number',
2754         optional => 1,
2755     },
2756     lock => {
2757         description => "The current config lock, if any.",
2758         type => 'string',
2759         optional => 1,
2760     },
2761     tags => {
2762         description  => "The current configured tags, if any",
2763         type => 'string',
2764         optional => 1,
2765     },
2766     'running-machine' => {
2767         description => "The currently running machine type (if running).",
2768         type => 'string',
2769         optional => 1,
2770     },
2771     'running-qemu' => {
2772         description => "The currently running QEMU version (if running).",
2773         type => 'string',
2774         optional => 1,
2775     },
2776 };
2777
2778 my $last_proc_pid_stat;
2779
2780 # get VM status information
2781 # This must be fast and should not block ($full == false)
2782 # We only query KVM using QMP if $full == true (this can be slow)
2783 sub vmstatus {
2784     my ($opt_vmid, $full) = @_;
2785
2786     my $res = {};
2787
2788     my $storecfg = PVE::Storage::config();
2789
2790     my $list = vzlist();
2791     my $defaults = load_defaults();
2792
2793     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2794
2795     my $cpucount = $cpuinfo->{cpus} || 1;
2796
2797     foreach my $vmid (keys %$list) {
2798         next if $opt_vmid && ($vmid ne $opt_vmid);
2799
2800         my $conf = PVE::QemuConfig->load_config($vmid);
2801
2802         my $d = { vmid => int($vmid) };
2803         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2804
2805         # fixme: better status?
2806         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2807
2808         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2809         if (defined($size)) {
2810             $d->{disk} = 0; # no info available
2811             $d->{maxdisk} = $size;
2812         } else {
2813             $d->{disk} = 0;
2814             $d->{maxdisk} = 0;
2815         }
2816
2817         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2818             * ($conf->{cores} || $defaults->{cores});
2819         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2820         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2821
2822         $d->{name} = $conf->{name} || "VM $vmid";
2823         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2824             : $defaults->{memory}*(1024*1024);
2825
2826         if ($conf->{balloon}) {
2827             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2828             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2829                 : $defaults->{shares};
2830         }
2831
2832         $d->{uptime} = 0;
2833         $d->{cpu} = 0;
2834         $d->{mem} = 0;
2835
2836         $d->{netout} = 0;
2837         $d->{netin} = 0;
2838
2839         $d->{diskread} = 0;
2840         $d->{diskwrite} = 0;
2841
2842         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2843
2844         $d->{serial} = 1 if conf_has_serial($conf);
2845         $d->{lock} = $conf->{lock} if $conf->{lock};
2846         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2847
2848         $res->{$vmid} = $d;
2849     }
2850
2851     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2852     foreach my $dev (keys %$netdev) {
2853         next if $dev !~ m/^tap([1-9]\d*)i/;
2854         my $vmid = $1;
2855         my $d = $res->{$vmid};
2856         next if !$d;
2857
2858         $d->{netout} += $netdev->{$dev}->{receive};
2859         $d->{netin} += $netdev->{$dev}->{transmit};
2860
2861         if ($full) {
2862             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2863             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2864         }
2865
2866     }
2867
2868     my $ctime = gettimeofday;
2869
2870     foreach my $vmid (keys %$list) {
2871
2872         my $d = $res->{$vmid};
2873         my $pid = $d->{pid};
2874         next if !$pid;
2875
2876         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2877         next if !$pstat; # not running
2878
2879         my $used = $pstat->{utime} + $pstat->{stime};
2880
2881         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2882
2883         if ($pstat->{vsize}) {
2884             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2885         }
2886
2887         my $old = $last_proc_pid_stat->{$pid};
2888         if (!$old) {
2889             $last_proc_pid_stat->{$pid} = {
2890                 time => $ctime,
2891                 used => $used,
2892                 cpu => 0,
2893             };
2894             next;
2895         }
2896
2897         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2898
2899         if ($dtime > 1000) {
2900             my $dutime = $used -  $old->{used};
2901
2902             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2903             $last_proc_pid_stat->{$pid} = {
2904                 time => $ctime,
2905                 used => $used,
2906                 cpu => $d->{cpu},
2907             };
2908         } else {
2909             $d->{cpu} = $old->{cpu};
2910         }
2911     }
2912
2913     return $res if !$full;
2914
2915     my $qmpclient = PVE::QMPClient->new();
2916
2917     my $ballooncb = sub {
2918         my ($vmid, $resp) = @_;
2919
2920         my $info = $resp->{'return'};
2921         return if !$info->{max_mem};
2922
2923         my $d = $res->{$vmid};
2924
2925         # use memory assigned to VM
2926         $d->{maxmem} = $info->{max_mem};
2927         $d->{balloon} = $info->{actual};
2928
2929         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2930             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2931             $d->{freemem} = $info->{free_mem};
2932         }
2933
2934         $d->{ballooninfo} = $info;
2935     };
2936
2937     my $blockstatscb = sub {
2938         my ($vmid, $resp) = @_;
2939         my $data = $resp->{'return'} || [];
2940         my $totalrdbytes = 0;
2941         my $totalwrbytes = 0;
2942
2943         for my $blockstat (@$data) {
2944             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2945             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2946
2947             $blockstat->{device} =~ s/drive-//;
2948             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2949         }
2950         $res->{$vmid}->{diskread} = $totalrdbytes;
2951         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2952     };
2953
2954     my $machinecb = sub {
2955         my ($vmid, $resp) = @_;
2956         my $data = $resp->{'return'} || [];
2957
2958         $res->{$vmid}->{'running-machine'} =
2959             PVE::QemuServer::Machine::current_from_query_machines($data);
2960     };
2961
2962     my $versioncb = sub {
2963         my ($vmid, $resp) = @_;
2964         my $data = $resp->{'return'} // {};
2965         my $version = 'unknown';
2966
2967         if (my $v = $data->{qemu}) {
2968             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2969         }
2970
2971         $res->{$vmid}->{'running-qemu'} = $version;
2972     };
2973
2974     my $statuscb = sub {
2975         my ($vmid, $resp) = @_;
2976
2977         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2978         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2979         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2980         # this fails if ballon driver is not loaded, so this must be
2981         # the last commnand (following command are aborted if this fails).
2982         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2983
2984         my $status = 'unknown';
2985         if (!defined($status = $resp->{'return'}->{status})) {
2986             warn "unable to get VM status\n";
2987             return;
2988         }
2989
2990         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2991     };
2992
2993     foreach my $vmid (keys %$list) {
2994         next if $opt_vmid && ($vmid ne $opt_vmid);
2995         next if !$res->{$vmid}->{pid}; # not running
2996         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2997     }
2998
2999     $qmpclient->queue_execute(undef, 2);
3000
3001     foreach my $vmid (keys %$list) {
3002         next if $opt_vmid && ($vmid ne $opt_vmid);
3003         next if !$res->{$vmid}->{pid}; #not running
3004
3005         # we can't use the $qmpclient since it might have already aborted on
3006         # 'query-balloon', but this might also fail for older versions...
3007         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3008         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3009     }
3010
3011     foreach my $vmid (keys %$list) {
3012         next if $opt_vmid && ($vmid ne $opt_vmid);
3013         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3014     }
3015
3016     return $res;
3017 }
3018
3019 sub conf_has_serial {
3020     my ($conf) = @_;
3021
3022     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3023         if ($conf->{"serial$i"}) {
3024             return 1;
3025         }
3026     }
3027
3028     return 0;
3029 }
3030
3031 sub conf_has_audio {
3032     my ($conf, $id) = @_;
3033
3034     $id //= 0;
3035     my $audio = $conf->{"audio$id"};
3036     return if !defined($audio);
3037
3038     my $audioproperties = parse_property_string($audio_fmt, $audio);
3039     my $audiodriver = $audioproperties->{driver} // 'spice';
3040
3041     return {
3042         dev => $audioproperties->{device},
3043         dev_id => "audiodev$id",
3044         backend => $audiodriver,
3045         backend_id => "$audiodriver-backend${id}",
3046     };
3047 }
3048
3049 sub audio_devs {
3050     my ($audio, $audiopciaddr, $machine_version) = @_;
3051
3052     my $devs = [];
3053
3054     my $id = $audio->{dev_id};
3055     my $audiodev = "";
3056     if (min_version($machine_version, 4, 2)) {
3057         $audiodev = ",audiodev=$audio->{backend_id}";
3058     }
3059
3060     if ($audio->{dev} eq 'AC97') {
3061         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3062     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3063         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3064         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3065         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3066     } else {
3067         die "unkown audio device '$audio->{dev}', implement me!";
3068     }
3069
3070     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3071
3072     return $devs;
3073 }
3074
3075 sub get_tpm_paths {
3076     my ($vmid) = @_;
3077     return {
3078         socket => "/var/run/qemu-server/$vmid.swtpm",
3079         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3080     };
3081 }
3082
3083 sub add_tpm_device {
3084     my ($vmid, $devices, $conf) = @_;
3085
3086     return if !$conf->{tpmstate0};
3087
3088     my $paths = get_tpm_paths($vmid);
3089
3090     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3091     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3092     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3093 }
3094
3095 sub start_swtpm {
3096     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3097
3098     return if !$tpmdrive;
3099
3100     my $state;
3101     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3102     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3103     if ($storeid) {
3104         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3105     } else {
3106         $state = $tpm->{file};
3107     }
3108
3109     my $paths = get_tpm_paths($vmid);
3110
3111     # during migration, we will get state from remote
3112     #
3113     if (!$migration) {
3114         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3115         my $setup_cmd = [
3116             "swtpm_setup",
3117             "--tpmstate",
3118             "file://$state",
3119             "--createek",
3120             "--create-ek-cert",
3121             "--create-platform-cert",
3122             "--lock-nvram",
3123             "--config",
3124             "/etc/swtpm_setup.conf", # do not use XDG configs
3125             "--runas",
3126             "0", # force creation as root, error if not possible
3127             "--not-overwrite", # ignore existing state, do not modify
3128         ];
3129
3130         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3131         # TPM 2.0 supports ECC crypto, use if possible
3132         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3133
3134         run_command($setup_cmd, outfunc => sub {
3135             print "swtpm_setup: $1\n";
3136         });
3137     }
3138
3139     my $emulator_cmd = [
3140         "swtpm",
3141         "socket",
3142         "--tpmstate",
3143         "backend-uri=file://$state,mode=0600",
3144         "--ctrl",
3145         "type=unixio,path=$paths->{socket},mode=0600",
3146         "--pid",
3147         "file=$paths->{pid}",
3148         "--terminate", # terminate on QEMU disconnect
3149         "--daemon",
3150     ];
3151     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3152     run_command($emulator_cmd, outfunc => sub { print $1; });
3153
3154     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3155     while (! -e $paths->{pid}) {
3156         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3157         usleep(50_000);
3158     }
3159
3160     # return untainted PID of swtpm daemon so it can be killed on error
3161     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3162     return $1;
3163 }
3164
3165 sub vga_conf_has_spice {
3166     my ($vga) = @_;
3167
3168     my $vgaconf = parse_vga($vga);
3169     my $vgatype = $vgaconf->{type};
3170     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3171
3172     return $1 || 1;
3173 }
3174
3175 sub is_native($) {
3176     my ($arch) = @_;
3177     return get_host_arch() eq $arch;
3178 }
3179
3180 sub get_vm_arch {
3181     my ($conf) = @_;
3182     return $conf->{arch} // get_host_arch();
3183 }
3184
3185 my $default_machines = {
3186     x86_64 => 'pc',
3187     aarch64 => 'virt',
3188 };
3189
3190 sub get_installed_machine_version {
3191     my ($kvmversion) = @_;
3192     $kvmversion = kvm_user_version() if !defined($kvmversion);
3193     $kvmversion =~ m/^(\d+\.\d+)/;
3194     return $1;
3195 }
3196
3197 sub windows_get_pinned_machine_version {
3198     my ($machine, $base_version, $kvmversion) = @_;
3199
3200     my $pin_version = $base_version;
3201     if (!defined($base_version) ||
3202         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3203     ) {
3204         $pin_version = get_installed_machine_version($kvmversion);
3205     }
3206     if (!$machine || $machine eq 'pc') {
3207         $machine = "pc-i440fx-$pin_version";
3208     } elsif ($machine eq 'q35') {
3209         $machine = "pc-q35-$pin_version";
3210     } elsif ($machine eq 'virt') {
3211         $machine = "virt-$pin_version";
3212     } else {
3213         warn "unknown machine type '$machine', not touching that!\n";
3214     }
3215
3216     return $machine;
3217 }
3218
3219 sub get_vm_machine {
3220     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3221
3222     my $machine = $forcemachine || $conf->{machine};
3223
3224     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3225         $kvmversion //= kvm_user_version();
3226         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3227         # layout which confuses windows quite a bit and may result in various regressions..
3228         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3229         if (windows_version($conf->{ostype})) {
3230             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3231         }
3232         $arch //= 'x86_64';
3233         $machine ||= $default_machines->{$arch};
3234         if ($add_pve_version) {
3235             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3236             $machine .= "+pve$pvever";
3237         }
3238     }
3239
3240     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3241         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3242         $machine = $1 if $is_pxe;
3243
3244         # for version-pinned machines that do not include a pve-version (e.g.
3245         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3246         $machine .= '+pve0';
3247
3248         $machine .= '.pxe' if $is_pxe;
3249     }
3250
3251     return $machine;
3252 }
3253
3254 sub get_ovmf_files($$$) {
3255     my ($arch, $efidisk, $smm) = @_;
3256
3257     my $types = $OVMF->{$arch}
3258         or die "no OVMF images known for architecture '$arch'\n";
3259
3260     my $type = 'default';
3261     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3262         $type = $smm ? "4m" : "4m-no-smm";
3263         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3264     }
3265
3266     return $types->{$type}->@*;
3267 }
3268
3269 my $Arch2Qemu = {
3270     aarch64 => '/usr/bin/qemu-system-aarch64',
3271     x86_64 => '/usr/bin/qemu-system-x86_64',
3272 };
3273 sub get_command_for_arch($) {
3274     my ($arch) = @_;
3275     return '/usr/bin/kvm' if is_native($arch);
3276
3277     my $cmd = $Arch2Qemu->{$arch}
3278         or die "don't know how to emulate architecture '$arch'\n";
3279     return $cmd;
3280 }
3281
3282 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3283 # to use in a QEMU command line (-cpu element), first array_intersect the result
3284 # of query_supported_ with query_understood_. This is necessary because:
3285 #
3286 # a) query_understood_ returns flags the host cannot use and
3287 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3288 #    flags, but CPU settings - with most of them being flags. Those settings
3289 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3290 #
3291 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3292 # expensive. If you need the value returned from this, you can get it much
3293 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3294 # $accel being 'kvm' or 'tcg'.
3295 #
3296 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3297 # changes, automatically populating pmxcfs.
3298 #
3299 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3300 # since kvm and tcg machines support different flags
3301 #
3302 sub query_supported_cpu_flags {
3303     my ($arch) = @_;
3304
3305     $arch //= get_host_arch();
3306     my $default_machine = $default_machines->{$arch};
3307
3308     my $flags = {};
3309
3310     # FIXME: Once this is merged, the code below should work for ARM as well:
3311     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3312     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3313         $arch eq "aarch64";
3314
3315     my $kvm_supported = defined(kvm_version());
3316     my $qemu_cmd = get_command_for_arch($arch);
3317     my $fakevmid = -1;
3318     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3319
3320     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3321     my $query_supported_run_qemu = sub {
3322         my ($kvm) = @_;
3323
3324         my $flags = {};
3325         my $cmd = [
3326             $qemu_cmd,
3327             '-machine', $default_machine,
3328             '-display', 'none',
3329             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3330             '-mon', 'chardev=qmp,mode=control',
3331             '-pidfile', $pidfile,
3332             '-S', '-daemonize'
3333         ];
3334
3335         if (!$kvm) {
3336             push @$cmd, '-accel', 'tcg';
3337         }
3338
3339         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3340         die "QEMU flag querying VM exited with code " . $rc if $rc;
3341
3342         eval {
3343             my $cmd_result = mon_cmd(
3344                 $fakevmid,
3345                 'query-cpu-model-expansion',
3346                 type => 'full',
3347                 model => { name => 'host' }
3348             );
3349
3350             my $props = $cmd_result->{model}->{props};
3351             foreach my $prop (keys %$props) {
3352                 next if $props->{$prop} ne '1';
3353                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3354                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3355                 # We only keep those with underscores, to match /proc/cpuinfo
3356                 $prop =~ s/\.|-/_/g;
3357                 $flags->{$prop} = 1;
3358             }
3359         };
3360         my $err = $@;
3361
3362         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3363         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3364
3365         die $err if $err;
3366
3367         return [ sort keys %$flags ];
3368     };
3369
3370     # We need to query QEMU twice, since KVM and TCG have different supported flags
3371     PVE::QemuConfig->lock_config($fakevmid, sub {
3372         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3373         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3374
3375         if ($kvm_supported) {
3376             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3377             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3378         }
3379     });
3380
3381     return $flags;
3382 }
3383
3384 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3385 my $understood_cpu_flag_dir = "/usr/share/kvm";
3386 sub query_understood_cpu_flags {
3387     my $arch = get_host_arch();
3388     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3389
3390     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3391         if ! -e $filepath;
3392
3393     my $raw = file_get_contents($filepath);
3394     $raw =~ s/^\s+|\s+$//g;
3395     my @flags = split(/\s+/, $raw);
3396
3397     return \@flags;
3398 }
3399
3400 my sub get_cpuunits {
3401     my ($conf) = @_;
3402     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3403
3404     my $cpuunits = $conf->{cpuunits};
3405     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3406
3407     if ($is_cgroupv2) {
3408         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3409     } else {
3410         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3411     }
3412     return $cpuunits;
3413 }
3414
3415 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3416 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3417 my sub should_disable_smm {
3418     my ($conf, $vga) = @_;
3419
3420     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3421         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3422 }
3423
3424 sub config_to_command {
3425     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3426         $pbs_backing) = @_;
3427
3428     my $cmd = [];
3429     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3430     my $devices = [];
3431     my $bridges = {};
3432     my $ostype = $conf->{ostype};
3433     my $winversion = windows_version($ostype);
3434     my $kvm = $conf->{kvm};
3435     my $nodename = nodename();
3436
3437     my $arch = get_vm_arch($conf);
3438     my $kvm_binary = get_command_for_arch($arch);
3439     my $kvmver = kvm_user_version($kvm_binary);
3440
3441     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3442         $kvmver //= "undefined";
3443         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3444     }
3445
3446     my $add_pve_version = min_version($kvmver, 4, 1);
3447
3448     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3449     my $machine_version = extract_version($machine_type, $kvmver);
3450     $kvm //= 1 if is_native($arch);
3451
3452     $machine_version =~ m/(\d+)\.(\d+)/;
3453     my ($machine_major, $machine_minor) = ($1, $2);
3454
3455     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3456         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3457     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3458         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3459             ." please upgrade node '$nodename'\n"
3460     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3461         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3462         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3463             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3464             ." node '$nodename'\n";
3465     }
3466
3467     # if a specific +pve version is required for a feature, use $version_guard
3468     # instead of min_version to allow machines to be run with the minimum
3469     # required version
3470     my $required_pve_version = 0;
3471     my $version_guard = sub {
3472         my ($major, $minor, $pve) = @_;
3473         return 0 if !min_version($machine_version, $major, $minor, $pve);
3474         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3475         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3476         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3477         return 1;
3478     };
3479
3480     if ($kvm && !defined kvm_version()) {
3481         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3482             ." or enable in BIOS.\n";
3483     }
3484
3485     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3486     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3487     my $use_old_bios_files = undef;
3488     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3489
3490     push @$cmd, $kvm_binary;
3491
3492     push @$cmd, '-id', $vmid;
3493
3494     my $vmname = $conf->{name} || "vm$vmid";
3495
3496     push @$cmd, '-name', $vmname;
3497
3498     push @$cmd, '-no-shutdown';
3499
3500     my $use_virtio = 0;
3501
3502     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3503     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3504     push @$cmd, '-mon', "chardev=qmp,mode=control";
3505
3506     if (min_version($machine_version, 2, 12)) {
3507         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3508         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3509     }
3510
3511     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3512
3513     push @$cmd, '-daemonize';
3514
3515     if ($conf->{smbios1}) {
3516         my $smbios_conf = parse_smbios1($conf->{smbios1});
3517         if ($smbios_conf->{base64}) {
3518             # Do not pass base64 flag to qemu
3519             delete $smbios_conf->{base64};
3520             my $smbios_string = "";
3521             foreach my $key (keys %$smbios_conf) {
3522                 my $value;
3523                 if ($key eq "uuid") {
3524                     $value = $smbios_conf->{uuid}
3525                 } else {
3526                     $value = decode_base64($smbios_conf->{$key});
3527                 }
3528                 # qemu accepts any binary data, only commas need escaping by double comma
3529                 $value =~ s/,/,,/g;
3530                 $smbios_string .= "," . $key . "=" . $value if $value;
3531             }
3532             push @$cmd, '-smbios', "type=1" . $smbios_string;
3533         } else {
3534             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3535         }
3536     }
3537
3538     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3539         my $d;
3540         if (my $efidisk = $conf->{efidisk0}) {
3541             $d = parse_drive('efidisk0', $efidisk);
3542         }
3543
3544         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3545         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3546
3547         my ($path, $format);
3548         my $read_only_str = '';
3549         if ($d) {
3550             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3551             $format = $d->{format};
3552             if ($storeid) {
3553                 $path = PVE::Storage::path($storecfg, $d->{file});
3554                 if (!defined($format)) {
3555                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3556                     $format = qemu_img_format($scfg, $volname);
3557                 }
3558             } else {
3559                 $path = $d->{file};
3560                 die "efidisk format must be specified\n"
3561                     if !defined($format);
3562             }
3563
3564             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3565         } else {
3566             warn "no efidisk configured! Using temporary efivars disk.\n";
3567             $path = "/tmp/$vmid-ovmf.fd";
3568             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3569             $format = 'raw';
3570         }
3571
3572         my $size_str = "";
3573
3574         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3575             $size_str = ",size=" . (-s $ovmf_vars);
3576         }
3577
3578         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3579         my $cache = "";
3580         if ($path =~ m/^rbd:/) {
3581                 $cache = ',cache=writeback';
3582                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3583         }
3584
3585         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3586         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3587     }
3588
3589     if ($q35) { # tell QEMU to load q35 config early
3590         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3591         if (min_version($machine_version, 4, 0)) {
3592             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3593         } else {
3594             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3595         }
3596     }
3597
3598     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3599         push @$cmd, $fixups->@*;
3600     }
3601
3602     if ($conf->{vmgenid}) {
3603         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3604     }
3605
3606     # add usb controllers
3607     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3608         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3609     push @$devices, @usbcontrollers if @usbcontrollers;
3610     my $vga = parse_vga($conf->{vga});
3611
3612     my $qxlnum = vga_conf_has_spice($conf->{vga});
3613     $vga->{type} = 'qxl' if $qxlnum;
3614
3615     if (!$vga->{type}) {
3616         if ($arch eq 'aarch64') {
3617             $vga->{type} = 'virtio';
3618         } elsif (min_version($machine_version, 2, 9)) {
3619             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3620         } else {
3621             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3622         }
3623     }
3624
3625     # enable absolute mouse coordinates (needed by vnc)
3626     my $tablet = $conf->{tablet};
3627     if (!defined($tablet)) {
3628         $tablet = $defaults->{tablet};
3629         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3630         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3631     }
3632
3633     if ($tablet) {
3634         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3635         my $kbd = print_keyboarddevice_full($conf, $arch);
3636         push @$devices, '-device', $kbd if defined($kbd);
3637     }
3638
3639     my $bootorder = device_bootorder($conf);
3640
3641     # host pci device passthrough
3642     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3643         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3644
3645     # usb devices
3646     my $usb_dev_features = {};
3647     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3648
3649     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3650         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3651     push @$devices, @usbdevices if @usbdevices;
3652
3653     # serial devices
3654     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3655         my $path = $conf->{"serial$i"} or next;
3656         if ($path eq 'socket') {
3657             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3658             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3659             # On aarch64, serial0 is the UART device. Qemu only allows
3660             # connecting UART devices via the '-serial' command line, as
3661             # the device has a fixed slot on the hardware...
3662             if ($arch eq 'aarch64' && $i == 0) {
3663                 push @$devices, '-serial', "chardev:serial$i";
3664             } else {
3665                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3666             }
3667         } else {
3668             die "no such serial device\n" if ! -c $path;
3669             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3670             push @$devices, '-device', "isa-serial,chardev=serial$i";
3671         }
3672     }
3673
3674     # parallel devices
3675     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3676         if (my $path = $conf->{"parallel$i"}) {
3677             die "no such parallel device\n" if ! -c $path;
3678             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3679             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3680             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3681         }
3682     }
3683
3684     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3685         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3686         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3687         push @$devices, @$audio_devs;
3688     }
3689
3690     add_tpm_device($vmid, $devices, $conf);
3691
3692     my $sockets = 1;
3693     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3694     $sockets = $conf->{sockets} if  $conf->{sockets};
3695
3696     my $cores = $conf->{cores} || 1;
3697
3698     my $maxcpus = $sockets * $cores;
3699
3700     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3701
3702     my $allowed_vcpus = $cpuinfo->{cpus};
3703
3704     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3705
3706     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3707         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3708         for (my $i = 2; $i <= $vcpus; $i++)  {
3709             my $cpustr = print_cpu_device($conf,$i);
3710             push @$cmd, '-device', $cpustr;
3711         }
3712
3713     } else {
3714
3715         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3716     }
3717     push @$cmd, '-nodefaults';
3718
3719     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3720
3721     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3722
3723     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3724
3725     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3726         push @$devices, '-device', print_vga_device(
3727             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3728         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3729         push @$cmd,  '-vnc', "unix:$socket,password=on";
3730     } else {
3731         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3732         push @$cmd, '-nographic';
3733     }
3734
3735     # time drift fix
3736     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3737     my $useLocaltime = $conf->{localtime};
3738
3739     if ($winversion >= 5) { # windows
3740         $useLocaltime = 1 if !defined($conf->{localtime});
3741
3742         # use time drift fix when acpi is enabled
3743         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3744             $tdf = 1 if !defined($conf->{tdf});
3745         }
3746     }
3747
3748     if ($winversion >= 6) {
3749         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3750         push @$cmd, '-no-hpet';
3751     }
3752
3753     push @$rtcFlags, 'driftfix=slew' if $tdf;
3754
3755     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3756         push @$rtcFlags, "base=$conf->{startdate}";
3757     } elsif ($useLocaltime) {
3758         push @$rtcFlags, 'base=localtime';
3759     }
3760
3761     if ($forcecpu) {
3762         push @$cmd, '-cpu', $forcecpu;
3763     } else {
3764         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3765     }
3766
3767     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3768
3769     push @$cmd, '-S' if $conf->{freeze};
3770
3771     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3772
3773     my $guest_agent = parse_guest_agent($conf);
3774
3775     if ($guest_agent->{enabled}) {
3776         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3777         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3778
3779         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3780             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3781             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3782             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3783         } elsif ($guest_agent->{type} eq 'isa') {
3784             push @$devices, '-device', "isa-serial,chardev=qga0";
3785         }
3786     }
3787
3788     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3789     if ($rng && $version_guard->(4, 1, 2)) {
3790         check_rng_source($rng->{source});
3791
3792         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3793         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3794         my $limiter_str = "";
3795         if ($max_bytes) {
3796             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3797         }
3798
3799         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3800         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3801         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3802     }
3803
3804     my $spice_port;
3805
3806     if ($qxlnum) {
3807         if ($qxlnum > 1) {
3808             if ($winversion){
3809                 for (my $i = 1; $i < $qxlnum; $i++){
3810                     push @$devices, '-device', print_vga_device(
3811                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3812                 }
3813             } else {
3814                 # assume other OS works like Linux
3815                 my ($ram, $vram) = ("134217728", "67108864");
3816                 if ($vga->{memory}) {
3817                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3818                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3819                 }
3820                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3821                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3822             }
3823         }
3824
3825         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3826
3827         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3828         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3829         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3830
3831         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3832         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3833         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3834
3835         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3836         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3837
3838         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3839         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3840         if ($spice_enhancement->{foldersharing}) {
3841             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3842             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3843         }
3844
3845         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3846         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3847             if $spice_enhancement->{videostreaming};
3848
3849         push @$devices, '-spice', "$spice_opts";
3850     }
3851
3852     # enable balloon by default, unless explicitly disabled
3853     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3854         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3855         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3856     }
3857
3858     if ($conf->{watchdog}) {
3859         my $wdopts = parse_watchdog($conf->{watchdog});
3860         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3861         my $watchdog = $wdopts->{model} || 'i6300esb';
3862         push @$devices, '-device', "$watchdog$pciaddr";
3863         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3864     }
3865
3866     my $vollist = [];
3867     my $scsicontroller = {};
3868     my $ahcicontroller = {};
3869     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3870
3871     # Add iscsi initiator name if available
3872     if (my $initiator = get_initiator_name()) {
3873         push @$devices, '-iscsi', "initiator-name=$initiator";
3874     }
3875
3876     PVE::QemuConfig->foreach_volume($conf, sub {
3877         my ($ds, $drive) = @_;
3878
3879         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3880             check_volume_storage_type($storecfg, $drive->{file});
3881             push @$vollist, $drive->{file};
3882         }
3883
3884         # ignore efidisk here, already added in bios/fw handling code above
3885         return if $drive->{interface} eq 'efidisk';
3886         # similar for TPM
3887         return if $drive->{interface} eq 'tpmstate';
3888
3889         $use_virtio = 1 if $ds =~ m/^virtio/;
3890
3891         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3892
3893         if ($drive->{interface} eq 'virtio'){
3894            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3895         }
3896
3897         if ($drive->{interface} eq 'scsi') {
3898
3899             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3900
3901             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3902                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3903
3904             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3905             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3906
3907             my $iothread = '';
3908             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3909                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3910                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3911             } elsif ($drive->{iothread}) {
3912                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3913             }
3914
3915             my $queues = '';
3916             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3917                 $queues = ",num_queues=$drive->{queues}";
3918             }
3919
3920             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3921                 if !$scsicontroller->{$controller};
3922             $scsicontroller->{$controller}=1;
3923         }
3924
3925         if ($drive->{interface} eq 'sata') {
3926             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3927             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3928             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3929                 if !$ahcicontroller->{$controller};
3930             $ahcicontroller->{$controller}=1;
3931         }
3932
3933         my $pbs_conf = $pbs_backing->{$ds};
3934         my $pbs_name = undef;
3935         if ($pbs_conf) {
3936             $pbs_name = "drive-$ds-pbs";
3937             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3938         }
3939
3940         my $drive_cmd = print_drive_commandline_full(
3941             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3942
3943         # extra protection for templates, but SATA and IDE don't support it..
3944         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3945
3946         push @$devices, '-drive',$drive_cmd;
3947         push @$devices, '-device', print_drivedevice_full(
3948             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3949     });
3950
3951     for (my $i = 0; $i < $MAX_NETS; $i++) {
3952         my $netname = "net$i";
3953
3954         next if !$conf->{$netname};
3955         my $d = parse_net($conf->{$netname});
3956         next if !$d;
3957
3958         $use_virtio = 1 if $d->{model} eq 'virtio';
3959
3960         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3961
3962         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3963         push @$devices, '-netdev', $netdevfull;
3964
3965         my $netdevicefull = print_netdevice_full(
3966             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3967
3968         push @$devices, '-device', $netdevicefull;
3969     }
3970
3971     if ($conf->{ivshmem}) {
3972         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3973
3974         my $bus;
3975         if ($q35) {
3976             $bus = print_pcie_addr("ivshmem");
3977         } else {
3978             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3979         }
3980
3981         my $ivshmem_name = $ivshmem->{name} // $vmid;
3982         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3983
3984         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3985         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3986             .",size=$ivshmem->{size}M";
3987     }
3988
3989     # pci.4 is nested in pci.1
3990     $bridges->{1} = 1 if $bridges->{4};
3991
3992     if (!$q35) { # add pci bridges
3993         if (min_version($machine_version, 2, 3)) {
3994            $bridges->{1} = 1;
3995            $bridges->{2} = 1;
3996         }
3997         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3998     }
3999
4000     for my $k (sort {$b cmp $a} keys %$bridges) {
4001         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4002
4003         my $k_name = $k;
4004         if ($k == 2 && $legacy_igd) {
4005             $k_name = "$k-igd";
4006         }
4007         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4008         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4009
4010         if ($q35) { # add after -readconfig pve-q35.cfg
4011             splice @$devices, 2, 0, '-device', $devstr;
4012         } else {
4013             unshift @$devices, '-device', $devstr if $k > 0;
4014         }
4015     }
4016
4017     if (!$kvm) {
4018         push @$machineFlags, 'accel=tcg';
4019     }
4020
4021     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4022
4023     my $machine_type_min = $machine_type;
4024     if ($add_pve_version) {
4025         $machine_type_min =~ s/\+pve\d+$//;
4026         $machine_type_min .= "+pve$required_pve_version";
4027     }
4028     push @$machineFlags, "type=${machine_type_min}";
4029
4030     push @$cmd, @$devices;
4031     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4032     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4033     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4034
4035     if (my $vmstate = $conf->{vmstate}) {
4036         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4037         push @$vollist, $vmstate;
4038         push @$cmd, '-loadstate', $statepath;
4039         print "activating and using '$vmstate' as vmstate\n";
4040     }
4041
4042     if (PVE::QemuConfig->is_template($conf)) {
4043         # needed to workaround base volumes being read-only
4044         push @$cmd, '-snapshot';
4045     }
4046
4047     # add custom args
4048     if ($conf->{args}) {
4049         my $aa = PVE::Tools::split_args($conf->{args});
4050         push @$cmd, @$aa;
4051     }
4052
4053     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4054 }
4055
4056 sub check_rng_source {
4057     my ($source) = @_;
4058
4059     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4060     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4061         if ! -e $source;
4062
4063     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4064     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4065         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4066         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4067         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4068             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4069             ." to the host.\n";
4070     }
4071 }
4072
4073 sub spice_port {
4074     my ($vmid) = @_;
4075
4076     my $res = mon_cmd($vmid, 'query-spice');
4077
4078     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4079 }
4080
4081 sub vm_devices_list {
4082     my ($vmid) = @_;
4083
4084     my $res = mon_cmd($vmid, 'query-pci');
4085     my $devices_to_check = [];
4086     my $devices = {};
4087     foreach my $pcibus (@$res) {
4088         push @$devices_to_check, @{$pcibus->{devices}},
4089     }
4090
4091     while (@$devices_to_check) {
4092         my $to_check = [];
4093         for my $d (@$devices_to_check) {
4094             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4095             next if !$d->{'pci_bridge'};
4096
4097             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4098             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4099         }
4100         $devices_to_check = $to_check;
4101     }
4102
4103     my $resblock = mon_cmd($vmid, 'query-block');
4104     foreach my $block (@$resblock) {
4105         if($block->{device} =~ m/^drive-(\S+)/){
4106                 $devices->{$1} = 1;
4107         }
4108     }
4109
4110     my $resmice = mon_cmd($vmid, 'query-mice');
4111     foreach my $mice (@$resmice) {
4112         if ($mice->{name} eq 'QEMU HID Tablet') {
4113             $devices->{tablet} = 1;
4114             last;
4115         }
4116     }
4117
4118     # for usb devices there is no query-usb
4119     # but we can iterate over the entries in
4120     # qom-list path=/machine/peripheral
4121     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4122     foreach my $per (@$resperipheral) {
4123         if ($per->{name} =~ m/^usb\d+$/) {
4124             $devices->{$per->{name}} = 1;
4125         }
4126     }
4127
4128     return $devices;
4129 }
4130
4131 sub vm_deviceplug {
4132     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4133
4134     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4135
4136     my $devices_list = vm_devices_list($vmid);
4137     return 1 if defined($devices_list->{$deviceid});
4138
4139     # add PCI bridge if we need it for the device
4140     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4141
4142     if ($deviceid eq 'tablet') {
4143         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4144     } elsif ($deviceid eq 'keyboard') {
4145         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4146     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4147         die "usb hotplug currently not reliable\n";
4148         # since we can't reliably hot unplug all added usb devices and usb
4149         # passthrough breaks live migration we disable usb hotplugging for now
4150         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4151     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4152         qemu_iothread_add($vmid, $deviceid, $device);
4153
4154         qemu_driveadd($storecfg, $vmid, $device);
4155         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4156
4157         qemu_deviceadd($vmid, $devicefull);
4158         eval { qemu_deviceaddverify($vmid, $deviceid); };
4159         if (my $err = $@) {
4160             eval { qemu_drivedel($vmid, $deviceid); };
4161             warn $@ if $@;
4162             die $err;
4163         }
4164     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4165         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4166         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4167         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4168
4169         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4170
4171         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4172             qemu_iothread_add($vmid, $deviceid, $device);
4173             $devicefull .= ",iothread=iothread-$deviceid";
4174         }
4175
4176         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4177             $devicefull .= ",num_queues=$device->{queues}";
4178         }
4179
4180         qemu_deviceadd($vmid, $devicefull);
4181         qemu_deviceaddverify($vmid, $deviceid);
4182     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4183         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4184         qemu_driveadd($storecfg, $vmid, $device);
4185
4186         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4187         eval { qemu_deviceadd($vmid, $devicefull); };
4188         if (my $err = $@) {
4189             eval { qemu_drivedel($vmid, $deviceid); };
4190             warn $@ if $@;
4191             die $err;
4192         }
4193     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4194         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4195
4196         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4197         my $use_old_bios_files = undef;
4198         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4199
4200         my $netdevicefull = print_netdevice_full(
4201             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4202         qemu_deviceadd($vmid, $netdevicefull);
4203         eval {
4204             qemu_deviceaddverify($vmid, $deviceid);
4205             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4206         };
4207         if (my $err = $@) {
4208             eval { qemu_netdevdel($vmid, $deviceid); };
4209             warn $@ if $@;
4210             die $err;
4211         }
4212     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4213         my $bridgeid = $2;
4214         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4215         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4216
4217         qemu_deviceadd($vmid, $devicefull);
4218         qemu_deviceaddverify($vmid, $deviceid);
4219     } else {
4220         die "can't hotplug device '$deviceid'\n";
4221     }
4222
4223     return 1;
4224 }
4225
4226 # fixme: this should raise exceptions on error!
4227 sub vm_deviceunplug {
4228     my ($vmid, $conf, $deviceid) = @_;
4229
4230     my $devices_list = vm_devices_list($vmid);
4231     return 1 if !defined($devices_list->{$deviceid});
4232
4233     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4234     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4235
4236     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4237         qemu_devicedel($vmid, $deviceid);
4238     } elsif ($deviceid =~ m/^usb\d+$/) {
4239         die "usb hotplug currently not reliable\n";
4240         # when unplugging usb devices this way, there may be remaining usb
4241         # controllers/hubs so we disable it for now
4242         #qemu_devicedel($vmid, $deviceid);
4243         #qemu_devicedelverify($vmid, $deviceid);
4244     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4245         my $device = parse_drive($deviceid, $conf->{$deviceid});
4246
4247         qemu_devicedel($vmid, $deviceid);
4248         qemu_devicedelverify($vmid, $deviceid);
4249         qemu_drivedel($vmid, $deviceid);
4250         qemu_iothread_del($vmid, $deviceid, $device);
4251     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4252         qemu_devicedel($vmid, $deviceid);
4253         qemu_devicedelverify($vmid, $deviceid);
4254     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4255         my $device = parse_drive($deviceid, $conf->{$deviceid});
4256
4257         qemu_devicedel($vmid, $deviceid);
4258         qemu_devicedelverify($vmid, $deviceid);
4259         qemu_drivedel($vmid, $deviceid);
4260         qemu_deletescsihw($conf, $vmid, $deviceid);
4261
4262         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4263             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4264     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4265         qemu_devicedel($vmid, $deviceid);
4266         qemu_devicedelverify($vmid, $deviceid);
4267         qemu_netdevdel($vmid, $deviceid);
4268     } else {
4269         die "can't unplug device '$deviceid'\n";
4270     }
4271
4272     return 1;
4273 }
4274
4275 sub qemu_deviceadd {
4276     my ($vmid, $devicefull) = @_;
4277
4278     $devicefull = "driver=".$devicefull;
4279     my %options =  split(/[=,]/, $devicefull);
4280
4281     mon_cmd($vmid, "device_add" , %options);
4282 }
4283
4284 sub qemu_devicedel {
4285     my ($vmid, $deviceid) = @_;
4286
4287     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4288 }
4289
4290 sub qemu_iothread_add {
4291     my ($vmid, $deviceid, $device) = @_;
4292
4293     if ($device->{iothread}) {
4294         my $iothreads = vm_iothreads_list($vmid);
4295         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4296     }
4297 }
4298
4299 sub qemu_iothread_del {
4300     my ($vmid, $deviceid, $device) = @_;
4301
4302     if ($device->{iothread}) {
4303         my $iothreads = vm_iothreads_list($vmid);
4304         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4305     }
4306 }
4307
4308 sub qemu_objectadd {
4309     my ($vmid, $objectid, $qomtype) = @_;
4310
4311     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4312
4313     return 1;
4314 }
4315
4316 sub qemu_objectdel {
4317     my ($vmid, $objectid) = @_;
4318
4319     mon_cmd($vmid, "object-del", id => $objectid);
4320
4321     return 1;
4322 }
4323
4324 sub qemu_driveadd {
4325     my ($storecfg, $vmid, $device) = @_;
4326
4327     my $kvmver = get_running_qemu_version($vmid);
4328     my $io_uring = min_version($kvmver, 6, 0);
4329     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4330     $drive =~ s/\\/\\\\/g;
4331     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4332
4333     # If the command succeeds qemu prints: "OK"
4334     return 1 if $ret =~ m/OK/s;
4335
4336     die "adding drive failed: $ret\n";
4337 }
4338
4339 sub qemu_drivedel {
4340     my ($vmid, $deviceid) = @_;
4341
4342     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4343     $ret =~ s/^\s+//;
4344
4345     return 1 if $ret eq "";
4346
4347     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4348     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4349
4350     die "deleting drive $deviceid failed : $ret\n";
4351 }
4352
4353 sub qemu_deviceaddverify {
4354     my ($vmid, $deviceid) = @_;
4355
4356     for (my $i = 0; $i <= 5; $i++) {
4357          my $devices_list = vm_devices_list($vmid);
4358          return 1 if defined($devices_list->{$deviceid});
4359          sleep 1;
4360     }
4361
4362     die "error on hotplug device '$deviceid'\n";
4363 }
4364
4365
4366 sub qemu_devicedelverify {
4367     my ($vmid, $deviceid) = @_;
4368
4369     # need to verify that the device is correctly removed as device_del
4370     # is async and empty return is not reliable
4371
4372     for (my $i = 0; $i <= 5; $i++) {
4373          my $devices_list = vm_devices_list($vmid);
4374          return 1 if !defined($devices_list->{$deviceid});
4375          sleep 1;
4376     }
4377
4378     die "error on hot-unplugging device '$deviceid'\n";
4379 }
4380
4381 sub qemu_findorcreatescsihw {
4382     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4383
4384     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4385
4386     my $scsihwid="$controller_prefix$controller";
4387     my $devices_list = vm_devices_list($vmid);
4388
4389     if (!defined($devices_list->{$scsihwid})) {
4390         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4391     }
4392
4393     return 1;
4394 }
4395
4396 sub qemu_deletescsihw {
4397     my ($conf, $vmid, $opt) = @_;
4398
4399     my $device = parse_drive($opt, $conf->{$opt});
4400
4401     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4402         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4403         return 1;
4404     }
4405
4406     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4407
4408     my $devices_list = vm_devices_list($vmid);
4409     foreach my $opt (keys %{$devices_list}) {
4410         if (is_valid_drivename($opt)) {
4411             my $drive = parse_drive($opt, $conf->{$opt});
4412             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4413                 return 1;
4414             }
4415         }
4416     }
4417
4418     my $scsihwid="scsihw$controller";
4419
4420     vm_deviceunplug($vmid, $conf, $scsihwid);
4421
4422     return 1;
4423 }
4424
4425 sub qemu_add_pci_bridge {
4426     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4427
4428     my $bridges = {};
4429
4430     my $bridgeid;
4431
4432     print_pci_addr($device, $bridges, $arch, $machine_type);
4433
4434     while (my ($k, $v) = each %$bridges) {
4435         $bridgeid = $k;
4436     }
4437     return 1 if !defined($bridgeid) || $bridgeid < 1;
4438
4439     my $bridge = "pci.$bridgeid";
4440     my $devices_list = vm_devices_list($vmid);
4441
4442     if (!defined($devices_list->{$bridge})) {
4443         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4444     }
4445
4446     return 1;
4447 }
4448
4449 sub qemu_set_link_status {
4450     my ($vmid, $device, $up) = @_;
4451
4452     mon_cmd($vmid, "set_link", name => $device,
4453                up => $up ? JSON::true : JSON::false);
4454 }
4455
4456 sub qemu_netdevadd {
4457     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4458
4459     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4460     my %options =  split(/[=,]/, $netdev);
4461
4462     if (defined(my $vhost = $options{vhost})) {
4463         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4464     }
4465
4466     if (defined(my $queues = $options{queues})) {
4467         $options{queues} = $queues + 0;
4468     }
4469
4470     mon_cmd($vmid, "netdev_add",  %options);
4471     return 1;
4472 }
4473
4474 sub qemu_netdevdel {
4475     my ($vmid, $deviceid) = @_;
4476
4477     mon_cmd($vmid, "netdev_del", id => $deviceid);
4478 }
4479
4480 sub qemu_usb_hotplug {
4481     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4482
4483     return if !$device;
4484
4485     # remove the old one first
4486     vm_deviceunplug($vmid, $conf, $deviceid);
4487
4488     # check if xhci controller is necessary and available
4489     if ($device->{usb3}) {
4490
4491         my $devicelist = vm_devices_list($vmid);
4492
4493         if (!$devicelist->{xhci}) {
4494             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4495             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4496         }
4497     }
4498     my $d = parse_usb_device($device->{host});
4499     $d->{usb3} = $device->{usb3};
4500
4501     # add the new one
4502     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4503 }
4504
4505 sub qemu_cpu_hotplug {
4506     my ($vmid, $conf, $vcpus) = @_;
4507
4508     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4509
4510     my $sockets = 1;
4511     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4512     $sockets = $conf->{sockets} if  $conf->{sockets};
4513     my $cores = $conf->{cores} || 1;
4514     my $maxcpus = $sockets * $cores;
4515
4516     $vcpus = $maxcpus if !$vcpus;
4517
4518     die "you can't add more vcpus than maxcpus\n"
4519         if $vcpus > $maxcpus;
4520
4521     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4522
4523     if ($vcpus < $currentvcpus) {
4524
4525         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4526
4527             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4528                 qemu_devicedel($vmid, "cpu$i");
4529                 my $retry = 0;
4530                 my $currentrunningvcpus = undef;
4531                 while (1) {
4532                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4533                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4534                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4535                     $retry++;
4536                     sleep 1;
4537                 }
4538                 #update conf after each succesfull cpu unplug
4539                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4540                 PVE::QemuConfig->write_config($vmid, $conf);
4541             }
4542         } else {
4543             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4544         }
4545
4546         return;
4547     }
4548
4549     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4550     die "vcpus in running vm does not match its configuration\n"
4551         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4552
4553     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4554
4555         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4556             my $cpustr = print_cpu_device($conf, $i);
4557             qemu_deviceadd($vmid, $cpustr);
4558
4559             my $retry = 0;
4560             my $currentrunningvcpus = undef;
4561             while (1) {
4562                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4563                 last if scalar(@{$currentrunningvcpus}) == $i;
4564                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4565                 sleep 1;
4566                 $retry++;
4567             }
4568             #update conf after each succesfull cpu hotplug
4569             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4570             PVE::QemuConfig->write_config($vmid, $conf);
4571         }
4572     } else {
4573
4574         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4575             mon_cmd($vmid, "cpu-add", id => int($i));
4576         }
4577     }
4578 }
4579
4580 sub qemu_block_set_io_throttle {
4581     my ($vmid, $deviceid,
4582         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4583         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4584         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4585         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4586
4587     return if !check_running($vmid) ;
4588
4589     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4590         bps => int($bps),
4591         bps_rd => int($bps_rd),
4592         bps_wr => int($bps_wr),
4593         iops => int($iops),
4594         iops_rd => int($iops_rd),
4595         iops_wr => int($iops_wr),
4596         bps_max => int($bps_max),
4597         bps_rd_max => int($bps_rd_max),
4598         bps_wr_max => int($bps_wr_max),
4599         iops_max => int($iops_max),
4600         iops_rd_max => int($iops_rd_max),
4601         iops_wr_max => int($iops_wr_max),
4602         bps_max_length => int($bps_max_length),
4603         bps_rd_max_length => int($bps_rd_max_length),
4604         bps_wr_max_length => int($bps_wr_max_length),
4605         iops_max_length => int($iops_max_length),
4606         iops_rd_max_length => int($iops_rd_max_length),
4607         iops_wr_max_length => int($iops_wr_max_length),
4608     );
4609
4610 }
4611
4612 sub qemu_block_resize {
4613     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4614
4615     my $running = check_running($vmid);
4616
4617     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4618
4619     return if !$running;
4620
4621     my $padding = (1024 - $size % 1024) % 1024;
4622     $size = $size + $padding;
4623
4624     mon_cmd(
4625         $vmid,
4626         "block_resize",
4627         device => $deviceid,
4628         size => int($size),
4629         timeout => 60,
4630     );
4631 }
4632
4633 sub qemu_volume_snapshot {
4634     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4635
4636     my $running = check_running($vmid);
4637
4638     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4639         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4640     } else {
4641         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4642     }
4643 }
4644
4645 sub qemu_volume_snapshot_delete {
4646     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4647
4648     my $running = check_running($vmid);
4649
4650     if($running) {
4651
4652         $running = undef;
4653         my $conf = PVE::QemuConfig->load_config($vmid);
4654         PVE::QemuConfig->foreach_volume($conf, sub {
4655             my ($ds, $drive) = @_;
4656             $running = 1 if $drive->{file} eq $volid;
4657         });
4658     }
4659
4660     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4661         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4662     } else {
4663         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4664     }
4665 }
4666
4667 sub set_migration_caps {
4668     my ($vmid, $savevm) = @_;
4669
4670     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4671
4672     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4673     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4674
4675     my $cap_ref = [];
4676
4677     my $enabled_cap = {
4678         "auto-converge" => 1,
4679         "xbzrle" => 1,
4680         "x-rdma-pin-all" => 0,
4681         "zero-blocks" => 0,
4682         "compress" => 0,
4683         "dirty-bitmaps" => $dirty_bitmaps,
4684     };
4685
4686     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4687
4688     for my $supported_capability (@$supported_capabilities) {
4689         push @$cap_ref, {
4690             capability => $supported_capability->{capability},
4691             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4692         };
4693     }
4694
4695     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4696 }
4697
4698 sub foreach_volid {
4699     my ($conf, $func, @param) = @_;
4700
4701     my $volhash = {};
4702
4703     my $test_volid = sub {
4704         my ($key, $drive, $snapname) = @_;
4705
4706         my $volid = $drive->{file};
4707         return if !$volid;
4708
4709         $volhash->{$volid}->{cdrom} //= 1;
4710         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4711
4712         my $replicate = $drive->{replicate} // 1;
4713         $volhash->{$volid}->{replicate} //= 0;
4714         $volhash->{$volid}->{replicate} = 1 if $replicate;
4715
4716         $volhash->{$volid}->{shared} //= 0;
4717         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4718
4719         $volhash->{$volid}->{referenced_in_config} //= 0;
4720         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4721
4722         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4723             if defined($snapname);
4724
4725         my $size = $drive->{size};
4726         $volhash->{$volid}->{size} //= $size if $size;
4727
4728         $volhash->{$volid}->{is_vmstate} //= 0;
4729         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4730
4731         $volhash->{$volid}->{is_tpmstate} //= 0;
4732         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4733
4734         $volhash->{$volid}->{is_unused} //= 0;
4735         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4736
4737         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4738     };
4739
4740     my $include_opts = {
4741         extra_keys => ['vmstate'],
4742         include_unused => 1,
4743     };
4744
4745     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4746     foreach my $snapname (keys %{$conf->{snapshots}}) {
4747         my $snap = $conf->{snapshots}->{$snapname};
4748         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4749     }
4750
4751     foreach my $volid (keys %$volhash) {
4752         &$func($volid, $volhash->{$volid}, @param);
4753     }
4754 }
4755
4756 my $fast_plug_option = {
4757     'lock' => 1,
4758     'name' => 1,
4759     'onboot' => 1,
4760     'shares' => 1,
4761     'startup' => 1,
4762     'description' => 1,
4763     'protection' => 1,
4764     'vmstatestorage' => 1,
4765     'hookscript' => 1,
4766     'tags' => 1,
4767 };
4768
4769 # hotplug changes in [PENDING]
4770 # $selection hash can be used to only apply specified options, for
4771 # example: { cores => 1 } (only apply changed 'cores')
4772 # $errors ref is used to return error messages
4773 sub vmconfig_hotplug_pending {
4774     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4775
4776     my $defaults = load_defaults();
4777     my $arch = get_vm_arch($conf);
4778     my $machine_type = get_vm_machine($conf, undef, $arch);
4779
4780     # commit values which do not have any impact on running VM first
4781     # Note: those option cannot raise errors, we we do not care about
4782     # $selection and always apply them.
4783
4784     my $add_error = sub {
4785         my ($opt, $msg) = @_;
4786         $errors->{$opt} = "hotplug problem - $msg";
4787     };
4788
4789     my $changes = 0;
4790     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4791         if ($fast_plug_option->{$opt}) {
4792             $conf->{$opt} = $conf->{pending}->{$opt};
4793             delete $conf->{pending}->{$opt};
4794             $changes = 1;
4795         }
4796     }
4797
4798     if ($changes) {
4799         PVE::QemuConfig->write_config($vmid, $conf);
4800     }
4801
4802     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4803
4804     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4805     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4806     foreach my $opt (sort keys %$pending_delete_hash) {
4807         next if $selection && !$selection->{$opt};
4808         my $force = $pending_delete_hash->{$opt}->{force};
4809         eval {
4810             if ($opt eq 'hotplug') {
4811                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4812             } elsif ($opt eq 'tablet') {
4813                 die "skip\n" if !$hotplug_features->{usb};
4814                 if ($defaults->{tablet}) {
4815                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4816                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4817                         if $arch eq 'aarch64';
4818                 } else {
4819                     vm_deviceunplug($vmid, $conf, 'tablet');
4820                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4821                 }
4822             } elsif ($opt =~ m/^usb\d+/) {
4823                 die "skip\n";
4824                 # since we cannot reliably hot unplug usb devices we are disabling it
4825                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4826                 #vm_deviceunplug($vmid, $conf, $opt);
4827             } elsif ($opt eq 'vcpus') {
4828                 die "skip\n" if !$hotplug_features->{cpu};
4829                 qemu_cpu_hotplug($vmid, $conf, undef);
4830             } elsif ($opt eq 'balloon') {
4831                 # enable balloon device is not hotpluggable
4832                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4833                 # here we reset the ballooning value to memory
4834                 my $balloon = $conf->{memory} || $defaults->{memory};
4835                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4836             } elsif ($fast_plug_option->{$opt}) {
4837                 # do nothing
4838             } elsif ($opt =~ m/^net(\d+)$/) {
4839                 die "skip\n" if !$hotplug_features->{network};
4840                 vm_deviceunplug($vmid, $conf, $opt);
4841             } elsif (is_valid_drivename($opt)) {
4842                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4843                 vm_deviceunplug($vmid, $conf, $opt);
4844                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4845             } elsif ($opt =~ m/^memory$/) {
4846                 die "skip\n" if !$hotplug_features->{memory};
4847                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4848             } elsif ($opt eq 'cpuunits') {
4849                 $cgroup->change_cpu_shares(undef, 1024);
4850             } elsif ($opt eq 'cpulimit') {
4851                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4852             } else {
4853                 die "skip\n";
4854             }
4855         };
4856         if (my $err = $@) {
4857             &$add_error($opt, $err) if $err ne "skip\n";
4858         } else {
4859             delete $conf->{$opt};
4860             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4861         }
4862     }
4863
4864     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4865     $apply_pending_cloudinit = sub {
4866         return if $apply_pending_cloudinit_done; # once is enough
4867         $apply_pending_cloudinit_done = 1; # once is enough
4868
4869         my ($key, $value) = @_;
4870
4871         my @cloudinit_opts = keys %$confdesc_cloudinit;
4872         foreach my $opt (keys %{$conf->{pending}}) {
4873             next if !grep { $_ eq $opt } @cloudinit_opts;
4874             $conf->{$opt} = delete $conf->{pending}->{$opt};
4875         }
4876
4877         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4878         foreach my $opt (sort keys %$pending_delete_hash) {
4879             next if !grep { $_ eq $opt } @cloudinit_opts;
4880             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4881             delete $conf->{$opt};
4882         }
4883
4884         my $new_conf = { %$conf };
4885         $new_conf->{$key} = $value;
4886         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4887     };
4888
4889     foreach my $opt (keys %{$conf->{pending}}) {
4890         next if $selection && !$selection->{$opt};
4891         my $value = $conf->{pending}->{$opt};
4892         eval {
4893             if ($opt eq 'hotplug') {
4894                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4895             } elsif ($opt eq 'tablet') {
4896                 die "skip\n" if !$hotplug_features->{usb};
4897                 if ($value == 1) {
4898                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4899                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4900                         if $arch eq 'aarch64';
4901                 } elsif ($value == 0) {
4902                     vm_deviceunplug($vmid, $conf, 'tablet');
4903                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4904                 }
4905             } elsif ($opt =~ m/^usb\d+$/) {
4906                 die "skip\n";
4907                 # since we cannot reliably hot unplug usb devices we disable it for now
4908                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4909                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4910                 #die "skip\n" if !$d;
4911                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4912             } elsif ($opt eq 'vcpus') {
4913                 die "skip\n" if !$hotplug_features->{cpu};
4914                 qemu_cpu_hotplug($vmid, $conf, $value);
4915             } elsif ($opt eq 'balloon') {
4916                 # enable/disable balloning device is not hotpluggable
4917                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4918                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4919                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4920
4921                 # allow manual ballooning if shares is set to zero
4922                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4923                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4924                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4925                 }
4926             } elsif ($opt =~ m/^net(\d+)$/) {
4927                 # some changes can be done without hotplug
4928                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4929                                     $vmid, $opt, $value, $arch, $machine_type);
4930             } elsif (is_valid_drivename($opt)) {
4931                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4932                 # some changes can be done without hotplug
4933                 my $drive = parse_drive($opt, $value);
4934                 if (drive_is_cloudinit($drive)) {
4935                     &$apply_pending_cloudinit($opt, $value);
4936                 }
4937                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4938                                      $vmid, $opt, $value, $arch, $machine_type);
4939             } elsif ($opt =~ m/^memory$/) { #dimms
4940                 die "skip\n" if !$hotplug_features->{memory};
4941                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4942             } elsif ($opt eq 'cpuunits') {
4943                 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4944                 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4945             } elsif ($opt eq 'cpulimit') {
4946                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4947                 $cgroup->change_cpu_quota($cpulimit, 100000);
4948             } elsif ($opt eq 'agent') {
4949                 vmconfig_update_agent($conf, $opt, $value);
4950             } else {
4951                 die "skip\n";  # skip non-hot-pluggable options
4952             }
4953         };
4954         if (my $err = $@) {
4955             &$add_error($opt, $err) if $err ne "skip\n";
4956         } else {
4957             $conf->{$opt} = $value;
4958             delete $conf->{pending}->{$opt};
4959         }
4960     }
4961
4962     PVE::QemuConfig->write_config($vmid, $conf);
4963 }
4964
4965 sub try_deallocate_drive {
4966     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4967
4968     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4969         my $volid = $drive->{file};
4970         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4971             my $sid = PVE::Storage::parse_volume_id($volid);
4972             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4973
4974             # check if the disk is really unused
4975             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4976                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4977             PVE::Storage::vdisk_free($storecfg, $volid);
4978             return 1;
4979         } else {
4980             # If vm is not owner of this disk remove from config
4981             return 1;
4982         }
4983     }
4984
4985     return;
4986 }
4987
4988 sub vmconfig_delete_or_detach_drive {
4989     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4990
4991     my $drive = parse_drive($opt, $conf->{$opt});
4992
4993     my $rpcenv = PVE::RPCEnvironment::get();
4994     my $authuser = $rpcenv->get_user();
4995
4996     if ($force) {
4997         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4998         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4999     } else {
5000         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5001     }
5002 }
5003
5004
5005
5006 sub vmconfig_apply_pending {
5007     my ($vmid, $conf, $storecfg, $errors) = @_;
5008
5009     return if !scalar(keys %{$conf->{pending}});
5010
5011     my $add_apply_error = sub {
5012         my ($opt, $msg) = @_;
5013         my $err_msg = "unable to apply pending change $opt : $msg";
5014         $errors->{$opt} = $err_msg;
5015         warn $err_msg;
5016     };
5017
5018     # cold plug
5019
5020     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5021     foreach my $opt (sort keys %$pending_delete_hash) {
5022         my $force = $pending_delete_hash->{$opt}->{force};
5023         eval {
5024             if ($opt =~ m/^unused/) {
5025                 die "internal error";
5026             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5027                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5028             }
5029         };
5030         if (my $err = $@) {
5031             $add_apply_error->($opt, $err);
5032         } else {
5033             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5034             delete $conf->{$opt};
5035         }
5036     }
5037
5038     PVE::QemuConfig->cleanup_pending($conf);
5039
5040     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5041         next if $opt eq 'delete'; # just to be sure
5042         eval {
5043             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5044                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5045             }
5046         };
5047         if (my $err = $@) {
5048             $add_apply_error->($opt, $err);
5049         } else {
5050             $conf->{$opt} = delete $conf->{pending}->{$opt};
5051         }
5052     }
5053
5054     # write all changes at once to avoid unnecessary i/o
5055     PVE::QemuConfig->write_config($vmid, $conf);
5056 }
5057
5058 sub vmconfig_update_net {
5059     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5060
5061     my $newnet = parse_net($value);
5062
5063     if ($conf->{$opt}) {
5064         my $oldnet = parse_net($conf->{$opt});
5065
5066         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5067             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5068             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5069             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5070
5071             # for non online change, we try to hot-unplug
5072             die "skip\n" if !$hotplug;
5073             vm_deviceunplug($vmid, $conf, $opt);
5074         } else {
5075
5076             die "internal error" if $opt !~ m/net(\d+)/;
5077             my $iface = "tap${vmid}i$1";
5078
5079             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5080                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5081                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5082                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5083                 PVE::Network::tap_unplug($iface);
5084
5085                 if ($have_sdn) {
5086                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5087                 } else {
5088                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5089                 }
5090             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5091                 # Rate can be applied on its own but any change above needs to
5092                 # include the rate in tap_plug since OVS resets everything.
5093                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5094             }
5095
5096             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5097                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5098             }
5099
5100             return 1;
5101         }
5102     }
5103
5104     if ($hotplug) {
5105         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5106     } else {
5107         die "skip\n";
5108     }
5109 }
5110
5111 sub vmconfig_update_agent {
5112     my ($conf, $opt, $value) = @_;
5113
5114     die "skip\n" if !$conf->{$opt};
5115
5116     my $hotplug_options = { fstrim_cloned_disks => 1 };
5117
5118     my $old_agent = parse_guest_agent($conf);
5119     my $agent = parse_guest_agent({$opt => $value});
5120
5121     for my $option (keys %$agent) { # added/changed options
5122         next if defined($hotplug_options->{$option});
5123         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5124     }
5125
5126     for my $option (keys %$old_agent) { # removed options
5127         next if defined($hotplug_options->{$option});
5128         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5129     }
5130
5131     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5132 }
5133
5134 sub vmconfig_update_disk {
5135     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5136
5137     my $drive = parse_drive($opt, $value);
5138
5139     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5140         my $media = $drive->{media} || 'disk';
5141         my $oldmedia = $old_drive->{media} || 'disk';
5142         die "unable to change media type\n" if $media ne $oldmedia;
5143
5144         if (!drive_is_cdrom($old_drive)) {
5145
5146             if ($drive->{file} ne $old_drive->{file}) {
5147
5148                 die "skip\n" if !$hotplug;
5149
5150                 # unplug and register as unused
5151                 vm_deviceunplug($vmid, $conf, $opt);
5152                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5153
5154             } else {
5155                 # update existing disk
5156
5157                 # skip non hotpluggable value
5158                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5159                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5160                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5161                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5162                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5163                     die "skip\n";
5164                 }
5165
5166                 # apply throttle
5167                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5168                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5169                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5170                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5171                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5172                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5173                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5174                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5175                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5176                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5177                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5178                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5179                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5180                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5181                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5182                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5183                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5184                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5185
5186                     qemu_block_set_io_throttle(
5187                         $vmid,"drive-$opt",
5188                         ($drive->{mbps} || 0)*1024*1024,
5189                         ($drive->{mbps_rd} || 0)*1024*1024,
5190                         ($drive->{mbps_wr} || 0)*1024*1024,
5191                          $drive->{iops} || 0,
5192                          $drive->{iops_rd} || 0,
5193                          $drive->{iops_wr} || 0,
5194                         ($drive->{mbps_max} || 0)*1024*1024,
5195                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5196                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5197                          $drive->{iops_max} || 0,
5198                          $drive->{iops_rd_max} || 0,
5199                          $drive->{iops_wr_max} || 0,
5200                          $drive->{bps_max_length} || 1,
5201                          $drive->{bps_rd_max_length} || 1,
5202                          $drive->{bps_wr_max_length} || 1,
5203                          $drive->{iops_max_length} || 1,
5204                          $drive->{iops_rd_max_length} || 1,
5205                          $drive->{iops_wr_max_length} || 1,
5206                     );
5207
5208                 }
5209
5210                 return 1;
5211             }
5212
5213         } else { # cdrom
5214
5215             if ($drive->{file} eq 'none') {
5216                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5217                 if (drive_is_cloudinit($old_drive)) {
5218                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5219                 }
5220             } else {
5221                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5222
5223                 # force eject if locked
5224                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5225
5226                 if ($path) {
5227                     mon_cmd($vmid, "blockdev-change-medium",
5228                         id => "$opt", filename => "$path");
5229                 }
5230             }
5231
5232             return 1;
5233         }
5234     }
5235
5236     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5237     # hotplug new disks
5238     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5239     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5240 }
5241
5242 # called in locked context by incoming migration
5243 sub vm_migrate_get_nbd_disks {
5244     my ($storecfg, $conf, $replicated_volumes) = @_;
5245
5246     my $local_volumes = {};
5247     PVE::QemuConfig->foreach_volume($conf, sub {
5248         my ($ds, $drive) = @_;
5249
5250         return if drive_is_cdrom($drive);
5251         return if $ds eq 'tpmstate0';
5252
5253         my $volid = $drive->{file};
5254
5255         return if !$volid;
5256
5257         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5258
5259         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5260         return if $scfg->{shared};
5261
5262         # replicated disks re-use existing state via bitmap
5263         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5264         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5265     });
5266     return $local_volumes;
5267 }
5268
5269 # called in locked context by incoming migration
5270 sub vm_migrate_alloc_nbd_disks {
5271     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5272
5273     my $nbd = {};
5274     foreach my $opt (sort keys %$source_volumes) {
5275         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5276
5277         if ($use_existing) {
5278             $nbd->{$opt}->{drivestr} = print_drive($drive);
5279             $nbd->{$opt}->{volid} = $volid;
5280             $nbd->{$opt}->{replicated} = 1;
5281             next;
5282         }
5283
5284         # storage mapping + volname = regular migration
5285         # storage mapping + format = remote migration
5286         # order of precedence, filtered by whether storage supports it:
5287         # 1. explicit requested format
5288         # 2. format of current volume
5289         # 3. default format of storage
5290         if (!$storagemap->{identity}) {
5291             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5292             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5293             if (!$format || !grep { $format eq $_ } @$validFormats) {
5294                 if ($volname) {
5295                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5296                     my $fileFormat = qemu_img_format($scfg, $volname);
5297                     $format = $fileFormat
5298                         if grep { $fileFormat eq $_ } @$validFormats;
5299                 }
5300                 $format //= $defFormat;
5301             }
5302         } else {
5303             # can't happen for remote migration, so $volname is always defined
5304             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5305             $format = qemu_img_format($scfg, $volname);
5306         }
5307
5308         my $size = $drive->{size} / 1024;
5309         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5310         my $newdrive = $drive;
5311         $newdrive->{format} = $format;
5312         $newdrive->{file} = $newvolid;
5313         my $drivestr = print_drive($newdrive);
5314         $nbd->{$opt}->{drivestr} = $drivestr;
5315         $nbd->{$opt}->{volid} = $newvolid;
5316     }
5317
5318     return $nbd;
5319 }
5320
5321 # see vm_start_nolock for parameters, additionally:
5322 # migrate_opts:
5323 #   storagemap = parsed storage map for allocating NBD disks
5324 sub vm_start {
5325     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5326
5327     return PVE::QemuConfig->lock_config($vmid, sub {
5328         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5329
5330         die "you can't start a vm if it's a template\n"
5331             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5332
5333         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5334         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5335
5336         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5337
5338         if ($has_backup_lock && $running) {
5339             # a backup is currently running, attempt to start the guest in the
5340             # existing QEMU instance
5341             return vm_resume($vmid);
5342         }
5343
5344         PVE::QemuConfig->check_lock($conf)
5345             if !($params->{skiplock} || $has_suspended_lock);
5346
5347         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5348
5349         die "VM $vmid already running\n" if $running;
5350
5351         if (my $storagemap = $migrate_opts->{storagemap}) {
5352             my $replicated = $migrate_opts->{replicated_volumes};
5353             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5354             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5355
5356             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5357                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5358             }
5359         }
5360
5361         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5362     });
5363 }
5364
5365
5366 # params:
5367 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5368 #   skiplock => 0/1, skip checking for config lock
5369 #   skiptemplate => 0/1, skip checking whether VM is template
5370 #   forcemachine => to force Qemu machine (rollback/migration)
5371 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5372 #   timeout => in seconds
5373 #   paused => start VM in paused state (backup)
5374 #   resume => resume from hibernation
5375 #   pbs-backing => {
5376 #      sata0 => {
5377 #         repository
5378 #         snapshot
5379 #         keyfile
5380 #         archive
5381 #      },
5382 #      virtio2 => ...
5383 #   }
5384 # migrate_opts:
5385 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5386 #   migratedfrom => source node
5387 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5388 #   network => CIDR of migration network
5389 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5390 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5391 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5392 #   tpmstate_vol => new volid of tpmstate0, not yet contained in config
5393 sub vm_start_nolock {
5394     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5395
5396     my $statefile = $params->{statefile};
5397     my $resume = $params->{resume};
5398
5399     my $migratedfrom = $migrate_opts->{migratedfrom};
5400     my $migration_type = $migrate_opts->{type};
5401
5402     my $res = {};
5403
5404     # clean up leftover reboot request files
5405     eval { clear_reboot_request($vmid); };
5406     warn $@ if $@;
5407
5408     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5409         vmconfig_apply_pending($vmid, $conf, $storecfg);
5410         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5411     }
5412
5413     # don't regenerate the ISO if the VM is started as part of a live migration
5414     # this way we can reuse the old ISO with the correct config
5415     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5416
5417     # override TPM state vol if migrated, conf is out of date still
5418     if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
5419         my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
5420         $parsed->{file} = $tpmvol;
5421         $conf->{tpmstate0} = print_drive($parsed);
5422     }
5423
5424     my $defaults = load_defaults();
5425
5426     # set environment variable useful inside network script
5427     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5428
5429     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5430
5431     my $forcemachine = $params->{forcemachine};
5432     my $forcecpu = $params->{forcecpu};
5433     if ($resume) {
5434         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5435         $forcemachine = $conf->{runningmachine};
5436         $forcecpu = $conf->{runningcpu};
5437         print "Resuming suspended VM\n";
5438     }
5439
5440     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5441         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5442
5443     my $migration_ip;
5444     my $get_migration_ip = sub {
5445         my ($nodename) = @_;
5446
5447         return $migration_ip if defined($migration_ip);
5448
5449         my $cidr = $migrate_opts->{network};
5450
5451         if (!defined($cidr)) {
5452             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5453             $cidr = $dc_conf->{migration}->{network};
5454         }
5455
5456         if (defined($cidr)) {
5457             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5458
5459             die "could not get IP: no address configured on local " .
5460                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5461
5462             die "could not get IP: multiple addresses configured on local " .
5463                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5464
5465             $migration_ip = @$ips[0];
5466         }
5467
5468         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5469             if !defined($migration_ip);
5470
5471         return $migration_ip;
5472     };
5473
5474     my $migrate_uri;
5475     if ($statefile) {
5476         if ($statefile eq 'tcp') {
5477             my $localip = "localhost";
5478             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5479             my $nodename = nodename();
5480
5481             if (!defined($migration_type)) {
5482                 if (defined($datacenterconf->{migration}->{type})) {
5483                     $migration_type = $datacenterconf->{migration}->{type};
5484                 } else {
5485                     $migration_type = 'secure';
5486                 }
5487             }
5488
5489             if ($migration_type eq 'insecure') {
5490                 $localip = $get_migration_ip->($nodename);
5491                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5492             }
5493
5494             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5495             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5496             $migrate_uri = "tcp:${localip}:${migrate_port}";
5497             push @$cmd, '-incoming', $migrate_uri;
5498             push @$cmd, '-S';
5499
5500         } elsif ($statefile eq 'unix') {
5501             # should be default for secure migrations as a ssh TCP forward
5502             # tunnel is not deterministic reliable ready and fails regurarly
5503             # to set up in time, so use UNIX socket forwards
5504             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5505             unlink $socket_addr;
5506
5507             $migrate_uri = "unix:$socket_addr";
5508
5509             push @$cmd, '-incoming', $migrate_uri;
5510             push @$cmd, '-S';
5511
5512         } elsif (-e $statefile) {
5513             push @$cmd, '-loadstate', $statefile;
5514         } else {
5515             my $statepath = PVE::Storage::path($storecfg, $statefile);
5516             push @$vollist, $statefile;
5517             push @$cmd, '-loadstate', $statepath;
5518         }
5519     } elsif ($params->{paused}) {
5520         push @$cmd, '-S';
5521     }
5522
5523     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5524
5525     my $pci_devices = {}; # host pci devices
5526     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5527         my $dev = $conf->{"hostpci$i"} or next;
5528         $pci_devices->{$i} = parse_hostpci($dev);
5529     }
5530
5531     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5532     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5533
5534     # map to a flat list of pci ids
5535     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5536
5537     # reserve all PCI IDs before actually doing anything with them
5538     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5539
5540     eval {
5541         for my $id (sort keys %$pci_devices) {
5542             my $d = $pci_devices->{$id};
5543             for my $dev ($d->{pciid}->@*) {
5544                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5545             }
5546         }
5547     };
5548     if (my $err = $@) {
5549         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5550         warn $@ if $@;
5551         die $err;
5552     }
5553
5554     PVE::Storage::activate_volumes($storecfg, $vollist);
5555
5556     eval {
5557         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5558     };
5559     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5560     # timeout should be more than enough here...
5561     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5562
5563     my $cpuunits = get_cpuunits($conf);
5564
5565     my %run_params = (
5566         timeout => $statefile ? undef : $start_timeout,
5567         umask => 0077,
5568         noerr => 1,
5569     );
5570
5571     # when migrating, prefix QEMU output so other side can pick up any
5572     # errors that might occur and show the user
5573     if ($migratedfrom) {
5574         $run_params{quiet} = 1;
5575         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5576     }
5577
5578     my %systemd_properties = (
5579         Slice => 'qemu.slice',
5580         KillMode => 'process',
5581         SendSIGKILL => 0,
5582         TimeoutStopUSec => ULONG_MAX, # infinity
5583     );
5584
5585     if (PVE::CGroup::cgroup_mode() == 2) {
5586         $systemd_properties{CPUWeight} = $cpuunits;
5587     } else {
5588         $systemd_properties{CPUShares} = $cpuunits;
5589     }
5590
5591     if (my $cpulimit = $conf->{cpulimit}) {
5592         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5593     }
5594     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5595
5596     my $run_qemu = sub {
5597         PVE::Tools::run_fork sub {
5598             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5599
5600             my $tpmpid;
5601             if (my $tpm = $conf->{tpmstate0}) {
5602                 # start the TPM emulator so QEMU can connect on start
5603                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5604             }
5605
5606             my $exitcode = run_command($cmd, %run_params);
5607             if ($exitcode) {
5608                 if ($tpmpid) {
5609                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5610                     kill 'TERM', $tpmpid;
5611                 }
5612                 die "QEMU exited with code $exitcode\n";
5613             }
5614         };
5615     };
5616
5617     if ($conf->{hugepages}) {
5618
5619         my $code = sub {
5620             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5621             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5622
5623             PVE::QemuServer::Memory::hugepages_mount();
5624             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5625
5626             eval { $run_qemu->() };
5627             if (my $err = $@) {
5628                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5629                     if !$conf->{keephugepages};
5630                 die $err;
5631             }
5632
5633             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5634                 if !$conf->{keephugepages};
5635         };
5636         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5637
5638     } else {
5639         eval { $run_qemu->() };
5640     }
5641
5642     if (my $err = $@) {
5643         # deactivate volumes if start fails
5644         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5645         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5646
5647         die "start failed: $err";
5648     }
5649
5650     # re-reserve all PCI IDs now that we can know the actual VM PID
5651     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5652     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5653     warn $@ if $@;
5654
5655     print "migration listens on $migrate_uri\n" if $migrate_uri;
5656     $res->{migrate_uri} = $migrate_uri;
5657
5658     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5659         eval { mon_cmd($vmid, "cont"); };
5660         warn $@ if $@;
5661     }
5662
5663     #start nbd server for storage migration
5664     if (my $nbd = $migrate_opts->{nbd}) {
5665         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5666
5667         my $migrate_storage_uri;
5668         # nbd_protocol_version > 0 for unix socket support
5669         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5670             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5671             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5672             $migrate_storage_uri = "nbd:unix:$socket_path";
5673         } else {
5674             my $nodename = nodename();
5675             my $localip = $get_migration_ip->($nodename);
5676             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5677             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5678
5679             mon_cmd($vmid, "nbd-server-start", addr => {
5680                 type => 'inet',
5681                 data => {
5682                     host => "${localip}",
5683                     port => "${storage_migrate_port}",
5684                 },
5685             });
5686             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5687             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5688         }
5689
5690         $res->{migrate_storage_uri} = $migrate_storage_uri;
5691
5692         foreach my $opt (sort keys %$nbd) {
5693             my $drivestr = $nbd->{$opt}->{drivestr};
5694             my $volid = $nbd->{$opt}->{volid};
5695             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5696             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5697             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5698             print "re-using replicated volume: $opt - $volid\n"
5699                 if $nbd->{$opt}->{replicated};
5700
5701             $res->{drives}->{$opt} = $nbd->{$opt};
5702             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5703         }
5704     }
5705
5706     if ($migratedfrom) {
5707         eval {
5708             set_migration_caps($vmid);
5709         };
5710         warn $@ if $@;
5711
5712         if ($spice_port) {
5713             print "spice listens on port $spice_port\n";
5714             $res->{spice_port} = $spice_port;
5715             if ($migrate_opts->{spice_ticket}) {
5716                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5717                     $migrate_opts->{spice_ticket});
5718                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5719             }
5720         }
5721
5722     } else {
5723         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5724             if !$statefile && $conf->{balloon};
5725
5726         foreach my $opt (keys %$conf) {
5727             next if $opt !~  m/^net\d+$/;
5728             my $nicconf = parse_net($conf->{$opt});
5729             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5730         }
5731     }
5732
5733     mon_cmd($vmid, 'qom-set',
5734                 path => "machine/peripheral/balloon0",
5735                 property => "guest-stats-polling-interval",
5736                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5737
5738     if ($resume) {
5739         print "Resumed VM, removing state\n";
5740         if (my $vmstate = $conf->{vmstate}) {
5741             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5742             PVE::Storage::vdisk_free($storecfg, $vmstate);
5743         }
5744         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5745         PVE::QemuConfig->write_config($vmid, $conf);
5746     }
5747
5748     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5749
5750     return $res;
5751 }
5752
5753 sub vm_commandline {
5754     my ($storecfg, $vmid, $snapname) = @_;
5755
5756     my $conf = PVE::QemuConfig->load_config($vmid);
5757
5758     my ($forcemachine, $forcecpu);
5759     if ($snapname) {
5760         my $snapshot = $conf->{snapshots}->{$snapname};
5761         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5762
5763         # check for machine or CPU overrides in snapshot
5764         $forcemachine = $snapshot->{runningmachine};
5765         $forcecpu = $snapshot->{runningcpu};
5766
5767         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5768
5769         $conf = $snapshot;
5770     }
5771
5772     my $defaults = load_defaults();
5773
5774     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5775
5776     return PVE::Tools::cmd2string($cmd);
5777 }
5778
5779 sub vm_reset {
5780     my ($vmid, $skiplock) = @_;
5781
5782     PVE::QemuConfig->lock_config($vmid, sub {
5783
5784         my $conf = PVE::QemuConfig->load_config($vmid);
5785
5786         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5787
5788         mon_cmd($vmid, "system_reset");
5789     });
5790 }
5791
5792 sub get_vm_volumes {
5793     my ($conf) = @_;
5794
5795     my $vollist = [];
5796     foreach_volid($conf, sub {
5797         my ($volid, $attr) = @_;
5798
5799         return if $volid =~ m|^/|;
5800
5801         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5802         return if !$sid;
5803
5804         push @$vollist, $volid;
5805     });
5806
5807     return $vollist;
5808 }
5809
5810 sub vm_stop_cleanup {
5811     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5812
5813     eval {
5814
5815         if (!$keepActive) {
5816             my $vollist = get_vm_volumes($conf);
5817             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5818
5819             if (my $tpmdrive = $conf->{tpmstate0}) {
5820                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5821                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5822                 if ($storeid) {
5823                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5824                 }
5825             }
5826         }
5827
5828         foreach my $ext (qw(mon qmp pid vnc qga)) {
5829             unlink "/var/run/qemu-server/${vmid}.$ext";
5830         }
5831
5832         if ($conf->{ivshmem}) {
5833             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5834             # just delete it for now, VMs which have this already open do not
5835             # are affected, but new VMs will get a separated one. If this
5836             # becomes an issue we either add some sort of ref-counting or just
5837             # add a "don't delete on stop" flag to the ivshmem format.
5838             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5839         }
5840
5841         my $ids = [];
5842         foreach my $key (keys %$conf) {
5843             next if $key !~ m/^hostpci(\d+)$/;
5844             my $hostpciindex = $1;
5845             my $d = parse_hostpci($conf->{$key});
5846             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5847
5848             foreach my $pci (@{$d->{pciid}}) {
5849                 my $pciid = $pci->{id};
5850                 push @$ids, $pci->{id};
5851                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5852             }
5853         }
5854         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5855
5856         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5857     };
5858     warn $@ if $@; # avoid errors - just warn
5859 }
5860
5861 # call only in locked context
5862 sub _do_vm_stop {
5863     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5864
5865     my $pid = check_running($vmid, $nocheck);
5866     return if !$pid;
5867
5868     my $conf;
5869     if (!$nocheck) {
5870         $conf = PVE::QemuConfig->load_config($vmid);
5871         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5872         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5873             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5874             $timeout = $opts->{down} if $opts->{down};
5875         }
5876         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5877     }
5878
5879     eval {
5880         if ($shutdown) {
5881             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5882                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5883             } else {
5884                 mon_cmd($vmid, "system_powerdown");
5885             }
5886         } else {
5887             mon_cmd($vmid, "quit");
5888         }
5889     };
5890     my $err = $@;
5891
5892     if (!$err) {
5893         $timeout = 60 if !defined($timeout);
5894
5895         my $count = 0;
5896         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5897             $count++;
5898             sleep 1;
5899         }
5900
5901         if ($count >= $timeout) {
5902             if ($force) {
5903                 warn "VM still running - terminating now with SIGTERM\n";
5904                 kill 15, $pid;
5905             } else {
5906                 die "VM quit/powerdown failed - got timeout\n";
5907             }
5908         } else {
5909             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5910             return;
5911         }
5912     } else {
5913        if (!check_running($vmid, $nocheck)) {
5914             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5915             return;
5916        }
5917        if ($force) {
5918             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5919             kill 15, $pid;
5920         } else {
5921             die "VM quit/powerdown failed\n";
5922         }
5923     }
5924
5925     # wait again
5926     $timeout = 10;
5927
5928     my $count = 0;
5929     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5930         $count++;
5931         sleep 1;
5932     }
5933
5934     if ($count >= $timeout) {
5935         warn "VM still running - terminating now with SIGKILL\n";
5936         kill 9, $pid;
5937         sleep 1;
5938     }
5939
5940     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5941 }
5942
5943 # Note: use $nocheck to skip tests if VM configuration file exists.
5944 # We need that when migration VMs to other nodes (files already moved)
5945 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5946 sub vm_stop {
5947     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5948
5949     $force = 1 if !defined($force) && !$shutdown;
5950
5951     if ($migratedfrom){
5952         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5953         kill 15, $pid if $pid;
5954         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5955         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5956         return;
5957     }
5958
5959     PVE::QemuConfig->lock_config($vmid, sub {
5960         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5961    });
5962 }
5963
5964 sub vm_reboot {
5965     my ($vmid, $timeout) = @_;
5966
5967     PVE::QemuConfig->lock_config($vmid, sub {
5968         eval {
5969
5970             # only reboot if running, as qmeventd starts it again on a stop event
5971             return if !check_running($vmid);
5972
5973             create_reboot_request($vmid);
5974
5975             my $storecfg = PVE::Storage::config();
5976             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5977
5978         };
5979         if (my $err = $@) {
5980             # avoid that the next normal shutdown will be confused for a reboot
5981             clear_reboot_request($vmid);
5982             die $err;
5983         }
5984    });
5985 }
5986
5987 # note: if using the statestorage parameter, the caller has to check privileges
5988 sub vm_suspend {
5989     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5990
5991     my $conf;
5992     my $path;
5993     my $storecfg;
5994     my $vmstate;
5995
5996     PVE::QemuConfig->lock_config($vmid, sub {
5997
5998         $conf = PVE::QemuConfig->load_config($vmid);
5999
6000         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6001         PVE::QemuConfig->check_lock($conf)
6002             if !($skiplock || $is_backing_up);
6003
6004         die "cannot suspend to disk during backup\n"
6005             if $is_backing_up && $includestate;
6006
6007         if ($includestate) {
6008             $conf->{lock} = 'suspending';
6009             my $date = strftime("%Y-%m-%d", localtime(time()));
6010             $storecfg = PVE::Storage::config();
6011             if (!$statestorage) {
6012                 $statestorage = find_vmstate_storage($conf, $storecfg);
6013                 # check permissions for the storage
6014                 my $rpcenv = PVE::RPCEnvironment::get();
6015                 if ($rpcenv->{type} ne 'cli') {
6016                     my $authuser = $rpcenv->get_user();
6017                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6018                 }
6019             }
6020
6021
6022             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6023                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6024             $path = PVE::Storage::path($storecfg, $vmstate);
6025             PVE::QemuConfig->write_config($vmid, $conf);
6026         } else {
6027             mon_cmd($vmid, "stop");
6028         }
6029     });
6030
6031     if ($includestate) {
6032         # save vm state
6033         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6034
6035         eval {
6036             set_migration_caps($vmid, 1);
6037             mon_cmd($vmid, "savevm-start", statefile => $path);
6038             for(;;) {
6039                 my $state = mon_cmd($vmid, "query-savevm");
6040                 if (!$state->{status}) {
6041                     die "savevm not active\n";
6042                 } elsif ($state->{status} eq 'active') {
6043                     sleep(1);
6044                     next;
6045                 } elsif ($state->{status} eq 'completed') {
6046                     print "State saved, quitting\n";
6047                     last;
6048                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6049                     die "query-savevm failed with error '$state->{error}'\n"
6050                 } else {
6051                     die "query-savevm returned status '$state->{status}'\n";
6052                 }
6053             }
6054         };
6055         my $err = $@;
6056
6057         PVE::QemuConfig->lock_config($vmid, sub {
6058             $conf = PVE::QemuConfig->load_config($vmid);
6059             if ($err) {
6060                 # cleanup, but leave suspending lock, to indicate something went wrong
6061                 eval {
6062                     mon_cmd($vmid, "savevm-end");
6063                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6064                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6065                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6066                     PVE::QemuConfig->write_config($vmid, $conf);
6067                 };
6068                 warn $@ if $@;
6069                 die $err;
6070             }
6071
6072             die "lock changed unexpectedly\n"
6073                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6074
6075             mon_cmd($vmid, "quit");
6076             $conf->{lock} = 'suspended';
6077             PVE::QemuConfig->write_config($vmid, $conf);
6078         });
6079     }
6080 }
6081
6082 sub vm_resume {
6083     my ($vmid, $skiplock, $nocheck) = @_;
6084
6085     PVE::QemuConfig->lock_config($vmid, sub {
6086         my $res = mon_cmd($vmid, 'query-status');
6087         my $resume_cmd = 'cont';
6088         my $reset = 0;
6089
6090         if ($res->{status}) {
6091             return if $res->{status} eq 'running'; # job done, go home
6092             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6093             $reset = 1 if $res->{status} eq 'shutdown';
6094         }
6095
6096         if (!$nocheck) {
6097
6098             my $conf = PVE::QemuConfig->load_config($vmid);
6099
6100             PVE::QemuConfig->check_lock($conf)
6101                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6102         }
6103
6104         if ($reset) {
6105             # required if a VM shuts down during a backup and we get a resume
6106             # request before the backup finishes for example
6107             mon_cmd($vmid, "system_reset");
6108         }
6109         mon_cmd($vmid, $resume_cmd);
6110     });
6111 }
6112
6113 sub vm_sendkey {
6114     my ($vmid, $skiplock, $key) = @_;
6115
6116     PVE::QemuConfig->lock_config($vmid, sub {
6117
6118         my $conf = PVE::QemuConfig->load_config($vmid);
6119
6120         # there is no qmp command, so we use the human monitor command
6121         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6122         die $res if $res ne '';
6123     });
6124 }
6125
6126 # vzdump restore implementaion
6127
6128 sub tar_archive_read_firstfile {
6129     my $archive = shift;
6130
6131     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6132
6133     # try to detect archive type first
6134     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6135         die "unable to open file '$archive'\n";
6136     my $firstfile = <$fh>;
6137     kill 15, $pid;
6138     close $fh;
6139
6140     die "ERROR: archive contaions no data\n" if !$firstfile;
6141     chomp $firstfile;
6142
6143     return $firstfile;
6144 }
6145
6146 sub tar_restore_cleanup {
6147     my ($storecfg, $statfile) = @_;
6148
6149     print STDERR "starting cleanup\n";
6150
6151     if (my $fd = IO::File->new($statfile, "r")) {
6152         while (defined(my $line = <$fd>)) {
6153             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6154                 my $volid = $2;
6155                 eval {
6156                     if ($volid =~ m|^/|) {
6157                         unlink $volid || die 'unlink failed\n';
6158                     } else {
6159                         PVE::Storage::vdisk_free($storecfg, $volid);
6160                     }
6161                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6162                 };
6163                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6164             } else {
6165                 print STDERR "unable to parse line in statfile - $line";
6166             }
6167         }
6168         $fd->close();
6169     }
6170 }
6171
6172 sub restore_file_archive {
6173     my ($archive, $vmid, $user, $opts) = @_;
6174
6175     return restore_vma_archive($archive, $vmid, $user, $opts)
6176         if $archive eq '-';
6177
6178     my $info = PVE::Storage::archive_info($archive);
6179     my $format = $opts->{format} // $info->{format};
6180     my $comp = $info->{compression};
6181
6182     # try to detect archive format
6183     if ($format eq 'tar') {
6184         return restore_tar_archive($archive, $vmid, $user, $opts);
6185     } else {
6186         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6187     }
6188 }
6189
6190 # hepler to remove disks that will not be used after restore
6191 my $restore_cleanup_oldconf = sub {
6192     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6193
6194     PVE::QemuConfig->foreach_volume($oldconf, sub {
6195         my ($ds, $drive) = @_;
6196
6197         return if drive_is_cdrom($drive, 1);
6198
6199         my $volid = $drive->{file};
6200         return if !$volid || $volid =~ m|^/|;
6201
6202         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6203         return if !$path || !$owner || ($owner != $vmid);
6204
6205         # Note: only delete disk we want to restore
6206         # other volumes will become unused
6207         if ($virtdev_hash->{$ds}) {
6208             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6209             if (my $err = $@) {
6210                 warn $err;
6211             }
6212         }
6213     });
6214
6215     # delete vmstate files, after the restore we have no snapshots anymore
6216     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6217         my $snap = $oldconf->{snapshots}->{$snapname};
6218         if ($snap->{vmstate}) {
6219             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6220             if (my $err = $@) {
6221                 warn $err;
6222             }
6223         }
6224     }
6225 };
6226
6227 # Helper to parse vzdump backup device hints
6228 #
6229 # $rpcenv: Environment, used to ckeck storage permissions
6230 # $user: User ID, to check storage permissions
6231 # $storecfg: Storage configuration
6232 # $fh: the file handle for reading the configuration
6233 # $devinfo: should contain device sizes for all backu-up'ed devices
6234 # $options: backup options (pool, default storage)
6235 #
6236 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6237 my $parse_backup_hints = sub {
6238     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6239
6240     my $check_storage = sub { # assert if an image can be allocate
6241         my ($storeid, $scfg) = @_;
6242         die "Content type 'images' is not available on storage '$storeid'\n"
6243             if !$scfg->{content}->{images};
6244         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6245             if $user ne 'root@pam';
6246     };
6247
6248     my $virtdev_hash = {};
6249     while (defined(my $line = <$fh>)) {
6250         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6251             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6252             die "archive does not contain data for drive '$virtdev'\n"
6253                 if !$devinfo->{$devname};
6254
6255             if (defined($options->{storage})) {
6256                 $storeid = $options->{storage} || 'local';
6257             } elsif (!$storeid) {
6258                 $storeid = 'local';
6259             }
6260             $format = 'raw' if !$format;
6261             $devinfo->{$devname}->{devname} = $devname;
6262             $devinfo->{$devname}->{virtdev} = $virtdev;
6263             $devinfo->{$devname}->{format} = $format;
6264             $devinfo->{$devname}->{storeid} = $storeid;
6265
6266             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6267             $check_storage->($storeid, $scfg); # permission and content type check
6268
6269             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6270         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6271             my $virtdev = $1;
6272             my $drive = parse_drive($virtdev, $2);
6273
6274             if (drive_is_cloudinit($drive)) {
6275                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6276                 $storeid = $options->{storage} if defined ($options->{storage});
6277                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6278                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6279
6280                 $check_storage->($storeid, $scfg); # permission and content type check
6281
6282                 $virtdev_hash->{$virtdev} = {
6283                     format => $format,
6284                     storeid => $storeid,
6285                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6286                     is_cloudinit => 1,
6287                 };
6288             }
6289         }
6290     }
6291
6292     return $virtdev_hash;
6293 };
6294
6295 # Helper to allocate and activate all volumes required for a restore
6296 #
6297 # $storecfg: Storage configuration
6298 # $virtdev_hash: as returned by parse_backup_hints()
6299 #
6300 # Returns: { $virtdev => $volid }
6301 my $restore_allocate_devices = sub {
6302     my ($storecfg, $virtdev_hash, $vmid) = @_;
6303
6304     my $map = {};
6305     foreach my $virtdev (sort keys %$virtdev_hash) {
6306         my $d = $virtdev_hash->{$virtdev};
6307         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6308         my $storeid = $d->{storeid};
6309         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6310
6311         # test if requested format is supported
6312         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6313         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6314         $d->{format} = $defFormat if !$supported;
6315
6316         my $name;
6317         if ($d->{is_cloudinit}) {
6318             $name = "vm-$vmid-cloudinit";
6319             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6320             if ($scfg->{path}) {
6321                 $name .= ".$d->{format}";
6322             }
6323         }
6324
6325         my $volid = PVE::Storage::vdisk_alloc(
6326             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6327
6328         print STDERR "new volume ID is '$volid'\n";
6329         $d->{volid} = $volid;
6330
6331         PVE::Storage::activate_volumes($storecfg, [$volid]);
6332
6333         $map->{$virtdev} = $volid;
6334     }
6335
6336     return $map;
6337 };
6338
6339 sub restore_update_config_line {
6340     my ($cookie, $map, $line, $unique) = @_;
6341
6342     return '' if $line =~ m/^\#qmdump\#/;
6343     return '' if $line =~ m/^\#vzdump\#/;
6344     return '' if $line =~ m/^lock:/;
6345     return '' if $line =~ m/^unused\d+:/;
6346     return '' if $line =~ m/^parent:/;
6347
6348     my $res = '';
6349
6350     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6351     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6352         # try to convert old 1.X settings
6353         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6354         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6355             my ($model, $macaddr) = split(/\=/, $devconfig);
6356             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6357             my $net = {
6358                 model => $model,
6359                 bridge => "vmbr$ind",
6360                 macaddr => $macaddr,
6361             };
6362             my $netstr = print_net($net);
6363
6364             $res .= "net$cookie->{netcount}: $netstr\n";
6365             $cookie->{netcount}++;
6366         }
6367     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6368         my ($id, $netstr) = ($1, $2);
6369         my $net = parse_net($netstr);
6370         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6371         $netstr = print_net($net);
6372         $res .= "$id: $netstr\n";
6373     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6374         my $virtdev = $1;
6375         my $value = $3;
6376         my $di = parse_drive($virtdev, $value);
6377         if (defined($di->{backup}) && !$di->{backup}) {
6378             $res .= "#$line";
6379         } elsif ($map->{$virtdev}) {
6380             delete $di->{format}; # format can change on restore
6381             $di->{file} = $map->{$virtdev};
6382             $value = print_drive($di);
6383             $res .= "$virtdev: $value\n";
6384         } else {
6385             $res .= $line;
6386         }
6387     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6388         my $vmgenid = $1;
6389         if ($vmgenid ne '0') {
6390             # always generate a new vmgenid if there was a valid one setup
6391             $vmgenid = generate_uuid();
6392         }
6393         $res .= "vmgenid: $vmgenid\n";
6394     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6395         my ($uuid, $uuid_str);
6396         UUID::generate($uuid);
6397         UUID::unparse($uuid, $uuid_str);
6398         my $smbios1 = parse_smbios1($2);
6399         $smbios1->{uuid} = $uuid_str;
6400         $res .= $1.print_smbios1($smbios1)."\n";
6401     } else {
6402         $res .= $line;
6403     }
6404
6405     return $res;
6406 }
6407
6408 my $restore_deactivate_volumes = sub {
6409     my ($storecfg, $devinfo) = @_;
6410
6411     my $vollist = [];
6412     foreach my $devname (keys %$devinfo) {
6413         my $volid = $devinfo->{$devname}->{volid};
6414         push @$vollist, $volid if $volid;
6415     }
6416
6417     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6418 };
6419
6420 my $restore_destroy_volumes = sub {
6421     my ($storecfg, $devinfo) = @_;
6422
6423     foreach my $devname (keys %$devinfo) {
6424         my $volid = $devinfo->{$devname}->{volid};
6425         next if !$volid;
6426         eval {
6427             if ($volid =~ m|^/|) {
6428                 unlink $volid || die 'unlink failed\n';
6429             } else {
6430                 PVE::Storage::vdisk_free($storecfg, $volid);
6431             }
6432             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6433         };
6434         print STDERR "unable to cleanup '$volid' - $@" if $@;
6435     }
6436 };
6437
6438 sub scan_volids {
6439     my ($cfg, $vmid) = @_;
6440
6441     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6442
6443     my $volid_hash = {};
6444     foreach my $storeid (keys %$info) {
6445         foreach my $item (@{$info->{$storeid}}) {
6446             next if !($item->{volid} && $item->{size});
6447             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6448             $volid_hash->{$item->{volid}} = $item;
6449         }
6450     }
6451
6452     return $volid_hash;
6453 }
6454
6455 sub update_disk_config {
6456     my ($vmid, $conf, $volid_hash) = @_;
6457
6458     my $changes;
6459     my $prefix = "VM $vmid";
6460
6461     # used and unused disks
6462     my $referenced = {};
6463
6464     # Note: it is allowed to define multiple storages with same path (alias), so
6465     # we need to check both 'volid' and real 'path' (two different volid can point
6466     # to the same path).
6467
6468     my $referencedpath = {};
6469
6470     # update size info
6471     PVE::QemuConfig->foreach_volume($conf, sub {
6472         my ($opt, $drive) = @_;
6473
6474         my $volid = $drive->{file};
6475         return if !$volid;
6476         my $volume = $volid_hash->{$volid};
6477
6478         # mark volid as "in-use" for next step
6479         $referenced->{$volid} = 1;
6480         if ($volume && (my $path = $volume->{path})) {
6481             $referencedpath->{$path} = 1;
6482         }
6483
6484         return if drive_is_cdrom($drive);
6485         return if !$volume;
6486
6487         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6488         if (defined($updated)) {
6489             $changes = 1;
6490             $conf->{$opt} = print_drive($updated);
6491             print "$prefix ($opt): $msg\n";
6492         }
6493     });
6494
6495     # remove 'unusedX' entry if volume is used
6496     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6497         my ($opt, $drive) = @_;
6498
6499         my $volid = $drive->{file};
6500         return if !$volid;
6501
6502         my $path;
6503         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6504         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6505             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6506             $changes = 1;
6507             delete $conf->{$opt};
6508         }
6509
6510         $referenced->{$volid} = 1;
6511         $referencedpath->{$path} = 1 if $path;
6512     });
6513
6514     foreach my $volid (sort keys %$volid_hash) {
6515         next if $volid =~ m/vm-$vmid-state-/;
6516         next if $referenced->{$volid};
6517         my $path = $volid_hash->{$volid}->{path};
6518         next if !$path; # just to be sure
6519         next if $referencedpath->{$path};
6520         $changes = 1;
6521         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6522         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6523         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6524     }
6525
6526     return $changes;
6527 }
6528
6529 sub rescan {
6530     my ($vmid, $nolock, $dryrun) = @_;
6531
6532     my $cfg = PVE::Storage::config();
6533
6534     print "rescan volumes...\n";
6535     my $volid_hash = scan_volids($cfg, $vmid);
6536
6537     my $updatefn =  sub {
6538         my ($vmid) = @_;
6539
6540         my $conf = PVE::QemuConfig->load_config($vmid);
6541
6542         PVE::QemuConfig->check_lock($conf);
6543
6544         my $vm_volids = {};
6545         foreach my $volid (keys %$volid_hash) {
6546             my $info = $volid_hash->{$volid};
6547             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6548         }
6549
6550         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6551
6552         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6553     };
6554
6555     if (defined($vmid)) {
6556         if ($nolock) {
6557             &$updatefn($vmid);
6558         } else {
6559             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6560         }
6561     } else {
6562         my $vmlist = config_list();
6563         foreach my $vmid (keys %$vmlist) {
6564             if ($nolock) {
6565                 &$updatefn($vmid);
6566             } else {
6567                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6568             }
6569         }
6570     }
6571 }
6572
6573 sub restore_proxmox_backup_archive {
6574     my ($archive, $vmid, $user, $options) = @_;
6575
6576     my $storecfg = PVE::Storage::config();
6577
6578     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6579     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6580
6581     my $fingerprint = $scfg->{fingerprint};
6582     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6583
6584     my $repo = PVE::PBSClient::get_repository($scfg);
6585
6586     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6587     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6588     local $ENV{PBS_PASSWORD} = $password;
6589     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6590
6591     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6592         PVE::Storage::parse_volname($storecfg, $archive);
6593
6594     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6595
6596     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6597
6598     my $tmpdir = "/var/tmp/vzdumptmp$$";
6599     rmtree $tmpdir;
6600     mkpath $tmpdir;
6601
6602     my $conffile = PVE::QemuConfig->config_file($vmid);
6603      # disable interrupts (always do cleanups)
6604     local $SIG{INT} =
6605         local $SIG{TERM} =
6606         local $SIG{QUIT} =
6607         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6608
6609     # Note: $oldconf is undef if VM does not exists
6610     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6611     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6612     my $new_conf_raw = '';
6613
6614     my $rpcenv = PVE::RPCEnvironment::get();
6615     my $devinfo = {};
6616
6617     eval {
6618         # enable interrupts
6619         local $SIG{INT} =
6620             local $SIG{TERM} =
6621             local $SIG{QUIT} =
6622             local $SIG{HUP} =
6623             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6624
6625         my $cfgfn = "$tmpdir/qemu-server.conf";
6626         my $firewall_config_fn = "$tmpdir/fw.conf";
6627         my $index_fn = "$tmpdir/index.json";
6628
6629         my $cmd = "restore";
6630
6631         my $param = [$pbs_backup_name, "index.json", $index_fn];
6632         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6633         my $index = PVE::Tools::file_get_contents($index_fn);
6634         $index = decode_json($index);
6635
6636         # print Dumper($index);
6637         foreach my $info (@{$index->{files}}) {
6638             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6639                 my $devname = $1;
6640                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6641                     $devinfo->{$devname}->{size} = $1;
6642                 } else {
6643                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6644                 }
6645             }
6646         }
6647
6648         my $is_qemu_server_backup = scalar(
6649             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6650         );
6651         if (!$is_qemu_server_backup) {
6652             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6653         }
6654         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6655
6656         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6657         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6658
6659         if ($has_firewall_config) {
6660             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6661             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6662
6663             my $pve_firewall_dir = '/etc/pve/firewall';
6664             mkdir $pve_firewall_dir; # make sure the dir exists
6665             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6666         }
6667
6668         my $fh = IO::File->new($cfgfn, "r") ||
6669             die "unable to read qemu-server.conf - $!\n";
6670
6671         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6672
6673         # fixme: rate limit?
6674
6675         # create empty/temp config
6676         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6677
6678         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6679
6680         # allocate volumes
6681         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6682
6683         foreach my $virtdev (sort keys %$virtdev_hash) {
6684             my $d = $virtdev_hash->{$virtdev};
6685             next if $d->{is_cloudinit}; # no need to restore cloudinit
6686
6687             # this fails if storage is unavailable
6688             my $volid = $d->{volid};
6689             my $path = PVE::Storage::path($storecfg, $volid);
6690
6691             # for live-restore we only want to preload the efidisk and TPM state
6692             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6693
6694             my $pbs_restore_cmd = [
6695                 '/usr/bin/pbs-restore',
6696                 '--repository', $repo,
6697                 $pbs_backup_name,
6698                 "$d->{devname}.img.fidx",
6699                 $path,
6700                 '--verbose',
6701                 ];
6702
6703             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6704             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6705
6706             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6707                 push @$pbs_restore_cmd, '--skip-zero';
6708             }
6709
6710             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6711             print "restore proxmox backup image: $dbg_cmdstring\n";
6712             run_command($pbs_restore_cmd);
6713         }
6714
6715         $fh->seek(0, 0) || die "seek failed - $!\n";
6716
6717         my $cookie = { netcount => 0 };
6718         while (defined(my $line = <$fh>)) {
6719             $new_conf_raw .= restore_update_config_line(
6720                 $cookie,
6721                 $map,
6722                 $line,
6723                 $options->{unique},
6724             );
6725         }
6726
6727         $fh->close();
6728     };
6729     my $err = $@;
6730
6731     if ($err || !$options->{live}) {
6732         $restore_deactivate_volumes->($storecfg, $devinfo);
6733     }
6734
6735     rmtree $tmpdir;
6736
6737     if ($err) {
6738         $restore_destroy_volumes->($storecfg, $devinfo);
6739         die $err;
6740     }
6741
6742     if ($options->{live}) {
6743         # keep lock during live-restore
6744         $new_conf_raw .= "\nlock: create";
6745     }
6746
6747     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6748
6749     PVE::Cluster::cfs_update(); # make sure we read new file
6750
6751     eval { rescan($vmid, 1); };
6752     warn $@ if $@;
6753
6754     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6755
6756     if ($options->{live}) {
6757         # enable interrupts
6758         local $SIG{INT} =
6759             local $SIG{TERM} =
6760             local $SIG{QUIT} =
6761             local $SIG{HUP} =
6762             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6763
6764         my $conf = PVE::QemuConfig->load_config($vmid);
6765         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6766
6767         # these special drives are already restored before start
6768         delete $devinfo->{'drive-efidisk0'};
6769         delete $devinfo->{'drive-tpmstate0-backup'};
6770         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6771
6772         PVE::QemuConfig->remove_lock($vmid, "create");
6773     }
6774 }
6775
6776 sub pbs_live_restore {
6777     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6778
6779     print "starting VM for live-restore\n";
6780     print "repository: '$repo', snapshot: '$snap'\n";
6781
6782     my $pbs_backing = {};
6783     for my $ds (keys %$restored_disks) {
6784         $ds =~ m/^drive-(.*)$/;
6785         my $confname = $1;
6786         $pbs_backing->{$confname} = {
6787             repository => $repo,
6788             snapshot => $snap,
6789             archive => "$ds.img.fidx",
6790         };
6791         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6792
6793         my $drive = parse_drive($confname, $conf->{$confname});
6794         print "restoring '$ds' to '$drive->{file}'\n";
6795     }
6796
6797     my $drives_streamed = 0;
6798     eval {
6799         # make sure HA doesn't interrupt our restore by stopping the VM
6800         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6801             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6802         }
6803
6804         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6805         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6806         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6807
6808         my $qmeventd_fd = register_qmeventd_handle($vmid);
6809
6810         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6811         # this will effectively collapse the backing image chain consisting of
6812         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6813         # removes itself once all backing images vanish with 'auto-remove=on')
6814         my $jobs = {};
6815         for my $ds (sort keys %$restored_disks) {
6816             my $job_id = "restore-$ds";
6817             mon_cmd($vmid, 'block-stream',
6818                 'job-id' => $job_id,
6819                 device => "$ds",
6820             );
6821             $jobs->{$job_id} = {};
6822         }
6823
6824         mon_cmd($vmid, 'cont');
6825         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6826
6827         print "restore-drive jobs finished successfully, removing all tracking block devices"
6828             ." to disconnect from Proxmox Backup Server\n";
6829
6830         for my $ds (sort keys %$restored_disks) {
6831             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6832         }
6833
6834         close($qmeventd_fd);
6835     };
6836
6837     my $err = $@;
6838
6839     if ($err) {
6840         warn "An error occurred during live-restore: $err\n";
6841         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6842         die "live-restore failed\n";
6843     }
6844 }
6845
6846 sub restore_vma_archive {
6847     my ($archive, $vmid, $user, $opts, $comp) = @_;
6848
6849     my $readfrom = $archive;
6850
6851     my $cfg = PVE::Storage::config();
6852     my $commands = [];
6853     my $bwlimit = $opts->{bwlimit};
6854
6855     my $dbg_cmdstring = '';
6856     my $add_pipe = sub {
6857         my ($cmd) = @_;
6858         push @$commands, $cmd;
6859         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6860         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6861         $readfrom = '-';
6862     };
6863
6864     my $input = undef;
6865     if ($archive eq '-') {
6866         $input = '<&STDIN';
6867     } else {
6868         # If we use a backup from a PVE defined storage we also consider that
6869         # storage's rate limit:
6870         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6871         if (defined($volid)) {
6872             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6873             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6874             if ($readlimit) {
6875                 print STDERR "applying read rate limit: $readlimit\n";
6876                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6877                 $add_pipe->($cstream);
6878             }
6879         }
6880     }
6881
6882     if ($comp) {
6883         my $info = PVE::Storage::decompressor_info('vma', $comp);
6884         my $cmd = $info->{decompressor};
6885         push @$cmd, $readfrom;
6886         $add_pipe->($cmd);
6887     }
6888
6889     my $tmpdir = "/var/tmp/vzdumptmp$$";
6890     rmtree $tmpdir;
6891
6892     # disable interrupts (always do cleanups)
6893     local $SIG{INT} =
6894         local $SIG{TERM} =
6895         local $SIG{QUIT} =
6896         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6897
6898     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6899     POSIX::mkfifo($mapfifo, 0600);
6900     my $fifofh;
6901     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6902
6903     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6904
6905     my $oldtimeout;
6906     my $timeout = 5;
6907
6908     my $devinfo = {};
6909
6910     my $rpcenv = PVE::RPCEnvironment::get();
6911
6912     my $conffile = PVE::QemuConfig->config_file($vmid);
6913
6914     # Note: $oldconf is undef if VM does not exist
6915     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6916     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6917     my $new_conf_raw = '';
6918
6919     my %storage_limits;
6920
6921     my $print_devmap = sub {
6922         my $cfgfn = "$tmpdir/qemu-server.conf";
6923
6924         # we can read the config - that is already extracted
6925         my $fh = IO::File->new($cfgfn, "r") ||
6926             die "unable to read qemu-server.conf - $!\n";
6927
6928         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6929         if (-f $fwcfgfn) {
6930             my $pve_firewall_dir = '/etc/pve/firewall';
6931             mkdir $pve_firewall_dir; # make sure the dir exists
6932             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6933         }
6934
6935         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6936
6937         foreach my $info (values %{$virtdev_hash}) {
6938             my $storeid = $info->{storeid};
6939             next if defined($storage_limits{$storeid});
6940
6941             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6942             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6943             $storage_limits{$storeid} = $limit * 1024;
6944         }
6945
6946         foreach my $devname (keys %$devinfo) {
6947             die "found no device mapping information for device '$devname'\n"
6948                 if !$devinfo->{$devname}->{virtdev};
6949         }
6950
6951         # create empty/temp config
6952         if ($oldconf) {
6953             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6954             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6955         }
6956
6957         # allocate volumes
6958         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6959
6960         # print restore information to $fifofh
6961         foreach my $virtdev (sort keys %$virtdev_hash) {
6962             my $d = $virtdev_hash->{$virtdev};
6963             next if $d->{is_cloudinit}; # no need to restore cloudinit
6964
6965             my $storeid = $d->{storeid};
6966             my $volid = $d->{volid};
6967
6968             my $map_opts = '';
6969             if (my $limit = $storage_limits{$storeid}) {
6970                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6971             }
6972
6973             my $write_zeros = 1;
6974             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6975                 $write_zeros = 0;
6976             }
6977
6978             my $path = PVE::Storage::path($cfg, $volid);
6979
6980             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6981
6982             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6983         }
6984
6985         $fh->seek(0, 0) || die "seek failed - $!\n";
6986
6987         my $cookie = { netcount => 0 };
6988         while (defined(my $line = <$fh>)) {
6989             $new_conf_raw .= restore_update_config_line(
6990                 $cookie,
6991                 $map,
6992                 $line,
6993                 $opts->{unique},
6994             );
6995         }
6996
6997         $fh->close();
6998     };
6999
7000     eval {
7001         # enable interrupts
7002         local $SIG{INT} =
7003             local $SIG{TERM} =
7004             local $SIG{QUIT} =
7005             local $SIG{HUP} =
7006             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7007         local $SIG{ALRM} = sub { die "got timeout\n"; };
7008
7009         $oldtimeout = alarm($timeout);
7010
7011         my $parser = sub {
7012             my $line = shift;
7013
7014             print "$line\n";
7015
7016             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7017                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7018                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7019             } elsif ($line =~ m/^CTIME: /) {
7020                 # we correctly received the vma config, so we can disable
7021                 # the timeout now for disk allocation (set to 10 minutes, so
7022                 # that we always timeout if something goes wrong)
7023                 alarm(600);
7024                 &$print_devmap();
7025                 print $fifofh "done\n";
7026                 my $tmp = $oldtimeout || 0;
7027                 $oldtimeout = undef;
7028                 alarm($tmp);
7029                 close($fifofh);
7030                 $fifofh = undef;
7031             }
7032         };
7033
7034         print "restore vma archive: $dbg_cmdstring\n";
7035         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7036     };
7037     my $err = $@;
7038
7039     alarm($oldtimeout) if $oldtimeout;
7040
7041     $restore_deactivate_volumes->($cfg, $devinfo);
7042
7043     close($fifofh) if $fifofh;
7044     unlink $mapfifo;
7045     rmtree $tmpdir;
7046
7047     if ($err) {
7048         $restore_destroy_volumes->($cfg, $devinfo);
7049         die $err;
7050     }
7051
7052     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7053
7054     PVE::Cluster::cfs_update(); # make sure we read new file
7055
7056     eval { rescan($vmid, 1); };
7057     warn $@ if $@;
7058
7059     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7060 }
7061
7062 sub restore_tar_archive {
7063     my ($archive, $vmid, $user, $opts) = @_;
7064
7065     if ($archive ne '-') {
7066         my $firstfile = tar_archive_read_firstfile($archive);
7067         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7068             if $firstfile ne 'qemu-server.conf';
7069     }
7070
7071     my $storecfg = PVE::Storage::config();
7072
7073     # avoid zombie disks when restoring over an existing VM -> cleanup first
7074     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7075     # skiplock=1 because qmrestore has set the 'create' lock itself already
7076     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7077     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7078
7079     my $tocmd = "/usr/lib/qemu-server/qmextract";
7080
7081     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7082     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7083     $tocmd .= ' --prealloc' if $opts->{prealloc};
7084     $tocmd .= ' --info' if $opts->{info};
7085
7086     # tar option "xf" does not autodetect compression when read from STDIN,
7087     # so we pipe to zcat
7088     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7089         PVE::Tools::shellquote("--to-command=$tocmd");
7090
7091     my $tmpdir = "/var/tmp/vzdumptmp$$";
7092     mkpath $tmpdir;
7093
7094     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7095     local $ENV{VZDUMP_VMID} = $vmid;
7096     local $ENV{VZDUMP_USER} = $user;
7097
7098     my $conffile = PVE::QemuConfig->config_file($vmid);
7099     my $new_conf_raw = '';
7100
7101     # disable interrupts (always do cleanups)
7102     local $SIG{INT} =
7103         local $SIG{TERM} =
7104         local $SIG{QUIT} =
7105         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7106
7107     eval {
7108         # enable interrupts
7109         local $SIG{INT} =
7110             local $SIG{TERM} =
7111             local $SIG{QUIT} =
7112             local $SIG{HUP} =
7113             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7114
7115         if ($archive eq '-') {
7116             print "extracting archive from STDIN\n";
7117             run_command($cmd, input => "<&STDIN");
7118         } else {
7119             print "extracting archive '$archive'\n";
7120             run_command($cmd);
7121         }
7122
7123         return if $opts->{info};
7124
7125         # read new mapping
7126         my $map = {};
7127         my $statfile = "$tmpdir/qmrestore.stat";
7128         if (my $fd = IO::File->new($statfile, "r")) {
7129             while (defined (my $line = <$fd>)) {
7130                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7131                     $map->{$1} = $2 if $1;
7132                 } else {
7133                     print STDERR "unable to parse line in statfile - $line\n";
7134                 }
7135             }
7136             $fd->close();
7137         }
7138
7139         my $confsrc = "$tmpdir/qemu-server.conf";
7140
7141         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7142
7143         my $cookie = { netcount => 0 };
7144         while (defined (my $line = <$srcfd>)) {
7145             $new_conf_raw .= restore_update_config_line(
7146                 $cookie,
7147                 $map,
7148                 $line,
7149                 $opts->{unique},
7150             );
7151         }
7152
7153         $srcfd->close();
7154     };
7155     if (my $err = $@) {
7156         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7157         die $err;
7158     }
7159
7160     rmtree $tmpdir;
7161
7162     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7163
7164     PVE::Cluster::cfs_update(); # make sure we read new file
7165
7166     eval { rescan($vmid, 1); };
7167     warn $@ if $@;
7168 };
7169
7170 sub foreach_storage_used_by_vm {
7171     my ($conf, $func) = @_;
7172
7173     my $sidhash = {};
7174
7175     PVE::QemuConfig->foreach_volume($conf, sub {
7176         my ($ds, $drive) = @_;
7177         return if drive_is_cdrom($drive);
7178
7179         my $volid = $drive->{file};
7180
7181         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7182         $sidhash->{$sid} = $sid if $sid;
7183     });
7184
7185     foreach my $sid (sort keys %$sidhash) {
7186         &$func($sid);
7187     }
7188 }
7189
7190 my $qemu_snap_storage = {
7191     rbd => 1,
7192 };
7193 sub do_snapshots_with_qemu {
7194     my ($storecfg, $volid, $deviceid) = @_;
7195
7196     return if $deviceid =~ m/tpmstate0/;
7197
7198     my $storage_name = PVE::Storage::parse_volume_id($volid);
7199     my $scfg = $storecfg->{ids}->{$storage_name};
7200     die "could not find storage '$storage_name'\n" if !defined($scfg);
7201
7202     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7203         return 1;
7204     }
7205
7206     if ($volid =~ m/\.(qcow2|qed)$/){
7207         return 1;
7208     }
7209
7210     return;
7211 }
7212
7213 sub qga_check_running {
7214     my ($vmid, $nowarn) = @_;
7215
7216     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7217     if ($@) {
7218         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7219         return 0;
7220     }
7221     return 1;
7222 }
7223
7224 sub template_create {
7225     my ($vmid, $conf, $disk) = @_;
7226
7227     my $storecfg = PVE::Storage::config();
7228
7229     PVE::QemuConfig->foreach_volume($conf, sub {
7230         my ($ds, $drive) = @_;
7231
7232         return if drive_is_cdrom($drive);
7233         return if $disk && $ds ne $disk;
7234
7235         my $volid = $drive->{file};
7236         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7237
7238         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7239         $drive->{file} = $voliddst;
7240         $conf->{$ds} = print_drive($drive);
7241         PVE::QemuConfig->write_config($vmid, $conf);
7242     });
7243 }
7244
7245 sub convert_iscsi_path {
7246     my ($path) = @_;
7247
7248     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7249         my $portal = $1;
7250         my $target = $2;
7251         my $lun = $3;
7252
7253         my $initiator_name = get_initiator_name();
7254
7255         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7256                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7257     }
7258
7259     die "cannot convert iscsi path '$path', unkown format\n";
7260 }
7261
7262 sub qemu_img_convert {
7263     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7264
7265     my $storecfg = PVE::Storage::config();
7266     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7267     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7268
7269     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7270
7271     my $cachemode;
7272     my $src_path;
7273     my $src_is_iscsi = 0;
7274     my $src_format;
7275
7276     if ($src_storeid) {
7277         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7278         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7279         $src_format = qemu_img_format($src_scfg, $src_volname);
7280         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7281         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7282         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7283     } elsif (-f $src_volid || -b $src_volid) {
7284         $src_path = $src_volid;
7285         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7286             $src_format = $1;
7287         }
7288     }
7289
7290     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7291
7292     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7293     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7294     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7295     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7296
7297     my $cmd = [];
7298     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7299     push @$cmd, '-l', "snapshot.name=$snapname"
7300         if $snapname && $src_format && $src_format eq "qcow2";
7301     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7302     push @$cmd, '-T', $cachemode if defined($cachemode);
7303
7304     if ($src_is_iscsi) {
7305         push @$cmd, '--image-opts';
7306         $src_path = convert_iscsi_path($src_path);
7307     } elsif ($src_format) {
7308         push @$cmd, '-f', $src_format;
7309     }
7310
7311     if ($dst_is_iscsi) {
7312         push @$cmd, '--target-image-opts';
7313         $dst_path = convert_iscsi_path($dst_path);
7314     } else {
7315         push @$cmd, '-O', $dst_format;
7316     }
7317
7318     push @$cmd, $src_path;
7319
7320     if (!$dst_is_iscsi && $is_zero_initialized) {
7321         push @$cmd, "zeroinit:$dst_path";
7322     } else {
7323         push @$cmd, $dst_path;
7324     }
7325
7326     my $parser = sub {
7327         my $line = shift;
7328         if($line =~ m/\((\S+)\/100\%\)/){
7329             my $percent = $1;
7330             my $transferred = int($size * $percent / 100);
7331             my $total_h = render_bytes($size, 1);
7332             my $transferred_h = render_bytes($transferred, 1);
7333
7334             print "transferred $transferred_h of $total_h ($percent%)\n";
7335         }
7336
7337     };
7338
7339     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7340     my $err = $@;
7341     die "copy failed: $err" if $err;
7342 }
7343
7344 sub qemu_img_format {
7345     my ($scfg, $volname) = @_;
7346
7347     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7348         return $1;
7349     } else {
7350         return "raw";
7351     }
7352 }
7353
7354 sub qemu_drive_mirror {
7355     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7356
7357     $jobs = {} if !$jobs;
7358
7359     my $qemu_target;
7360     my $format;
7361     $jobs->{"drive-$drive"} = {};
7362
7363     if ($dst_volid =~ /^nbd:/) {
7364         $qemu_target = $dst_volid;
7365         $format = "nbd";
7366     } else {
7367         my $storecfg = PVE::Storage::config();
7368         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7369
7370         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7371
7372         $format = qemu_img_format($dst_scfg, $dst_volname);
7373
7374         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7375
7376         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7377     }
7378
7379     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7380     $opts->{format} = $format if $format;
7381
7382     if (defined($src_bitmap)) {
7383         $opts->{sync} = 'incremental';
7384         $opts->{bitmap} = $src_bitmap;
7385         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7386     }
7387
7388     if (defined($bwlimit)) {
7389         $opts->{speed} = $bwlimit * 1024;
7390         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7391     } else {
7392         print "drive mirror is starting for drive-$drive\n";
7393     }
7394
7395     # if a job already runs for this device we get an error, catch it for cleanup
7396     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7397     if (my $err = $@) {
7398         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7399         warn "$@\n" if $@;
7400         die "mirroring error: $err\n";
7401     }
7402
7403     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7404 }
7405
7406 # $completion can be either
7407 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7408 # 'cancel': wait until all jobs are ready, block-job-cancel them
7409 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7410 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7411 sub qemu_drive_mirror_monitor {
7412     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7413
7414     $completion //= 'complete';
7415     $op //= "mirror";
7416
7417     eval {
7418         my $err_complete = 0;
7419
7420         my $starttime = time ();
7421         while (1) {
7422             die "block job ('$op') timed out\n" if $err_complete > 300;
7423
7424             my $stats = mon_cmd($vmid, "query-block-jobs");
7425             my $ctime = time();
7426
7427             my $running_jobs = {};
7428             for my $stat (@$stats) {
7429                 next if $stat->{type} ne $op;
7430                 $running_jobs->{$stat->{device}} = $stat;
7431             }
7432
7433             my $readycounter = 0;
7434
7435             for my $job_id (sort keys %$jobs) {
7436                 my $job = $running_jobs->{$job_id};
7437
7438                 my $vanished = !defined($job);
7439                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7440                 if($complete || ($vanished && $completion eq 'auto')) {
7441                     print "$job_id: $op-job finished\n";
7442                     delete $jobs->{$job_id};
7443                     next;
7444                 }
7445
7446                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7447
7448                 my $busy = $job->{busy};
7449                 my $ready = $job->{ready};
7450                 if (my $total = $job->{len}) {
7451                     my $transferred = $job->{offset} || 0;
7452                     my $remaining = $total - $transferred;
7453                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7454
7455                     my $duration = $ctime - $starttime;
7456                     my $total_h = render_bytes($total, 1);
7457                     my $transferred_h = render_bytes($transferred, 1);
7458
7459                     my $status = sprintf(
7460                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7461                         render_duration($duration),
7462                     );
7463
7464                     if ($ready) {
7465                         if ($busy) {
7466                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7467                         } else {
7468                             $status .= ", ready";
7469                         }
7470                     }
7471                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7472                     $jobs->{$job_id}->{ready} = $ready;
7473                 }
7474
7475                 $readycounter++ if $job->{ready};
7476             }
7477
7478             last if scalar(keys %$jobs) == 0;
7479
7480             if ($readycounter == scalar(keys %$jobs)) {
7481                 print "all '$op' jobs are ready\n";
7482
7483                 # do the complete later (or has already been done)
7484                 last if $completion eq 'skip' || $completion eq 'auto';
7485
7486                 if ($vmiddst && $vmiddst != $vmid) {
7487                     my $agent_running = $qga && qga_check_running($vmid);
7488                     if ($agent_running) {
7489                         print "freeze filesystem\n";
7490                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7491                         warn $@ if $@;
7492                     } else {
7493                         print "suspend vm\n";
7494                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7495                         warn $@ if $@;
7496                     }
7497
7498                     # if we clone a disk for a new target vm, we don't switch the disk
7499                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7500
7501                     if ($agent_running) {
7502                         print "unfreeze filesystem\n";
7503                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7504                         warn $@ if $@;
7505                     } else {
7506                         print "resume vm\n";
7507                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7508                         warn $@ if $@;
7509                     }
7510
7511                     last;
7512                 } else {
7513
7514                     for my $job_id (sort keys %$jobs) {
7515                         # try to switch the disk if source and destination are on the same guest
7516                         print "$job_id: Completing block job_id...\n";
7517
7518                         my $op;
7519                         if ($completion eq 'complete') {
7520                             $op = 'block-job-complete';
7521                         } elsif ($completion eq 'cancel') {
7522                             $op = 'block-job-cancel';
7523                         } else {
7524                             die "invalid completion value: $completion\n";
7525                         }
7526                         eval { mon_cmd($vmid, $op, device => $job_id) };
7527                         if ($@ =~ m/cannot be completed/) {
7528                             print "$job_id: block job cannot be completed, trying again.\n";
7529                             $err_complete++;
7530                         }else {
7531                             print "$job_id: Completed successfully.\n";
7532                             $jobs->{$job_id}->{complete} = 1;
7533                         }
7534                     }
7535                 }
7536             }
7537             sleep 1;
7538         }
7539     };
7540     my $err = $@;
7541
7542     if ($err) {
7543         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7544         die "block job ($op) error: $err";
7545     }
7546 }
7547
7548 sub qemu_blockjobs_cancel {
7549     my ($vmid, $jobs) = @_;
7550
7551     foreach my $job (keys %$jobs) {
7552         print "$job: Cancelling block job\n";
7553         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7554         $jobs->{$job}->{cancel} = 1;
7555     }
7556
7557     while (1) {
7558         my $stats = mon_cmd($vmid, "query-block-jobs");
7559
7560         my $running_jobs = {};
7561         foreach my $stat (@$stats) {
7562             $running_jobs->{$stat->{device}} = $stat;
7563         }
7564
7565         foreach my $job (keys %$jobs) {
7566
7567             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7568                 print "$job: Done.\n";
7569                 delete $jobs->{$job};
7570             }
7571         }
7572
7573         last if scalar(keys %$jobs) == 0;
7574
7575         sleep 1;
7576     }
7577 }
7578
7579 sub clone_disk {
7580     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7581
7582     my ($vmid, $running) = $source->@{qw(vmid running)};
7583     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7584
7585     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7586     my ($storage, $format) = $dest->@{qw(storage format)};
7587
7588     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7589
7590     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7591         die "cloning from/to EFI disk requires EFI disk\n"
7592             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7593         die "cloning from/to TPM state requires TPM state\n"
7594             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7595
7596         # This would lead to two device nodes in QEMU pointing to the same backing image!
7597         die "cannot change drive name when cloning disk from/to the same VM\n"
7598             if $use_drive_mirror && $vmid == $newvmid;
7599     }
7600
7601     die "cannot move TPM state while VM is running\n"
7602         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7603
7604     my $newvolid;
7605
7606     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7607     print "$src_drivename " if $src_drivename;
7608     print "($drive->{file})\n";
7609
7610     if (!$full) {
7611         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7612         push @$newvollist, $newvolid;
7613     } else {
7614
7615         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7616         $storeid = $storage if $storage;
7617
7618         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7619
7620         my $name = undef;
7621         my $size = undef;
7622         if (drive_is_cloudinit($drive)) {
7623             $name = "vm-$newvmid-cloudinit";
7624             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7625             if ($scfg->{path}) {
7626                 $name .= ".$dst_format";
7627             }
7628             $snapname = undef;
7629             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7630         } elsif ($dst_drivename eq 'efidisk0') {
7631             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7632         } elsif ($dst_drivename eq 'tpmstate0') {
7633             $dst_format = 'raw';
7634             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7635         } else {
7636             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7637         }
7638         $newvolid = PVE::Storage::vdisk_alloc(
7639             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7640         );
7641         push @$newvollist, $newvolid;
7642
7643         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7644
7645         if (drive_is_cloudinit($drive)) {
7646             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7647             # if this is the case, we have to complete any block-jobs still there from
7648             # previous drive-mirrors
7649             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7650                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7651             }
7652             goto no_data_clone;
7653         }
7654
7655         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7656         if ($use_drive_mirror) {
7657             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7658                 $completion, $qga, $bwlimit);
7659         } else {
7660             # TODO: handle bwlimits
7661             if ($dst_drivename eq 'efidisk0') {
7662                 # the relevant data on the efidisk may be smaller than the source
7663                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7664                 # that is given by the OVMF_VARS.fd
7665                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7666                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7667
7668                 # better for Ceph if block size is not too small, see bug #3324
7669                 my $bs = 1024*1024;
7670
7671                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7672                     "if=$src_path", "of=$dst_path"]);
7673             } else {
7674                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7675             }
7676         }
7677     }
7678
7679 no_data_clone:
7680     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7681
7682     my $disk = dclone($drive);
7683     delete $disk->{format};
7684     $disk->{file} = $newvolid;
7685     $disk->{size} = $size if defined($size);
7686
7687     return $disk;
7688 }
7689
7690 sub get_running_qemu_version {
7691     my ($vmid) = @_;
7692     my $res = mon_cmd($vmid, "query-version");
7693     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7694 }
7695
7696 sub qemu_use_old_bios_files {
7697     my ($machine_type) = @_;
7698
7699     return if !$machine_type;
7700
7701     my $use_old_bios_files = undef;
7702
7703     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7704         $machine_type = $1;
7705         $use_old_bios_files = 1;
7706     } else {
7707         my $version = extract_version($machine_type, kvm_user_version());
7708         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7709         # load new efi bios files on migration. So this hack is required to allow
7710         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7711         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7712         $use_old_bios_files = !min_version($version, 2, 4);
7713     }
7714
7715     return ($use_old_bios_files, $machine_type);
7716 }
7717
7718 sub get_efivars_size {
7719     my ($conf, $efidisk) = @_;
7720
7721     my $arch = get_vm_arch($conf);
7722     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7723     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7724     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7725     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7726     return -s $ovmf_vars;
7727 }
7728
7729 sub update_efidisk_size {
7730     my ($conf) = @_;
7731
7732     return if !defined($conf->{efidisk0});
7733
7734     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7735     $disk->{size} = get_efivars_size($conf);
7736     $conf->{efidisk0} = print_drive($disk);
7737
7738     return;
7739 }
7740
7741 sub update_tpmstate_size {
7742     my ($conf) = @_;
7743
7744     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7745     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7746     $conf->{tpmstate0} = print_drive($disk);
7747 }
7748
7749 sub create_efidisk($$$$$$$) {
7750     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7751
7752     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7753     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7754
7755     my $vars_size_b = -s $ovmf_vars;
7756     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7757     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7758     PVE::Storage::activate_volumes($storecfg, [$volid]);
7759
7760     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7761     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7762
7763     return ($volid, $size/1024);
7764 }
7765
7766 sub vm_iothreads_list {
7767     my ($vmid) = @_;
7768
7769     my $res = mon_cmd($vmid, 'query-iothreads');
7770
7771     my $iothreads = {};
7772     foreach my $iothread (@$res) {
7773         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7774     }
7775
7776     return $iothreads;
7777 }
7778
7779 sub scsihw_infos {
7780     my ($conf, $drive) = @_;
7781
7782     my $maxdev = 0;
7783
7784     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7785         $maxdev = 7;
7786     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7787         $maxdev = 1;
7788     } else {
7789         $maxdev = 256;
7790     }
7791
7792     my $controller = int($drive->{index} / $maxdev);
7793     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7794         ? "virtioscsi"
7795         : "scsihw";
7796
7797     return ($maxdev, $controller, $controller_prefix);
7798 }
7799
7800 sub windows_version {
7801     my ($ostype) = @_;
7802
7803     return 0 if !$ostype;
7804
7805     my $winversion = 0;
7806
7807     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7808         $winversion = 5;
7809     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7810         $winversion = 6;
7811     } elsif ($ostype =~ m/^win(\d+)$/) {
7812         $winversion = $1;
7813     }
7814
7815     return $winversion;
7816 }
7817
7818 sub resolve_dst_disk_format {
7819         my ($storecfg, $storeid, $src_volname, $format) = @_;
7820         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7821
7822         if (!$format) {
7823             # if no target format is specified, use the source disk format as hint
7824             if ($src_volname) {
7825                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7826                 $format = qemu_img_format($scfg, $src_volname);
7827             } else {
7828                 return $defFormat;
7829             }
7830         }
7831
7832         # test if requested format is supported - else use default
7833         my $supported = grep { $_ eq $format } @$validFormats;
7834         $format = $defFormat if !$supported;
7835         return $format;
7836 }
7837
7838 # NOTE: if this logic changes, please update docs & possibly gui logic
7839 sub find_vmstate_storage {
7840     my ($conf, $storecfg) = @_;
7841
7842     # first, return storage from conf if set
7843     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7844
7845     my ($target, $shared, $local);
7846
7847     foreach_storage_used_by_vm($conf, sub {
7848         my ($sid) = @_;
7849         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7850         my $dst = $scfg->{shared} ? \$shared : \$local;
7851         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7852     });
7853
7854     # second, use shared storage where VM has at least one disk
7855     # third, use local storage where VM has at least one disk
7856     # fall back to local storage
7857     $target = $shared // $local // 'local';
7858
7859     return $target;
7860 }
7861
7862 sub generate_uuid {
7863     my ($uuid, $uuid_str);
7864     UUID::generate($uuid);
7865     UUID::unparse($uuid, $uuid_str);
7866     return $uuid_str;
7867 }
7868
7869 sub generate_smbios1_uuid {
7870     return "uuid=".generate_uuid();
7871 }
7872
7873 sub nbd_stop {
7874     my ($vmid) = @_;
7875
7876     mon_cmd($vmid, 'nbd-server-stop');
7877 }
7878
7879 sub create_reboot_request {
7880     my ($vmid) = @_;
7881     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7882         or die "failed to create reboot trigger file: $!\n";
7883     close($fh);
7884 }
7885
7886 sub clear_reboot_request {
7887     my ($vmid) = @_;
7888     my $path = "/run/qemu-server/$vmid.reboot";
7889     my $res = 0;
7890
7891     $res = unlink($path);
7892     die "could not remove reboot request for $vmid: $!"
7893         if !$res && $! != POSIX::ENOENT;
7894
7895     return $res;
7896 }
7897
7898 sub bootorder_from_legacy {
7899     my ($conf, $bootcfg) = @_;
7900
7901     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7902     my $bootindex_hash = {};
7903     my $i = 1;
7904     foreach my $o (split(//, $boot)) {
7905         $bootindex_hash->{$o} = $i*100;
7906         $i++;
7907     }
7908
7909     my $bootorder = {};
7910
7911     PVE::QemuConfig->foreach_volume($conf, sub {
7912         my ($ds, $drive) = @_;
7913
7914         if (drive_is_cdrom ($drive, 1)) {
7915             if ($bootindex_hash->{d}) {
7916                 $bootorder->{$ds} = $bootindex_hash->{d};
7917                 $bootindex_hash->{d} += 1;
7918             }
7919         } elsif ($bootindex_hash->{c}) {
7920             $bootorder->{$ds} = $bootindex_hash->{c}
7921                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7922             $bootindex_hash->{c} += 1;
7923         }
7924     });
7925
7926     if ($bootindex_hash->{n}) {
7927         for (my $i = 0; $i < $MAX_NETS; $i++) {
7928             my $netname = "net$i";
7929             next if !$conf->{$netname};
7930             $bootorder->{$netname} = $bootindex_hash->{n};
7931             $bootindex_hash->{n} += 1;
7932         }
7933     }
7934
7935     return $bootorder;
7936 }
7937
7938 # Generate default device list for 'boot: order=' property. Matches legacy
7939 # default boot order, but with explicit device names. This is important, since
7940 # the fallback for when neither 'order' nor the old format is specified relies
7941 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7942 sub get_default_bootdevices {
7943     my ($conf) = @_;
7944
7945     my @ret = ();
7946
7947     # harddisk
7948     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7949     push @ret, $first if $first;
7950
7951     # cdrom
7952     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7953     push @ret, $first if $first;
7954
7955     # network
7956     for (my $i = 0; $i < $MAX_NETS; $i++) {
7957         my $netname = "net$i";
7958         next if !$conf->{$netname};
7959         push @ret, $netname;
7960         last;
7961     }
7962
7963     return \@ret;
7964 }
7965
7966 sub device_bootorder {
7967     my ($conf) = @_;
7968
7969     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7970
7971     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7972
7973     my $bootorder = {};
7974     if (!defined($boot) || $boot->{legacy}) {
7975         $bootorder = bootorder_from_legacy($conf, $boot);
7976     } elsif ($boot->{order}) {
7977         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7978         for my $dev (PVE::Tools::split_list($boot->{order})) {
7979             $bootorder->{$dev} = $i++;
7980         }
7981     }
7982
7983     return $bootorder;
7984 }
7985
7986 sub register_qmeventd_handle {
7987     my ($vmid) = @_;
7988
7989     my $fh;
7990     my $peer = "/var/run/qmeventd.sock";
7991     my $count = 0;
7992
7993     for (;;) {
7994         $count++;
7995         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7996         last if $fh;
7997         if ($! != EINTR && $! != EAGAIN) {
7998             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7999         }
8000         if ($count > 4) {
8001             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8002               . "after $count retries\n";
8003         }
8004         usleep(25000);
8005     }
8006
8007     # send handshake to mark VM as backing up
8008     print $fh to_json({vzdump => {vmid => "$vmid"}});
8009
8010     # return handle to be closed later when inhibit is no longer required
8011     return $fh;
8012 }
8013
8014 # bash completion helper
8015
8016 sub complete_backup_archives {
8017     my ($cmdname, $pname, $cvalue) = @_;
8018
8019     my $cfg = PVE::Storage::config();
8020
8021     my $storeid;
8022
8023     if ($cvalue =~ m/^([^:]+):/) {
8024         $storeid = $1;
8025     }
8026
8027     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8028
8029     my $res = [];
8030     foreach my $id (keys %$data) {
8031         foreach my $item (@{$data->{$id}}) {
8032             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8033             push @$res, $item->{volid} if defined($item->{volid});
8034         }
8035     }
8036
8037     return $res;
8038 }
8039
8040 my $complete_vmid_full = sub {
8041     my ($running) = @_;
8042
8043     my $idlist = vmstatus();
8044
8045     my $res = [];
8046
8047     foreach my $id (keys %$idlist) {
8048         my $d = $idlist->{$id};
8049         if (defined($running)) {
8050             next if $d->{template};
8051             next if $running && $d->{status} ne 'running';
8052             next if !$running && $d->{status} eq 'running';
8053         }
8054         push @$res, $id;
8055
8056     }
8057     return $res;
8058 };
8059
8060 sub complete_vmid {
8061     return &$complete_vmid_full();
8062 }
8063
8064 sub complete_vmid_stopped {
8065     return &$complete_vmid_full(0);
8066 }
8067
8068 sub complete_vmid_running {
8069     return &$complete_vmid_full(1);
8070 }
8071
8072 sub complete_storage {
8073
8074     my $cfg = PVE::Storage::config();
8075     my $ids = $cfg->{ids};
8076
8077     my $res = [];
8078     foreach my $sid (keys %$ids) {
8079         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8080         next if !$ids->{$sid}->{content}->{images};
8081         push @$res, $sid;
8082     }
8083
8084     return $res;
8085 }
8086
8087 sub complete_migration_storage {
8088     my ($cmd, $param, $current_value, $all_args) = @_;
8089
8090     my $targetnode = @$all_args[1];
8091
8092     my $cfg = PVE::Storage::config();
8093     my $ids = $cfg->{ids};
8094
8095     my $res = [];
8096     foreach my $sid (keys %$ids) {
8097         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8098         next if !$ids->{$sid}->{content}->{images};
8099         push @$res, $sid;
8100     }
8101
8102     return $res;
8103 }
8104
8105 sub vm_is_paused {
8106     my ($vmid) = @_;
8107     my $qmpstatus = eval {
8108         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8109         mon_cmd($vmid, "query-status");
8110     };
8111     warn "$@\n" if $@;
8112     return $qmpstatus && $qmpstatus->{status} eq "paused";
8113 }
8114
8115 sub check_volume_storage_type {
8116     my ($storecfg, $vol) = @_;
8117
8118     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8119     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8120     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8121
8122     die "storage '$storeid' does not support content-type '$vtype'\n"
8123         if !$scfg->{content}->{$vtype};
8124
8125     return 1;
8126 }
8127
8128 1;