PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. This option is generally not required and"
 371         ." is often better handled from within the guest OS.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => "cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.",
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.",
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1040
1041     return verify_volume_id_or_absolute_path($volid, $noerr);
1042 }
1043
1044 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1045 sub verify_volume_id_or_absolute_path {
1046     my ($volid, $noerr) = @_;
1047
1048     return $volid if $volid =~ m|^/|;
1049
1050     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1051     if ($@) {
1052         return if $noerr;
1053         die $@;
1054     }
1055     return $volid;
1056 }
1057
1058 my $usb_fmt = {
1059     host => {
1060         default_key => 1,
1061         type => 'string', format => 'pve-qm-usb-device',
1062         format_description => 'HOSTUSBDEVICE|spice',
1063         description => <<EODESCR,
1064 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1065
1066  'bus-port(.port)*' (decimal numbers) or
1067  'vendor_id:product_id' (hexadeciaml numbers) or
1068  'spice'
1069
1070 You can use the 'lsusb -t' command to list existing usb devices.
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1073 machines - use with special care.
1074
1075 The value 'spice' can be used to add a usb redirection devices for spice.
1076 EODESCR
1077     },
1078     usb3 => {
1079         optional => 1,
1080         type => 'boolean',
1081         description => "Specifies whether if given host option is a USB3 device or port.",
1082         default => 0,
1083     },
1084 };
1085
1086 my $usbdesc = {
1087     optional => 1,
1088     type => 'string', format => $usb_fmt,
1089     description => "Configure an USB device (n is 0 to 4).",
1090 };
1091 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1092
1093 my $serialdesc = {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '(/dev/.+|socket)',
1097         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1098         verbose_description =>  <<EODESCR,
1099 Create a serial device inside the VM (n is 0 to 3), and pass through a
1100 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1101 host side (use 'qm terminal' to open a terminal connection).
1102
1103 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1104 use with special care.
1105
1106 CAUTION: Experimental! User reported problems with this option.
1107 EODESCR
1108 };
1109
1110 my $paralleldesc= {
1111         optional => 1,
1112         type => 'string',
1113         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1114         description =>  "Map host parallel devices (n is 0 to 2).",
1115         verbose_description =>  <<EODESCR,
1116 Map host parallel devices (n is 0 to 2).
1117
1118 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1119 machines - use with special care.
1120
1121 CAUTION: Experimental! User reported problems with this option.
1122 EODESCR
1123 };
1124
1125 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1126     $confdesc->{"parallel$i"} = $paralleldesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1130     $confdesc->{"serial$i"} = $serialdesc;
1131 }
1132
1133 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1134     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1135 }
1136
1137 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1138     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1139 }
1140
1141 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1142     $confdesc->{"usb$i"} = $usbdesc;
1143 }
1144
1145 my $boot_fmt = {
1146     legacy => {
1147         optional => 1,
1148         default_key => 1,
1149         type => 'string',
1150         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1151                      . " Deprecated, use 'order=' instead.",
1152         pattern => '[acdn]{1,4}',
1153         format_description => "[acdn]{1,4}",
1154
1155         # note: this is also the fallback if boot: is not given at all
1156         default => 'cdn',
1157     },
1158     order => {
1159         optional => 1,
1160         type => 'string',
1161         format => 'pve-qm-bootdev-list',
1162         format_description => "device[;device...]",
1163         description => <<EODESC,
1164 The guest will attempt to boot from devices in the order they appear here.
1165
1166 Disks, optical drives and passed-through storage USB devices will be directly
1167 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1168 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1169
1170 Note that only devices in this list will be marked as bootable and thus loaded
1171 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1172 (e.g. software-raid), you need to specify all of them here.
1173
1174 Overrides the deprecated 'legacy=[acdn]*' value when given.
1175 EODESC
1176     },
1177 };
1178 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1179
1180 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1181 sub verify_bootdev {
1182     my ($dev, $noerr) = @_;
1183
1184     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1185     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1186
1187     my $check = sub {
1188         my ($base) = @_;
1189         return 0 if $dev !~ m/^$base\d+$/;
1190         return 0 if !$confdesc->{$dev};
1191         return 1;
1192     };
1193
1194     return $dev if $check->("net");
1195     return $dev if $check->("usb");
1196     return $dev if $check->("hostpci");
1197
1198     return if $noerr;
1199     die "invalid boot device '$dev'\n";
1200 }
1201
1202 sub print_bootorder {
1203     my ($devs) = @_;
1204     return "" if !@$devs;
1205     my $data = { order => join(';', @$devs) };
1206     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1207 }
1208
1209 my $kvm_api_version = 0;
1210
1211 sub kvm_version {
1212     return $kvm_api_version if $kvm_api_version;
1213
1214     open my $fh, '<', '/dev/kvm' or return;
1215
1216     # 0xae00 => KVM_GET_API_VERSION
1217     $kvm_api_version = ioctl($fh, 0xae00, 0);
1218     close($fh);
1219
1220     return $kvm_api_version;
1221 }
1222
1223 my $kvm_user_version = {};
1224 my $kvm_mtime = {};
1225
1226 sub kvm_user_version {
1227     my ($binary) = @_;
1228
1229     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1230     my $st = stat($binary);
1231
1232     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1233     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1234         $cachedmtime == $st->mtime;
1235
1236     $kvm_user_version->{$binary} = 'unknown';
1237     $kvm_mtime->{$binary} = $st->mtime;
1238
1239     my $code = sub {
1240         my $line = shift;
1241         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1242             $kvm_user_version->{$binary} = $2;
1243         }
1244     };
1245
1246     eval { run_command([$binary, '--version'], outfunc => $code); };
1247     warn $@ if $@;
1248
1249     return $kvm_user_version->{$binary};
1250
1251 }
1252 my sub extract_version {
1253     my ($machine_type, $version) = @_;
1254     $version = kvm_user_version() if !defined($version);
1255     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1256 }
1257
1258 sub kernel_has_vhost_net {
1259     return -c '/dev/vhost-net';
1260 }
1261
1262 sub option_exists {
1263     my $key = shift;
1264     return defined($confdesc->{$key});
1265 }
1266
1267 my $cdrom_path;
1268 sub get_cdrom_path {
1269
1270     return  $cdrom_path if $cdrom_path;
1271
1272     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1273     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1274     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1275 }
1276
1277 sub get_iso_path {
1278     my ($storecfg, $vmid, $cdrom) = @_;
1279
1280     if ($cdrom eq 'cdrom') {
1281         return get_cdrom_path();
1282     } elsif ($cdrom eq 'none') {
1283         return '';
1284     } elsif ($cdrom =~ m|^/|) {
1285         return $cdrom;
1286     } else {
1287         return PVE::Storage::path($storecfg, $cdrom);
1288     }
1289 }
1290
1291 # try to convert old style file names to volume IDs
1292 sub filename_to_volume_id {
1293     my ($vmid, $file, $media) = @_;
1294
1295      if (!($file eq 'none' || $file eq 'cdrom' ||
1296           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1297
1298         return if $file =~ m|/|;
1299
1300         if ($media && $media eq 'cdrom') {
1301             $file = "local:iso/$file";
1302         } else {
1303             $file = "local:$vmid/$file";
1304         }
1305     }
1306
1307     return $file;
1308 }
1309
1310 sub verify_media_type {
1311     my ($opt, $vtype, $media) = @_;
1312
1313     return if !$media;
1314
1315     my $etype;
1316     if ($media eq 'disk') {
1317         $etype = 'images';
1318     } elsif ($media eq 'cdrom') {
1319         $etype = 'iso';
1320     } else {
1321         die "internal error";
1322     }
1323
1324     return if ($vtype eq $etype);
1325
1326     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1327 }
1328
1329 sub cleanup_drive_path {
1330     my ($opt, $storecfg, $drive) = @_;
1331
1332     # try to convert filesystem paths to volume IDs
1333
1334     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1335         ($drive->{file} !~ m|^/dev/.+|) &&
1336         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1337         ($drive->{file} !~ m/^\d+$/)) {
1338         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1339         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1340             if !$vtype;
1341         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1342         verify_media_type($opt, $vtype, $drive->{media});
1343         $drive->{file} = $volid;
1344     }
1345
1346     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1347 }
1348
1349 sub parse_hotplug_features {
1350     my ($data) = @_;
1351
1352     my $res = {};
1353
1354     return $res if $data eq '0';
1355
1356     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1357
1358     foreach my $feature (PVE::Tools::split_list($data)) {
1359         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1360             $res->{$1} = 1;
1361         } else {
1362             die "invalid hotplug feature '$feature'\n";
1363         }
1364     }
1365     return $res;
1366 }
1367
1368 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1369 sub pve_verify_hotplug_features {
1370     my ($value, $noerr) = @_;
1371
1372     return $value if parse_hotplug_features($value);
1373
1374     return if $noerr;
1375
1376     die "unable to parse hotplug option\n";
1377 }
1378
1379 sub scsi_inquiry {
1380     my($fh, $noerr) = @_;
1381
1382     my $SG_IO = 0x2285;
1383     my $SG_GET_VERSION_NUM = 0x2282;
1384
1385     my $versionbuf = "\x00" x 8;
1386     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1387     if (!$ret) {
1388         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1389         return;
1390     }
1391     my $version = unpack("I", $versionbuf);
1392     if ($version < 30000) {
1393         die "scsi generic interface too old\n"  if !$noerr;
1394         return;
1395     }
1396
1397     my $buf = "\x00" x 36;
1398     my $sensebuf = "\x00" x 8;
1399     my $cmd = pack("C x3 C x1", 0x12, 36);
1400
1401     # see /usr/include/scsi/sg.h
1402     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1403
1404     my $packet = pack(
1405         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1406     );
1407
1408     $ret = ioctl($fh, $SG_IO, $packet);
1409     if (!$ret) {
1410         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my @res = unpack($sg_io_hdr_t, $packet);
1415     if ($res[17] || $res[18]) {
1416         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1417         return;
1418     }
1419
1420     my $res = {};
1421     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1422
1423     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1424     $res->{type} &= 0x1F;
1425
1426     return $res;
1427 }
1428
1429 sub path_is_scsi {
1430     my ($path) = @_;
1431
1432     my $fh = IO::File->new("+<$path") || return;
1433     my $res = scsi_inquiry($fh, 1);
1434     close($fh);
1435
1436     return $res;
1437 }
1438
1439 sub print_tabletdevice_full {
1440     my ($conf, $arch) = @_;
1441
1442     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1443
1444     # we use uhci for old VMs because tablet driver was buggy in older qemu
1445     my $usbbus;
1446     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1447         $usbbus = 'ehci';
1448     } else {
1449         $usbbus = 'uhci';
1450     }
1451
1452     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1453 }
1454
1455 sub print_keyboarddevice_full {
1456     my ($conf, $arch) = @_;
1457
1458     return if $arch ne 'aarch64';
1459
1460     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1461 }
1462
1463 my sub get_drive_id {
1464     my ($drive) = @_;
1465     return "$drive->{interface}$drive->{index}";
1466 }
1467
1468 sub print_drivedevice_full {
1469     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1470
1471     my $device = '';
1472     my $maxdev = 0;
1473
1474     my $drive_id = get_drive_id($drive);
1475     if ($drive->{interface} eq 'virtio') {
1476         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1477         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1478         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1479     } elsif ($drive->{interface} eq 'scsi') {
1480
1481         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1482         my $unit = $drive->{index} % $maxdev;
1483         my $devicetype = 'hd';
1484         my $path = '';
1485         if (drive_is_cdrom($drive)) {
1486             $devicetype = 'cd';
1487         } else {
1488             if ($drive->{file} =~ m|^/|) {
1489                 $path = $drive->{file};
1490                 if (my $info = path_is_scsi($path)) {
1491                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1492                         $devicetype = 'block';
1493                     } elsif ($info->{type} == 1) { # tape
1494                         $devicetype = 'generic';
1495                     }
1496                 }
1497             } else {
1498                  $path = PVE::Storage::path($storecfg, $drive->{file});
1499             }
1500
1501             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1502             my $version = extract_version($machine_type, kvm_user_version());
1503             if ($path =~ m/^iscsi\:\/\// &&
1504                !min_version($version, 4, 1)) {
1505                 $devicetype = 'generic';
1506             }
1507         }
1508
1509         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1510             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1511         } else {
1512             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1513                 .",lun=$drive->{index}";
1514         }
1515         $device .= ",drive=drive-$drive_id,id=$drive_id";
1516
1517         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1518             $device .= ",rotation_rate=1";
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521
1522     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1523         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1524         my $controller = int($drive->{index} / $maxdev);
1525         my $unit = $drive->{index} % $maxdev;
1526         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1527
1528         $device = "ide-$devicetype";
1529         if ($drive->{interface} eq 'ide') {
1530             $device .= ",bus=ide.$controller,unit=$unit";
1531         } else {
1532             $device .= ",bus=ahci$controller.$unit";
1533         }
1534         $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536         if ($devicetype eq 'hd') {
1537             if (my $model = $drive->{model}) {
1538                 $model = URI::Escape::uri_unescape($model);
1539                 $device .= ",model=$model";
1540             }
1541             if ($drive->{ssd}) {
1542                 $device .= ",rotation_rate=1";
1543             }
1544         }
1545         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1546     } elsif ($drive->{interface} eq 'usb') {
1547         die "implement me";
1548         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1549     } else {
1550         die "unsupported interface type";
1551     }
1552
1553     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1554
1555     if (my $serial = $drive->{serial}) {
1556         $serial = URI::Escape::uri_unescape($serial);
1557         $device .= ",serial=$serial";
1558     }
1559
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_commandline_full {
1579     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format = $drive->{format};
1584     my $drive_id = get_drive_id($drive);
1585
1586     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1587     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1588
1589     if (drive_is_cdrom($drive)) {
1590         $path = get_iso_path($storecfg, $vmid, $volid);
1591         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1592     } else {
1593         if ($storeid) {
1594             $path = PVE::Storage::path($storecfg, $volid);
1595             $format //= qemu_img_format($scfg, $volname);
1596         } else {
1597             $path = $volid;
1598             $format //= "raw";
1599         }
1600    }
1601
1602    my $is_rbd = $path =~ m/^rbd:/;
1603
1604     my $opts = '';
1605     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1606     foreach my $o (@qemu_drive_options) {
1607         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1608     }
1609
1610     # snapshot only accepts on|off
1611     if (defined($drive->{snapshot})) {
1612         my $v = $drive->{snapshot} ? 'on' : 'off';
1613         $opts .= ",snapshot=$v";
1614     }
1615
1616     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1617         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1618     }
1619
1620     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1621         my ($dir, $qmpname) = @$type;
1622         if (my $v = $drive->{"mbps$dir"}) {
1623             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1624         }
1625         if (my $v = $drive->{"mbps${dir}_max"}) {
1626             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1627         }
1628         if (my $v = $drive->{"bps${dir}_max_length"}) {
1629             $opts .= ",throttling.bps$qmpname-max-length=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}"}) {
1632             $opts .= ",throttling.iops$qmpname=$v";
1633         }
1634         if (my $v = $drive->{"iops${dir}_max"}) {
1635             $opts .= ",throttling.iops$qmpname-max=$v";
1636         }
1637         if (my $v = $drive->{"iops${dir}_max_length"}) {
1638             $opts .= ",throttling.iops$qmpname-max-length=$v";
1639         }
1640     }
1641
1642     if ($pbs_name) {
1643         $format = "rbd" if $is_rbd;
1644         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1645             if !$format;
1646         $opts .= ",format=alloc-track,file.driver=$format";
1647     } elsif ($format) {
1648         $opts .= ",format=$format";
1649     }
1650
1651     my $cache_direct = 0;
1652
1653     if (my $cache = $drive->{cache}) {
1654         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1655     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1656         $opts .= ",cache=none";
1657         $cache_direct = 1;
1658     }
1659
1660     # io_uring with cache mode writeback or writethrough on krbd will hang...
1661     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1662
1663     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1664     # sometimes, just plain disable...
1665     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1666
1667     if (!$drive->{aio}) {
1668         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1669             # io_uring supports all cache modes
1670             $opts .= ",aio=io_uring";
1671         } else {
1672             # aio native works only with O_DIRECT
1673             if($cache_direct) {
1674                 $opts .= ",aio=native";
1675             } else {
1676                 $opts .= ",aio=threads";
1677             }
1678         }
1679     }
1680
1681     if (!drive_is_cdrom($drive)) {
1682         my $detectzeroes;
1683         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1684             $detectzeroes = 'off';
1685         } elsif ($drive->{discard}) {
1686             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1687         } else {
1688             # This used to be our default with discard not being specified:
1689             $detectzeroes = 'on';
1690         }
1691
1692         # note: 'detect-zeroes' works per blockdev and we want it to persist
1693         # after the alloc-track is removed, so put it on 'file' directly
1694         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1695         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1696     }
1697
1698     if ($pbs_name) {
1699         $opts .= ",backing=$pbs_name";
1700         $opts .= ",auto-remove=on";
1701     }
1702
1703     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1704     my $file_param = "file";
1705     if ($pbs_name) {
1706         # non-rbd drivers require the underlying file to be a seperate block
1707         # node, so add a second .file indirection
1708         $file_param .= ".file" if !$is_rbd;
1709         $file_param .= ".filename";
1710     }
1711     my $pathinfo = $path ? "$file_param=$path," : '';
1712
1713     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1714 }
1715
1716 sub print_pbs_blockdev {
1717     my ($pbs_conf, $pbs_name) = @_;
1718     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1719     $blockdev .= ",repository=$pbs_conf->{repository}";
1720     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1721     $blockdev .= ",archive=$pbs_conf->{archive}";
1722     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1723     return $blockdev;
1724 }
1725
1726 sub print_netdevice_full {
1727     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1728
1729     my $device = $net->{model};
1730     if ($net->{model} eq 'virtio') {
1731          $device = 'virtio-net-pci';
1732      };
1733
1734     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1735     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1736     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1737         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1738         # and out of each queue plus one config interrupt and control vector queue
1739         my $vectors = $net->{queues} * 2 + 2;
1740         $tmpstr .= ",vectors=$vectors,mq=on";
1741     }
1742     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1743
1744     if (my $mtu = $net->{mtu}) {
1745         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1746             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1747             if ($mtu == 1) {
1748                  $mtu = $bridge_mtu;
1749             } elsif ($mtu < 576) {
1750                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1751             } elsif ($mtu > $bridge_mtu) {
1752                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1753             }
1754             $tmpstr .= ",host_mtu=$mtu";
1755         } else {
1756             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1757         }
1758     }
1759
1760     if ($use_old_bios_files) {
1761         my $romfile;
1762         if ($device eq 'virtio-net-pci') {
1763             $romfile = 'pxe-virtio.rom';
1764         } elsif ($device eq 'e1000') {
1765             $romfile = 'pxe-e1000.rom';
1766         } elsif ($device eq 'e1000e') {
1767             $romfile = 'pxe-e1000e.rom';
1768         } elsif ($device eq 'ne2k') {
1769             $romfile = 'pxe-ne2k_pci.rom';
1770         } elsif ($device eq 'pcnet') {
1771             $romfile = 'pxe-pcnet.rom';
1772         } elsif ($device eq 'rtl8139') {
1773             $romfile = 'pxe-rtl8139.rom';
1774         }
1775         $tmpstr .= ",romfile=$romfile" if $romfile;
1776     }
1777
1778     return $tmpstr;
1779 }
1780
1781 sub print_netdev_full {
1782     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1783
1784     my $i = '';
1785     if ($netid =~ m/^net(\d+)$/) {
1786         $i = int($1);
1787     }
1788
1789     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1790
1791     my $ifname = "tap${vmid}i$i";
1792
1793     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1794     die "interface name '$ifname' is too long (max 15 character)\n"
1795         if length($ifname) >= 16;
1796
1797     my $vhostparam = '';
1798     if (is_native($arch)) {
1799         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1800     }
1801
1802     my $vmname = $conf->{name} || "vm$vmid";
1803
1804     my $netdev = "";
1805     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1806
1807     if ($net->{bridge}) {
1808         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1809             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1810     } else {
1811         $netdev = "type=user,id=$netid,hostname=$vmname";
1812     }
1813
1814     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1815
1816     return $netdev;
1817 }
1818
1819 my $vga_map = {
1820     'cirrus' => 'cirrus-vga',
1821     'std' => 'VGA',
1822     'vmware' => 'vmware-svga',
1823     'virtio' => 'virtio-vga',
1824     'virtio-gl' => 'virtio-vga-gl',
1825 };
1826
1827 sub print_vga_device {
1828     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1829
1830     my $type = $vga_map->{$vga->{type}};
1831     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1832         $type = 'virtio-gpu';
1833     }
1834     my $vgamem_mb = $vga->{memory};
1835
1836     my $max_outputs = '';
1837     if ($qxlnum) {
1838         $type = $id ? 'qxl' : 'qxl-vga';
1839
1840         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1841             # set max outputs so linux can have up to 4 qxl displays with one device
1842             if (min_version($machine_version, 4, 1)) {
1843                 $max_outputs = ",max_outputs=4";
1844             }
1845         }
1846     }
1847
1848     die "no devicetype for $vga->{type}\n" if !$type;
1849
1850     my $memory = "";
1851     if ($vgamem_mb) {
1852         if ($vga->{type} =~ /^virtio/) {
1853             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1854             $memory = ",max_hostmem=$bytes";
1855         } elsif ($qxlnum) {
1856             # from https://www.spice-space.org/multiple-monitors.html
1857             $memory = ",vgamem_mb=$vga->{memory}";
1858             my $ram = $vgamem_mb * 4;
1859             my $vram = $vgamem_mb * 2;
1860             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1861         } else {
1862             $memory = ",vgamem_mb=$vga->{memory}";
1863         }
1864     } elsif ($qxlnum && $id) {
1865         $memory = ",ram_size=67108864,vram_size=33554432";
1866     }
1867
1868     my $edidoff = "";
1869     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1870         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1871     }
1872
1873     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1874     my $vgaid = "vga" . ($id // '');
1875     my $pciaddr;
1876     if ($q35 && $vgaid eq 'vga') {
1877         # the first display uses pcie.0 bus on q35 machines
1878         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1879     } else {
1880         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1881     }
1882
1883     if ($vga->{type} eq 'virtio-gl') {
1884         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1885         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1886             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1887
1888         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1889             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1890     }
1891
1892     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1893 }
1894
1895 sub parse_number_sets {
1896     my ($set) = @_;
1897     my $res = [];
1898     foreach my $part (split(/;/, $set)) {
1899         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1900             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1901             push @$res, [ $1, $2 ];
1902         } else {
1903             die "invalid range: $part\n";
1904         }
1905     }
1906     return $res;
1907 }
1908
1909 sub parse_numa {
1910     my ($data) = @_;
1911
1912     my $res = parse_property_string($numa_fmt, $data);
1913     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1914     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1915     return $res;
1916 }
1917
1918 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1919 sub parse_net {
1920     my ($data) = @_;
1921
1922     my $res = eval { parse_property_string($net_fmt, $data) };
1923     if ($@) {
1924         warn $@;
1925         return;
1926     }
1927     if (!defined($res->{macaddr})) {
1928         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1929         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1930     }
1931     return $res;
1932 }
1933
1934 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1935 sub parse_ipconfig {
1936     my ($data) = @_;
1937
1938     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1939     if ($@) {
1940         warn $@;
1941         return;
1942     }
1943
1944     if ($res->{gw} && !$res->{ip}) {
1945         warn 'gateway specified without specifying an IP address';
1946         return;
1947     }
1948     if ($res->{gw6} && !$res->{ip6}) {
1949         warn 'IPv6 gateway specified without specifying an IPv6 address';
1950         return;
1951     }
1952     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1953         warn 'gateway specified together with DHCP';
1954         return;
1955     }
1956     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1957         # gw6 + auto/dhcp
1958         warn "IPv6 gateway specified together with $res->{ip6} address";
1959         return;
1960     }
1961
1962     if (!$res->{ip} && !$res->{ip6}) {
1963         return { ip => 'dhcp', ip6 => 'dhcp' };
1964     }
1965
1966     return $res;
1967 }
1968
1969 sub print_net {
1970     my $net = shift;
1971
1972     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1973 }
1974
1975 sub add_random_macs {
1976     my ($settings) = @_;
1977
1978     foreach my $opt (keys %$settings) {
1979         next if $opt !~ m/^net(\d+)$/;
1980         my $net = parse_net($settings->{$opt});
1981         next if !$net;
1982         $settings->{$opt} = print_net($net);
1983     }
1984 }
1985
1986 sub vm_is_volid_owner {
1987     my ($storecfg, $vmid, $volid) = @_;
1988
1989     if ($volid !~  m|^/|) {
1990         my ($path, $owner);
1991         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1992         if ($owner && ($owner == $vmid)) {
1993             return 1;
1994         }
1995     }
1996
1997     return;
1998 }
1999
2000 sub vmconfig_register_unused_drive {
2001     my ($storecfg, $vmid, $conf, $drive) = @_;
2002
2003     if (drive_is_cloudinit($drive)) {
2004         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2005         warn $@ if $@;
2006     } elsif (!drive_is_cdrom($drive)) {
2007         my $volid = $drive->{file};
2008         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2009             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2010         }
2011     }
2012 }
2013
2014 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2015 my $smbios1_fmt = {
2016     uuid => {
2017         type => 'string',
2018         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2019         format_description => 'UUID',
2020         description => "Set SMBIOS1 UUID.",
2021         optional => 1,
2022     },
2023     version => {
2024         type => 'string',
2025         pattern => '[A-Za-z0-9+\/]+={0,2}',
2026         format_description => 'Base64 encoded string',
2027         description => "Set SMBIOS1 version.",
2028         optional => 1,
2029     },
2030     serial => {
2031         type => 'string',
2032         pattern => '[A-Za-z0-9+\/]+={0,2}',
2033         format_description => 'Base64 encoded string',
2034         description => "Set SMBIOS1 serial number.",
2035         optional => 1,
2036     },
2037     manufacturer => {
2038         type => 'string',
2039         pattern => '[A-Za-z0-9+\/]+={0,2}',
2040         format_description => 'Base64 encoded string',
2041         description => "Set SMBIOS1 manufacturer.",
2042         optional => 1,
2043     },
2044     product => {
2045         type => 'string',
2046         pattern => '[A-Za-z0-9+\/]+={0,2}',
2047         format_description => 'Base64 encoded string',
2048         description => "Set SMBIOS1 product ID.",
2049         optional => 1,
2050     },
2051     sku => {
2052         type => 'string',
2053         pattern => '[A-Za-z0-9+\/]+={0,2}',
2054         format_description => 'Base64 encoded string',
2055         description => "Set SMBIOS1 SKU string.",
2056         optional => 1,
2057     },
2058     family => {
2059         type => 'string',
2060         pattern => '[A-Za-z0-9+\/]+={0,2}',
2061         format_description => 'Base64 encoded string',
2062         description => "Set SMBIOS1 family string.",
2063         optional => 1,
2064     },
2065     base64 => {
2066         type => 'boolean',
2067         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2068         optional => 1,
2069     },
2070 };
2071
2072 sub parse_smbios1 {
2073     my ($data) = @_;
2074
2075     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2076     warn $@ if $@;
2077     return $res;
2078 }
2079
2080 sub print_smbios1 {
2081     my ($smbios1) = @_;
2082     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2083 }
2084
2085 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2086
2087 sub parse_watchdog {
2088     my ($value) = @_;
2089
2090     return if !$value;
2091
2092     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2093     warn $@ if $@;
2094     return $res;
2095 }
2096
2097 sub parse_guest_agent {
2098     my ($conf) = @_;
2099
2100     return {} if !defined($conf->{agent});
2101
2102     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2103     warn $@ if $@;
2104
2105     # if the agent is disabled ignore the other potentially set properties
2106     return {} if !$res->{enabled};
2107     return $res;
2108 }
2109
2110 sub get_qga_key {
2111     my ($conf, $key) = @_;
2112     return undef if !defined($conf->{agent});
2113
2114     my $agent = parse_guest_agent($conf);
2115     return $agent->{$key};
2116 }
2117
2118 sub parse_vga {
2119     my ($value) = @_;
2120
2121     return {} if !$value;
2122     my $res = eval { parse_property_string($vga_fmt, $value) };
2123     warn $@ if $@;
2124     return $res;
2125 }
2126
2127 sub parse_rng {
2128     my ($value) = @_;
2129
2130     return if !$value;
2131
2132     my $res = eval { parse_property_string($rng_fmt, $value) };
2133     warn $@ if $@;
2134     return $res;
2135 }
2136
2137 sub parse_meta_info {
2138     my ($value) = @_;
2139
2140     return if !$value;
2141
2142     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2143     warn $@ if $@;
2144     return $res;
2145 }
2146
2147 sub new_meta_info_string {
2148     my () = @_; # for now do not allow to override any value
2149
2150     return PVE::JSONSchema::print_property_string(
2151         {
2152             'creation-qemu' => kvm_user_version(),
2153             ctime => "". int(time()),
2154         },
2155         $meta_info_fmt
2156     );
2157 }
2158
2159 sub qemu_created_version_fixups {
2160     my ($conf, $forcemachine, $kvmver) = @_;
2161
2162     my $meta = parse_meta_info($conf->{meta}) // {};
2163     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2164
2165     # check if we need to apply some handling for VMs that always use the latest machine version but
2166     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2167     # would be required (we do not want to pin machine version for non-windows OS type)
2168     if (
2169         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2170         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2171         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2172         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2173     ) {
2174         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2175         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2176             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2177             # and thus with the predictable interface naming of systemd
2178             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2179         }
2180     }
2181     return;
2182 }
2183
2184 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2185 sub verify_usb_device {
2186     my ($value, $noerr) = @_;
2187
2188     return $value if parse_usb_device($value);
2189
2190     return if $noerr;
2191
2192     die "unable to parse usb device\n";
2193 }
2194
2195 # add JSON properties for create and set function
2196 sub json_config_properties {
2197     my ($prop, $with_disk_alloc) = @_;
2198
2199     my $skip_json_config_opts = {
2200         parent => 1,
2201         snaptime => 1,
2202         vmstate => 1,
2203         runningmachine => 1,
2204         runningcpu => 1,
2205         meta => 1,
2206     };
2207
2208     foreach my $opt (keys %$confdesc) {
2209         next if $skip_json_config_opts->{$opt};
2210
2211         if ($with_disk_alloc && is_valid_drivename($opt)) {
2212             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2213         } else {
2214             $prop->{$opt} = $confdesc->{$opt};
2215         }
2216     }
2217
2218     return $prop;
2219 }
2220
2221 # Properties that we can read from an OVF file
2222 sub json_ovf_properties {
2223     my $prop = {};
2224
2225     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2226         $prop->{$device} = {
2227             type => 'string',
2228             format => 'pve-volume-id-or-absolute-path',
2229             description => "Disk image that gets imported to $device",
2230             optional => 1,
2231         };
2232     }
2233
2234     $prop->{cores} = {
2235         type => 'integer',
2236         description => "The number of CPU cores.",
2237         optional => 1,
2238     };
2239     $prop->{memory} = {
2240         type => 'integer',
2241         description => "Amount of RAM for the VM in MB.",
2242         optional => 1,
2243     };
2244     $prop->{name} = {
2245         type => 'string',
2246         description => "Name of the VM.",
2247         optional => 1,
2248     };
2249
2250     return $prop;
2251 }
2252
2253 # return copy of $confdesc_cloudinit to generate documentation
2254 sub cloudinit_config_properties {
2255
2256     return dclone($confdesc_cloudinit);
2257 }
2258
2259 sub check_type {
2260     my ($key, $value) = @_;
2261
2262     die "unknown setting '$key'\n" if !$confdesc->{$key};
2263
2264     my $type = $confdesc->{$key}->{type};
2265
2266     if (!defined($value)) {
2267         die "got undefined value\n";
2268     }
2269
2270     if ($value =~ m/[\n\r]/) {
2271         die "property contains a line feed\n";
2272     }
2273
2274     if ($type eq 'boolean') {
2275         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2276         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2277         die "type check ('boolean') failed - got '$value'\n";
2278     } elsif ($type eq 'integer') {
2279         return int($1) if $value =~ m/^(\d+)$/;
2280         die "type check ('integer') failed - got '$value'\n";
2281     } elsif ($type eq 'number') {
2282         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2283         die "type check ('number') failed - got '$value'\n";
2284     } elsif ($type eq 'string') {
2285         if (my $fmt = $confdesc->{$key}->{format}) {
2286             PVE::JSONSchema::check_format($fmt, $value);
2287             return $value;
2288         }
2289         $value =~ s/^\"(.*)\"$/$1/;
2290         return $value;
2291     } else {
2292         die "internal error"
2293     }
2294 }
2295
2296 sub destroy_vm {
2297     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2298
2299     my $conf = PVE::QemuConfig->load_config($vmid);
2300
2301     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2302
2303     if ($conf->{template}) {
2304         # check if any base image is still used by a linked clone
2305         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2306                 my ($ds, $drive) = @_;
2307                 return if drive_is_cdrom($drive);
2308
2309                 my $volid = $drive->{file};
2310                 return if !$volid || $volid =~ m|^/|;
2311
2312                 die "base volume '$volid' is still in use by linked cloned\n"
2313                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2314
2315         });
2316     }
2317
2318     my $volids = {};
2319     my $remove_owned_drive = sub {
2320         my ($ds, $drive) = @_;
2321         return if drive_is_cdrom($drive, 1);
2322
2323         my $volid = $drive->{file};
2324         return if !$volid || $volid =~ m|^/|;
2325         return if $volids->{$volid};
2326
2327         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2328         return if !$path || !$owner || ($owner != $vmid);
2329
2330         $volids->{$volid} = 1;
2331         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2332         warn "Could not remove disk '$volid', check manually: $@" if $@;
2333     };
2334
2335     # only remove disks owned by this VM (referenced in the config)
2336     my $include_opts = {
2337         include_unused => 1,
2338         extra_keys => ['vmstate'],
2339     };
2340     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2341
2342     for my $snap (values %{$conf->{snapshots}}) {
2343         next if !defined($snap->{vmstate});
2344         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2345         next if !defined($drive);
2346         $remove_owned_drive->('vmstate', $drive);
2347     }
2348
2349     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2350
2351     if ($purge_unreferenced) { # also remove unreferenced disk
2352         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2353         PVE::Storage::foreach_volid($vmdisks, sub {
2354             my ($volid, $sid, $volname, $d) = @_;
2355             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2356             warn $@ if $@;
2357         });
2358     }
2359
2360     if (defined $replacement_conf) {
2361         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2362     } else {
2363         PVE::QemuConfig->destroy_config($vmid);
2364     }
2365 }
2366
2367 sub parse_vm_config {
2368     my ($filename, $raw, $strict) = @_;
2369
2370     return if !defined($raw);
2371
2372     my $res = {
2373         digest => Digest::SHA::sha1_hex($raw),
2374         snapshots => {},
2375         pending => {},
2376     };
2377
2378     my $handle_error = sub {
2379         my ($msg) = @_;
2380
2381         if ($strict) {
2382             die $msg;
2383         } else {
2384             warn $msg;
2385         }
2386     };
2387
2388     $filename =~ m|/qemu-server/(\d+)\.conf$|
2389         || die "got strange filename '$filename'";
2390
2391     my $vmid = $1;
2392
2393     my $conf = $res;
2394     my $descr;
2395     my $section = '';
2396
2397     my @lines = split(/\n/, $raw);
2398     foreach my $line (@lines) {
2399         next if $line =~ m/^\s*$/;
2400
2401         if ($line =~ m/^\[PENDING\]\s*$/i) {
2402             $section = 'pending';
2403             if (defined($descr)) {
2404                 $descr =~ s/\s+$//;
2405                 $conf->{description} = $descr;
2406             }
2407             $descr = undef;
2408             $conf = $res->{$section} = {};
2409             next;
2410
2411         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2412             $section = $1;
2413             if (defined($descr)) {
2414                 $descr =~ s/\s+$//;
2415                 $conf->{description} = $descr;
2416             }
2417             $descr = undef;
2418             $conf = $res->{snapshots}->{$section} = {};
2419             next;
2420         }
2421
2422         if ($line =~ m/^\#(.*)$/) {
2423             $descr = '' if !defined($descr);
2424             $descr .= PVE::Tools::decode_text($1) . "\n";
2425             next;
2426         }
2427
2428         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2429             $descr = '' if !defined($descr);
2430             $descr .= PVE::Tools::decode_text($2);
2431         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2432             $conf->{snapstate} = $1;
2433         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2434             my $key = $1;
2435             my $value = $2;
2436             $conf->{$key} = $value;
2437         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2438             my $value = $1;
2439             if ($section eq 'pending') {
2440                 $conf->{delete} = $value; # we parse this later
2441             } else {
2442                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2443             }
2444         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2445             my $key = $1;
2446             my $value = $2;
2447             eval { $value = check_type($key, $value); };
2448             if ($@) {
2449                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2450             } else {
2451                 $key = 'ide2' if $key eq 'cdrom';
2452                 my $fmt = $confdesc->{$key}->{format};
2453                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2454                     my $v = parse_drive($key, $value);
2455                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2456                         $v->{file} = $volid;
2457                         $value = print_drive($v);
2458                     } else {
2459                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2460                         next;
2461                     }
2462                 }
2463
2464                 $conf->{$key} = $value;
2465             }
2466         } else {
2467             $handle_error->("vm $vmid - unable to parse config: $line\n");
2468         }
2469     }
2470
2471     if (defined($descr)) {
2472         $descr =~ s/\s+$//;
2473         $conf->{description} = $descr;
2474     }
2475     delete $res->{snapstate}; # just to be sure
2476
2477     return $res;
2478 }
2479
2480 sub write_vm_config {
2481     my ($filename, $conf) = @_;
2482
2483     delete $conf->{snapstate}; # just to be sure
2484
2485     if ($conf->{cdrom}) {
2486         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2487         $conf->{ide2} = $conf->{cdrom};
2488         delete $conf->{cdrom};
2489     }
2490
2491     # we do not use 'smp' any longer
2492     if ($conf->{sockets}) {
2493         delete $conf->{smp};
2494     } elsif ($conf->{smp}) {
2495         $conf->{sockets} = $conf->{smp};
2496         delete $conf->{cores};
2497         delete $conf->{smp};
2498     }
2499
2500     my $used_volids = {};
2501
2502     my $cleanup_config = sub {
2503         my ($cref, $pending, $snapname) = @_;
2504
2505         foreach my $key (keys %$cref) {
2506             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2507                 $key eq 'snapstate' || $key eq 'pending';
2508             my $value = $cref->{$key};
2509             if ($key eq 'delete') {
2510                 die "propertry 'delete' is only allowed in [PENDING]\n"
2511                     if !$pending;
2512                 # fixme: check syntax?
2513                 next;
2514             }
2515             eval { $value = check_type($key, $value); };
2516             die "unable to parse value of '$key' - $@" if $@;
2517
2518             $cref->{$key} = $value;
2519
2520             if (!$snapname && is_valid_drivename($key)) {
2521                 my $drive = parse_drive($key, $value);
2522                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2523             }
2524         }
2525     };
2526
2527     &$cleanup_config($conf);
2528
2529     &$cleanup_config($conf->{pending}, 1);
2530
2531     foreach my $snapname (keys %{$conf->{snapshots}}) {
2532         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2533         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2534     }
2535
2536     # remove 'unusedX' settings if we re-add a volume
2537     foreach my $key (keys %$conf) {
2538         my $value = $conf->{$key};
2539         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2540             delete $conf->{$key};
2541         }
2542     }
2543
2544     my $generate_raw_config = sub {
2545         my ($conf, $pending) = @_;
2546
2547         my $raw = '';
2548
2549         # add description as comment to top of file
2550         if (defined(my $descr = $conf->{description})) {
2551             if ($descr) {
2552                 foreach my $cl (split(/\n/, $descr)) {
2553                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2554                 }
2555             } else {
2556                 $raw .= "#\n" if $pending;
2557             }
2558         }
2559
2560         foreach my $key (sort keys %$conf) {
2561             next if $key =~ /^(digest|description|pending|snapshots)$/;
2562             $raw .= "$key: $conf->{$key}\n";
2563         }
2564         return $raw;
2565     };
2566
2567     my $raw = &$generate_raw_config($conf);
2568
2569     if (scalar(keys %{$conf->{pending}})){
2570         $raw .= "\n[PENDING]\n";
2571         $raw .= &$generate_raw_config($conf->{pending}, 1);
2572     }
2573
2574     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2575         $raw .= "\n[$snapname]\n";
2576         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2577     }
2578
2579     return $raw;
2580 }
2581
2582 sub load_defaults {
2583
2584     my $res = {};
2585
2586     # we use static defaults from our JSON schema configuration
2587     foreach my $key (keys %$confdesc) {
2588         if (defined(my $default = $confdesc->{$key}->{default})) {
2589             $res->{$key} = $default;
2590         }
2591     }
2592
2593     return $res;
2594 }
2595
2596 sub config_list {
2597     my $vmlist = PVE::Cluster::get_vmlist();
2598     my $res = {};
2599     return $res if !$vmlist || !$vmlist->{ids};
2600     my $ids = $vmlist->{ids};
2601     my $nodename = nodename();
2602
2603     foreach my $vmid (keys %$ids) {
2604         my $d = $ids->{$vmid};
2605         next if !$d->{node} || $d->{node} ne $nodename;
2606         next if !$d->{type} || $d->{type} ne 'qemu';
2607         $res->{$vmid}->{exists} = 1;
2608     }
2609     return $res;
2610 }
2611
2612 # test if VM uses local resources (to prevent migration)
2613 sub check_local_resources {
2614     my ($conf, $noerr) = @_;
2615
2616     my @loc_res = ();
2617
2618     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2619     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2620
2621     push @loc_res, "ivshmem" if $conf->{ivshmem};
2622
2623     foreach my $k (keys %$conf) {
2624         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2625         # sockets are safe: they will recreated be on the target side post-migrate
2626         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2627         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2628     }
2629
2630     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2631
2632     return \@loc_res;
2633 }
2634
2635 # check if used storages are available on all nodes (use by migrate)
2636 sub check_storage_availability {
2637     my ($storecfg, $conf, $node) = @_;
2638
2639     PVE::QemuConfig->foreach_volume($conf, sub {
2640         my ($ds, $drive) = @_;
2641
2642         my $volid = $drive->{file};
2643         return if !$volid;
2644
2645         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2646         return if !$sid;
2647
2648         # check if storage is available on both nodes
2649         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2650         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2651
2652         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2653
2654         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2655             if !$scfg->{content}->{$vtype};
2656    });
2657 }
2658
2659 # list nodes where all VM images are available (used by has_feature API)
2660 sub shared_nodes {
2661     my ($conf, $storecfg) = @_;
2662
2663     my $nodelist = PVE::Cluster::get_nodelist();
2664     my $nodehash = { map { $_ => 1 } @$nodelist };
2665     my $nodename = nodename();
2666
2667     PVE::QemuConfig->foreach_volume($conf, sub {
2668         my ($ds, $drive) = @_;
2669
2670         my $volid = $drive->{file};
2671         return if !$volid;
2672
2673         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2674         if ($storeid) {
2675             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2676             if ($scfg->{disable}) {
2677                 $nodehash = {};
2678             } elsif (my $avail = $scfg->{nodes}) {
2679                 foreach my $node (keys %$nodehash) {
2680                     delete $nodehash->{$node} if !$avail->{$node};
2681                 }
2682             } elsif (!$scfg->{shared}) {
2683                 foreach my $node (keys %$nodehash) {
2684                     delete $nodehash->{$node} if $node ne $nodename
2685                 }
2686             }
2687         }
2688     });
2689
2690     return $nodehash
2691 }
2692
2693 sub check_local_storage_availability {
2694     my ($conf, $storecfg) = @_;
2695
2696     my $nodelist = PVE::Cluster::get_nodelist();
2697     my $nodehash = { map { $_ => {} } @$nodelist };
2698
2699     PVE::QemuConfig->foreach_volume($conf, sub {
2700         my ($ds, $drive) = @_;
2701
2702         my $volid = $drive->{file};
2703         return if !$volid;
2704
2705         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2706         if ($storeid) {
2707             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2708
2709             if ($scfg->{disable}) {
2710                 foreach my $node (keys %$nodehash) {
2711                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2712                 }
2713             } elsif (my $avail = $scfg->{nodes}) {
2714                 foreach my $node (keys %$nodehash) {
2715                     if (!$avail->{$node}) {
2716                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2717                     }
2718                 }
2719             }
2720         }
2721     });
2722
2723     foreach my $node (values %$nodehash) {
2724         if (my $unavail = $node->{unavailable_storages}) {
2725             $node->{unavailable_storages} = [ sort keys %$unavail ];
2726         }
2727     }
2728
2729     return $nodehash
2730 }
2731
2732 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2733 sub check_running {
2734     my ($vmid, $nocheck, $node) = @_;
2735
2736     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2737     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2738 }
2739
2740 sub vzlist {
2741
2742     my $vzlist = config_list();
2743
2744     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2745
2746     while (defined(my $de = $fd->read)) {
2747         next if $de !~ m/^(\d+)\.pid$/;
2748         my $vmid = $1;
2749         next if !defined($vzlist->{$vmid});
2750         if (my $pid = check_running($vmid)) {
2751             $vzlist->{$vmid}->{pid} = $pid;
2752         }
2753     }
2754
2755     return $vzlist;
2756 }
2757
2758 our $vmstatus_return_properties = {
2759     vmid => get_standard_option('pve-vmid'),
2760     status => {
2761         description => "Qemu process status.",
2762         type => 'string',
2763         enum => ['stopped', 'running'],
2764     },
2765     maxmem => {
2766         description => "Maximum memory in bytes.",
2767         type => 'integer',
2768         optional => 1,
2769         renderer => 'bytes',
2770     },
2771     maxdisk => {
2772         description => "Root disk size in bytes.",
2773         type => 'integer',
2774         optional => 1,
2775         renderer => 'bytes',
2776     },
2777     name => {
2778         description => "VM name.",
2779         type => 'string',
2780         optional => 1,
2781     },
2782     qmpstatus => {
2783         description => "Qemu QMP agent status.",
2784         type => 'string',
2785         optional => 1,
2786     },
2787     pid => {
2788         description => "PID of running qemu process.",
2789         type => 'integer',
2790         optional => 1,
2791     },
2792     uptime => {
2793         description => "Uptime.",
2794         type => 'integer',
2795         optional => 1,
2796         renderer => 'duration',
2797     },
2798     cpus => {
2799         description => "Maximum usable CPUs.",
2800         type => 'number',
2801         optional => 1,
2802     },
2803     lock => {
2804         description => "The current config lock, if any.",
2805         type => 'string',
2806         optional => 1,
2807     },
2808     tags => {
2809         description  => "The current configured tags, if any",
2810         type => 'string',
2811         optional => 1,
2812     },
2813     'running-machine' => {
2814         description => "The currently running machine type (if running).",
2815         type => 'string',
2816         optional => 1,
2817     },
2818     'running-qemu' => {
2819         description => "The currently running QEMU version (if running).",
2820         type => 'string',
2821         optional => 1,
2822     },
2823 };
2824
2825 my $last_proc_pid_stat;
2826
2827 # get VM status information
2828 # This must be fast and should not block ($full == false)
2829 # We only query KVM using QMP if $full == true (this can be slow)
2830 sub vmstatus {
2831     my ($opt_vmid, $full) = @_;
2832
2833     my $res = {};
2834
2835     my $storecfg = PVE::Storage::config();
2836
2837     my $list = vzlist();
2838     my $defaults = load_defaults();
2839
2840     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2841
2842     my $cpucount = $cpuinfo->{cpus} || 1;
2843
2844     foreach my $vmid (keys %$list) {
2845         next if $opt_vmid && ($vmid ne $opt_vmid);
2846
2847         my $conf = PVE::QemuConfig->load_config($vmid);
2848
2849         my $d = { vmid => int($vmid) };
2850         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2851
2852         # fixme: better status?
2853         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2854
2855         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2856         if (defined($size)) {
2857             $d->{disk} = 0; # no info available
2858             $d->{maxdisk} = $size;
2859         } else {
2860             $d->{disk} = 0;
2861             $d->{maxdisk} = 0;
2862         }
2863
2864         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2865             * ($conf->{cores} || $defaults->{cores});
2866         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2867         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2868
2869         $d->{name} = $conf->{name} || "VM $vmid";
2870         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2871             : $defaults->{memory}*(1024*1024);
2872
2873         if ($conf->{balloon}) {
2874             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2875             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2876                 : $defaults->{shares};
2877         }
2878
2879         $d->{uptime} = 0;
2880         $d->{cpu} = 0;
2881         $d->{mem} = 0;
2882
2883         $d->{netout} = 0;
2884         $d->{netin} = 0;
2885
2886         $d->{diskread} = 0;
2887         $d->{diskwrite} = 0;
2888
2889         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2890
2891         $d->{serial} = 1 if conf_has_serial($conf);
2892         $d->{lock} = $conf->{lock} if $conf->{lock};
2893         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2894
2895         $res->{$vmid} = $d;
2896     }
2897
2898     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2899     foreach my $dev (keys %$netdev) {
2900         next if $dev !~ m/^tap([1-9]\d*)i/;
2901         my $vmid = $1;
2902         my $d = $res->{$vmid};
2903         next if !$d;
2904
2905         $d->{netout} += $netdev->{$dev}->{receive};
2906         $d->{netin} += $netdev->{$dev}->{transmit};
2907
2908         if ($full) {
2909             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2910             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2911         }
2912
2913     }
2914
2915     my $ctime = gettimeofday;
2916
2917     foreach my $vmid (keys %$list) {
2918
2919         my $d = $res->{$vmid};
2920         my $pid = $d->{pid};
2921         next if !$pid;
2922
2923         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2924         next if !$pstat; # not running
2925
2926         my $used = $pstat->{utime} + $pstat->{stime};
2927
2928         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2929
2930         if ($pstat->{vsize}) {
2931             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2932         }
2933
2934         my $old = $last_proc_pid_stat->{$pid};
2935         if (!$old) {
2936             $last_proc_pid_stat->{$pid} = {
2937                 time => $ctime,
2938                 used => $used,
2939                 cpu => 0,
2940             };
2941             next;
2942         }
2943
2944         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2945
2946         if ($dtime > 1000) {
2947             my $dutime = $used -  $old->{used};
2948
2949             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2950             $last_proc_pid_stat->{$pid} = {
2951                 time => $ctime,
2952                 used => $used,
2953                 cpu => $d->{cpu},
2954             };
2955         } else {
2956             $d->{cpu} = $old->{cpu};
2957         }
2958     }
2959
2960     return $res if !$full;
2961
2962     my $qmpclient = PVE::QMPClient->new();
2963
2964     my $ballooncb = sub {
2965         my ($vmid, $resp) = @_;
2966
2967         my $info = $resp->{'return'};
2968         return if !$info->{max_mem};
2969
2970         my $d = $res->{$vmid};
2971
2972         # use memory assigned to VM
2973         $d->{maxmem} = $info->{max_mem};
2974         $d->{balloon} = $info->{actual};
2975
2976         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2977             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2978             $d->{freemem} = $info->{free_mem};
2979         }
2980
2981         $d->{ballooninfo} = $info;
2982     };
2983
2984     my $blockstatscb = sub {
2985         my ($vmid, $resp) = @_;
2986         my $data = $resp->{'return'} || [];
2987         my $totalrdbytes = 0;
2988         my $totalwrbytes = 0;
2989
2990         for my $blockstat (@$data) {
2991             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2992             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2993
2994             $blockstat->{device} =~ s/drive-//;
2995             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2996         }
2997         $res->{$vmid}->{diskread} = $totalrdbytes;
2998         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2999     };
3000
3001     my $machinecb = sub {
3002         my ($vmid, $resp) = @_;
3003         my $data = $resp->{'return'} || [];
3004
3005         $res->{$vmid}->{'running-machine'} =
3006             PVE::QemuServer::Machine::current_from_query_machines($data);
3007     };
3008
3009     my $versioncb = sub {
3010         my ($vmid, $resp) = @_;
3011         my $data = $resp->{'return'} // {};
3012         my $version = 'unknown';
3013
3014         if (my $v = $data->{qemu}) {
3015             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3016         }
3017
3018         $res->{$vmid}->{'running-qemu'} = $version;
3019     };
3020
3021     my $statuscb = sub {
3022         my ($vmid, $resp) = @_;
3023
3024         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3025         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3026         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3027         # this fails if ballon driver is not loaded, so this must be
3028         # the last commnand (following command are aborted if this fails).
3029         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3030
3031         my $status = 'unknown';
3032         if (!defined($status = $resp->{'return'}->{status})) {
3033             warn "unable to get VM status\n";
3034             return;
3035         }
3036
3037         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3038     };
3039
3040     foreach my $vmid (keys %$list) {
3041         next if $opt_vmid && ($vmid ne $opt_vmid);
3042         next if !$res->{$vmid}->{pid}; # not running
3043         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3044     }
3045
3046     $qmpclient->queue_execute(undef, 2);
3047
3048     foreach my $vmid (keys %$list) {
3049         next if $opt_vmid && ($vmid ne $opt_vmid);
3050         next if !$res->{$vmid}->{pid}; #not running
3051
3052         # we can't use the $qmpclient since it might have already aborted on
3053         # 'query-balloon', but this might also fail for older versions...
3054         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3055         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3056     }
3057
3058     foreach my $vmid (keys %$list) {
3059         next if $opt_vmid && ($vmid ne $opt_vmid);
3060         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3061     }
3062
3063     return $res;
3064 }
3065
3066 sub conf_has_serial {
3067     my ($conf) = @_;
3068
3069     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3070         if ($conf->{"serial$i"}) {
3071             return 1;
3072         }
3073     }
3074
3075     return 0;
3076 }
3077
3078 sub conf_has_audio {
3079     my ($conf, $id) = @_;
3080
3081     $id //= 0;
3082     my $audio = $conf->{"audio$id"};
3083     return if !defined($audio);
3084
3085     my $audioproperties = parse_property_string($audio_fmt, $audio);
3086     my $audiodriver = $audioproperties->{driver} // 'spice';
3087
3088     return {
3089         dev => $audioproperties->{device},
3090         dev_id => "audiodev$id",
3091         backend => $audiodriver,
3092         backend_id => "$audiodriver-backend${id}",
3093     };
3094 }
3095
3096 sub audio_devs {
3097     my ($audio, $audiopciaddr, $machine_version) = @_;
3098
3099     my $devs = [];
3100
3101     my $id = $audio->{dev_id};
3102     my $audiodev = "";
3103     if (min_version($machine_version, 4, 2)) {
3104         $audiodev = ",audiodev=$audio->{backend_id}";
3105     }
3106
3107     if ($audio->{dev} eq 'AC97') {
3108         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3109     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3110         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3111         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3112         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3113     } else {
3114         die "unkown audio device '$audio->{dev}', implement me!";
3115     }
3116
3117     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3118
3119     return $devs;
3120 }
3121
3122 sub get_tpm_paths {
3123     my ($vmid) = @_;
3124     return {
3125         socket => "/var/run/qemu-server/$vmid.swtpm",
3126         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3127     };
3128 }
3129
3130 sub add_tpm_device {
3131     my ($vmid, $devices, $conf) = @_;
3132
3133     return if !$conf->{tpmstate0};
3134
3135     my $paths = get_tpm_paths($vmid);
3136
3137     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3138     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3139     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3140 }
3141
3142 sub start_swtpm {
3143     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3144
3145     return if !$tpmdrive;
3146
3147     my $state;
3148     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3149     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3150     if ($storeid) {
3151         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3152     } else {
3153         $state = $tpm->{file};
3154     }
3155
3156     my $paths = get_tpm_paths($vmid);
3157
3158     # during migration, we will get state from remote
3159     #
3160     if (!$migration) {
3161         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3162         my $setup_cmd = [
3163             "swtpm_setup",
3164             "--tpmstate",
3165             "file://$state",
3166             "--createek",
3167             "--create-ek-cert",
3168             "--create-platform-cert",
3169             "--lock-nvram",
3170             "--config",
3171             "/etc/swtpm_setup.conf", # do not use XDG configs
3172             "--runas",
3173             "0", # force creation as root, error if not possible
3174             "--not-overwrite", # ignore existing state, do not modify
3175         ];
3176
3177         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3178         # TPM 2.0 supports ECC crypto, use if possible
3179         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3180
3181         run_command($setup_cmd, outfunc => sub {
3182             print "swtpm_setup: $1\n";
3183         });
3184     }
3185
3186     my $emulator_cmd = [
3187         "swtpm",
3188         "socket",
3189         "--tpmstate",
3190         "backend-uri=file://$state,mode=0600",
3191         "--ctrl",
3192         "type=unixio,path=$paths->{socket},mode=0600",
3193         "--pid",
3194         "file=$paths->{pid}",
3195         "--terminate", # terminate on QEMU disconnect
3196         "--daemon",
3197     ];
3198     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3199     run_command($emulator_cmd, outfunc => sub { print $1; });
3200
3201     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3202     while (! -e $paths->{pid}) {
3203         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3204         usleep(50_000);
3205     }
3206
3207     # return untainted PID of swtpm daemon so it can be killed on error
3208     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3209     return $1;
3210 }
3211
3212 sub vga_conf_has_spice {
3213     my ($vga) = @_;
3214
3215     my $vgaconf = parse_vga($vga);
3216     my $vgatype = $vgaconf->{type};
3217     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3218
3219     return $1 || 1;
3220 }
3221
3222 sub is_native($) {
3223     my ($arch) = @_;
3224     return get_host_arch() eq $arch;
3225 }
3226
3227 sub get_vm_arch {
3228     my ($conf) = @_;
3229     return $conf->{arch} // get_host_arch();
3230 }
3231
3232 my $default_machines = {
3233     x86_64 => 'pc',
3234     aarch64 => 'virt',
3235 };
3236
3237 sub get_installed_machine_version {
3238     my ($kvmversion) = @_;
3239     $kvmversion = kvm_user_version() if !defined($kvmversion);
3240     $kvmversion =~ m/^(\d+\.\d+)/;
3241     return $1;
3242 }
3243
3244 sub windows_get_pinned_machine_version {
3245     my ($machine, $base_version, $kvmversion) = @_;
3246
3247     my $pin_version = $base_version;
3248     if (!defined($base_version) ||
3249         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3250     ) {
3251         $pin_version = get_installed_machine_version($kvmversion);
3252     }
3253     if (!$machine || $machine eq 'pc') {
3254         $machine = "pc-i440fx-$pin_version";
3255     } elsif ($machine eq 'q35') {
3256         $machine = "pc-q35-$pin_version";
3257     } elsif ($machine eq 'virt') {
3258         $machine = "virt-$pin_version";
3259     } else {
3260         warn "unknown machine type '$machine', not touching that!\n";
3261     }
3262
3263     return $machine;
3264 }
3265
3266 sub get_vm_machine {
3267     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3268
3269     my $machine = $forcemachine || $conf->{machine};
3270
3271     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3272         $kvmversion //= kvm_user_version();
3273         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3274         # layout which confuses windows quite a bit and may result in various regressions..
3275         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3276         if (windows_version($conf->{ostype})) {
3277             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3278         }
3279         $arch //= 'x86_64';
3280         $machine ||= $default_machines->{$arch};
3281         if ($add_pve_version) {
3282             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3283             $machine .= "+pve$pvever";
3284         }
3285     }
3286
3287     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3288         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3289         $machine = $1 if $is_pxe;
3290
3291         # for version-pinned machines that do not include a pve-version (e.g.
3292         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3293         $machine .= '+pve0';
3294
3295         $machine .= '.pxe' if $is_pxe;
3296     }
3297
3298     return $machine;
3299 }
3300
3301 sub get_ovmf_files($$$) {
3302     my ($arch, $efidisk, $smm) = @_;
3303
3304     my $types = $OVMF->{$arch}
3305         or die "no OVMF images known for architecture '$arch'\n";
3306
3307     my $type = 'default';
3308     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3309         $type = $smm ? "4m" : "4m-no-smm";
3310         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3311     }
3312
3313     return $types->{$type}->@*;
3314 }
3315
3316 my $Arch2Qemu = {
3317     aarch64 => '/usr/bin/qemu-system-aarch64',
3318     x86_64 => '/usr/bin/qemu-system-x86_64',
3319 };
3320 sub get_command_for_arch($) {
3321     my ($arch) = @_;
3322     return '/usr/bin/kvm' if is_native($arch);
3323
3324     my $cmd = $Arch2Qemu->{$arch}
3325         or die "don't know how to emulate architecture '$arch'\n";
3326     return $cmd;
3327 }
3328
3329 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3330 # to use in a QEMU command line (-cpu element), first array_intersect the result
3331 # of query_supported_ with query_understood_. This is necessary because:
3332 #
3333 # a) query_understood_ returns flags the host cannot use and
3334 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3335 #    flags, but CPU settings - with most of them being flags. Those settings
3336 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3337 #
3338 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3339 # expensive. If you need the value returned from this, you can get it much
3340 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3341 # $accel being 'kvm' or 'tcg'.
3342 #
3343 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3344 # changes, automatically populating pmxcfs.
3345 #
3346 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3347 # since kvm and tcg machines support different flags
3348 #
3349 sub query_supported_cpu_flags {
3350     my ($arch) = @_;
3351
3352     $arch //= get_host_arch();
3353     my $default_machine = $default_machines->{$arch};
3354
3355     my $flags = {};
3356
3357     # FIXME: Once this is merged, the code below should work for ARM as well:
3358     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3359     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3360         $arch eq "aarch64";
3361
3362     my $kvm_supported = defined(kvm_version());
3363     my $qemu_cmd = get_command_for_arch($arch);
3364     my $fakevmid = -1;
3365     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3366
3367     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3368     my $query_supported_run_qemu = sub {
3369         my ($kvm) = @_;
3370
3371         my $flags = {};
3372         my $cmd = [
3373             $qemu_cmd,
3374             '-machine', $default_machine,
3375             '-display', 'none',
3376             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3377             '-mon', 'chardev=qmp,mode=control',
3378             '-pidfile', $pidfile,
3379             '-S', '-daemonize'
3380         ];
3381
3382         if (!$kvm) {
3383             push @$cmd, '-accel', 'tcg';
3384         }
3385
3386         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3387         die "QEMU flag querying VM exited with code " . $rc if $rc;
3388
3389         eval {
3390             my $cmd_result = mon_cmd(
3391                 $fakevmid,
3392                 'query-cpu-model-expansion',
3393                 type => 'full',
3394                 model => { name => 'host' }
3395             );
3396
3397             my $props = $cmd_result->{model}->{props};
3398             foreach my $prop (keys %$props) {
3399                 next if $props->{$prop} ne '1';
3400                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3401                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3402                 # We only keep those with underscores, to match /proc/cpuinfo
3403                 $prop =~ s/\.|-/_/g;
3404                 $flags->{$prop} = 1;
3405             }
3406         };
3407         my $err = $@;
3408
3409         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3410         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3411
3412         die $err if $err;
3413
3414         return [ sort keys %$flags ];
3415     };
3416
3417     # We need to query QEMU twice, since KVM and TCG have different supported flags
3418     PVE::QemuConfig->lock_config($fakevmid, sub {
3419         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3420         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3421
3422         if ($kvm_supported) {
3423             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3424             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3425         }
3426     });
3427
3428     return $flags;
3429 }
3430
3431 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3432 my $understood_cpu_flag_dir = "/usr/share/kvm";
3433 sub query_understood_cpu_flags {
3434     my $arch = get_host_arch();
3435     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3436
3437     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3438         if ! -e $filepath;
3439
3440     my $raw = file_get_contents($filepath);
3441     $raw =~ s/^\s+|\s+$//g;
3442     my @flags = split(/\s+/, $raw);
3443
3444     return \@flags;
3445 }
3446
3447 my sub get_cpuunits {
3448     my ($conf) = @_;
3449     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3450
3451     my $cpuunits = $conf->{cpuunits};
3452     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3453
3454     if ($is_cgroupv2) {
3455         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3456     } else {
3457         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3458     }
3459     return $cpuunits;
3460 }
3461
3462 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3463 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3464 my sub should_disable_smm {
3465     my ($conf, $vga) = @_;
3466
3467     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3468         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3469 }
3470
3471 sub config_to_command {
3472     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3473         $pbs_backing) = @_;
3474
3475     my $cmd = [];
3476     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3477     my $devices = [];
3478     my $bridges = {};
3479     my $ostype = $conf->{ostype};
3480     my $winversion = windows_version($ostype);
3481     my $kvm = $conf->{kvm};
3482     my $nodename = nodename();
3483
3484     my $arch = get_vm_arch($conf);
3485     my $kvm_binary = get_command_for_arch($arch);
3486     my $kvmver = kvm_user_version($kvm_binary);
3487
3488     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3489         $kvmver //= "undefined";
3490         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3491     }
3492
3493     my $add_pve_version = min_version($kvmver, 4, 1);
3494
3495     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3496     my $machine_version = extract_version($machine_type, $kvmver);
3497     $kvm //= 1 if is_native($arch);
3498
3499     $machine_version =~ m/(\d+)\.(\d+)/;
3500     my ($machine_major, $machine_minor) = ($1, $2);
3501
3502     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3503         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3504     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3505         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3506             ." please upgrade node '$nodename'\n"
3507     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3508         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3509         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3510             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3511             ." node '$nodename'\n";
3512     }
3513
3514     # if a specific +pve version is required for a feature, use $version_guard
3515     # instead of min_version to allow machines to be run with the minimum
3516     # required version
3517     my $required_pve_version = 0;
3518     my $version_guard = sub {
3519         my ($major, $minor, $pve) = @_;
3520         return 0 if !min_version($machine_version, $major, $minor, $pve);
3521         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3522         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3523         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3524         return 1;
3525     };
3526
3527     if ($kvm && !defined kvm_version()) {
3528         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3529             ." or enable in BIOS.\n";
3530     }
3531
3532     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3533     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3534     my $use_old_bios_files = undef;
3535     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3536
3537     push @$cmd, $kvm_binary;
3538
3539     push @$cmd, '-id', $vmid;
3540
3541     my $vmname = $conf->{name} || "vm$vmid";
3542
3543     push @$cmd, '-name', $vmname;
3544
3545     push @$cmd, '-no-shutdown';
3546
3547     my $use_virtio = 0;
3548
3549     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3550     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3551     push @$cmd, '-mon', "chardev=qmp,mode=control";
3552
3553     if (min_version($machine_version, 2, 12)) {
3554         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3555         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3556     }
3557
3558     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3559
3560     push @$cmd, '-daemonize';
3561
3562     if ($conf->{smbios1}) {
3563         my $smbios_conf = parse_smbios1($conf->{smbios1});
3564         if ($smbios_conf->{base64}) {
3565             # Do not pass base64 flag to qemu
3566             delete $smbios_conf->{base64};
3567             my $smbios_string = "";
3568             foreach my $key (keys %$smbios_conf) {
3569                 my $value;
3570                 if ($key eq "uuid") {
3571                     $value = $smbios_conf->{uuid}
3572                 } else {
3573                     $value = decode_base64($smbios_conf->{$key});
3574                 }
3575                 # qemu accepts any binary data, only commas need escaping by double comma
3576                 $value =~ s/,/,,/g;
3577                 $smbios_string .= "," . $key . "=" . $value if $value;
3578             }
3579             push @$cmd, '-smbios', "type=1" . $smbios_string;
3580         } else {
3581             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3582         }
3583     }
3584
3585     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3586         my $d;
3587         if (my $efidisk = $conf->{efidisk0}) {
3588             $d = parse_drive('efidisk0', $efidisk);
3589         }
3590
3591         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3592         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3593
3594         my ($path, $format);
3595         my $read_only_str = '';
3596         if ($d) {
3597             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3598             $format = $d->{format};
3599             if ($storeid) {
3600                 $path = PVE::Storage::path($storecfg, $d->{file});
3601                 if (!defined($format)) {
3602                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3603                     $format = qemu_img_format($scfg, $volname);
3604                 }
3605             } else {
3606                 $path = $d->{file};
3607                 die "efidisk format must be specified\n"
3608                     if !defined($format);
3609             }
3610
3611             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3612         } else {
3613             warn "no efidisk configured! Using temporary efivars disk.\n";
3614             $path = "/tmp/$vmid-ovmf.fd";
3615             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3616             $format = 'raw';
3617         }
3618
3619         my $size_str = "";
3620
3621         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3622             $size_str = ",size=" . (-s $ovmf_vars);
3623         }
3624
3625         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3626         my $cache = "";
3627         if ($path =~ m/^rbd:/) {
3628                 $cache = ',cache=writeback';
3629                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3630         }
3631
3632         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3633         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3634     }
3635
3636     if ($q35) { # tell QEMU to load q35 config early
3637         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3638         if (min_version($machine_version, 4, 0)) {
3639             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3640         } else {
3641             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3642         }
3643     }
3644
3645     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3646         push @$cmd, $fixups->@*;
3647     }
3648
3649     if ($conf->{vmgenid}) {
3650         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3651     }
3652
3653     # add usb controllers
3654     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3655         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3656     push @$devices, @usbcontrollers if @usbcontrollers;
3657     my $vga = parse_vga($conf->{vga});
3658
3659     my $qxlnum = vga_conf_has_spice($conf->{vga});
3660     $vga->{type} = 'qxl' if $qxlnum;
3661
3662     if (!$vga->{type}) {
3663         if ($arch eq 'aarch64') {
3664             $vga->{type} = 'virtio';
3665         } elsif (min_version($machine_version, 2, 9)) {
3666             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3667         } else {
3668             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3669         }
3670     }
3671
3672     # enable absolute mouse coordinates (needed by vnc)
3673     my $tablet = $conf->{tablet};
3674     if (!defined($tablet)) {
3675         $tablet = $defaults->{tablet};
3676         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3677         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3678     }
3679
3680     if ($tablet) {
3681         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3682         my $kbd = print_keyboarddevice_full($conf, $arch);
3683         push @$devices, '-device', $kbd if defined($kbd);
3684     }
3685
3686     my $bootorder = device_bootorder($conf);
3687
3688     # host pci device passthrough
3689     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3690         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3691
3692     # usb devices
3693     my $usb_dev_features = {};
3694     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3695
3696     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3697         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3698     push @$devices, @usbdevices if @usbdevices;
3699
3700     # serial devices
3701     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3702         my $path = $conf->{"serial$i"} or next;
3703         if ($path eq 'socket') {
3704             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3705             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3706             # On aarch64, serial0 is the UART device. Qemu only allows
3707             # connecting UART devices via the '-serial' command line, as
3708             # the device has a fixed slot on the hardware...
3709             if ($arch eq 'aarch64' && $i == 0) {
3710                 push @$devices, '-serial', "chardev:serial$i";
3711             } else {
3712                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3713             }
3714         } else {
3715             die "no such serial device\n" if ! -c $path;
3716             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3717             push @$devices, '-device', "isa-serial,chardev=serial$i";
3718         }
3719     }
3720
3721     # parallel devices
3722     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3723         if (my $path = $conf->{"parallel$i"}) {
3724             die "no such parallel device\n" if ! -c $path;
3725             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3726             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3727             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3728         }
3729     }
3730
3731     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3732         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3733         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3734         push @$devices, @$audio_devs;
3735     }
3736
3737     add_tpm_device($vmid, $devices, $conf);
3738
3739     my $sockets = 1;
3740     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3741     $sockets = $conf->{sockets} if  $conf->{sockets};
3742
3743     my $cores = $conf->{cores} || 1;
3744
3745     my $maxcpus = $sockets * $cores;
3746
3747     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3748
3749     my $allowed_vcpus = $cpuinfo->{cpus};
3750
3751     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3752
3753     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3754         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3755         for (my $i = 2; $i <= $vcpus; $i++)  {
3756             my $cpustr = print_cpu_device($conf,$i);
3757             push @$cmd, '-device', $cpustr;
3758         }
3759
3760     } else {
3761
3762         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3763     }
3764     push @$cmd, '-nodefaults';
3765
3766     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3767
3768     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3769
3770     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3771
3772     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3773         push @$devices, '-device', print_vga_device(
3774             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3775
3776         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3777
3778         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3779         push @$cmd,  '-vnc', "unix:$socket,password=on";
3780     } else {
3781         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3782         push @$cmd, '-nographic';
3783     }
3784
3785     # time drift fix
3786     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3787     my $useLocaltime = $conf->{localtime};
3788
3789     if ($winversion >= 5) { # windows
3790         $useLocaltime = 1 if !defined($conf->{localtime});
3791
3792         # use time drift fix when acpi is enabled
3793         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3794             $tdf = 1 if !defined($conf->{tdf});
3795         }
3796     }
3797
3798     if ($winversion >= 6) {
3799         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3800         push @$cmd, '-no-hpet';
3801     }
3802
3803     push @$rtcFlags, 'driftfix=slew' if $tdf;
3804
3805     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3806         push @$rtcFlags, "base=$conf->{startdate}";
3807     } elsif ($useLocaltime) {
3808         push @$rtcFlags, 'base=localtime';
3809     }
3810
3811     if ($forcecpu) {
3812         push @$cmd, '-cpu', $forcecpu;
3813     } else {
3814         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3815     }
3816
3817     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3818
3819     push @$cmd, '-S' if $conf->{freeze};
3820
3821     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3822
3823     my $guest_agent = parse_guest_agent($conf);
3824
3825     if ($guest_agent->{enabled}) {
3826         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3827         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3828
3829         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3830             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3831             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3832             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3833         } elsif ($guest_agent->{type} eq 'isa') {
3834             push @$devices, '-device', "isa-serial,chardev=qga0";
3835         }
3836     }
3837
3838     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3839     if ($rng && $version_guard->(4, 1, 2)) {
3840         check_rng_source($rng->{source});
3841
3842         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3843         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3844         my $limiter_str = "";
3845         if ($max_bytes) {
3846             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3847         }
3848
3849         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3850         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3851         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3852     }
3853
3854     my $spice_port;
3855
3856     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3857         if ($qxlnum > 1) {
3858             if ($winversion){
3859                 for (my $i = 1; $i < $qxlnum; $i++){
3860                     push @$devices, '-device', print_vga_device(
3861                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3862                 }
3863             } else {
3864                 # assume other OS works like Linux
3865                 my ($ram, $vram) = ("134217728", "67108864");
3866                 if ($vga->{memory}) {
3867                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3868                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3869                 }
3870                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3871                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3872             }
3873         }
3874
3875         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3876
3877         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3878         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3879         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3880
3881         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3882         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3883         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3884
3885         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3886         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3887
3888         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3889         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3890         if ($spice_enhancement->{foldersharing}) {
3891             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3892             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3893         }
3894
3895         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3896         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3897             if $spice_enhancement->{videostreaming};
3898
3899         push @$devices, '-spice', "$spice_opts";
3900     }
3901
3902     # enable balloon by default, unless explicitly disabled
3903     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3904         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3905         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3906         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3907         push @$devices, '-device', $ballooncmd;
3908     }
3909
3910     if ($conf->{watchdog}) {
3911         my $wdopts = parse_watchdog($conf->{watchdog});
3912         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3913         my $watchdog = $wdopts->{model} || 'i6300esb';
3914         push @$devices, '-device', "$watchdog$pciaddr";
3915         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3916     }
3917
3918     my $vollist = [];
3919     my $scsicontroller = {};
3920     my $ahcicontroller = {};
3921     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3922
3923     # Add iscsi initiator name if available
3924     if (my $initiator = get_initiator_name()) {
3925         push @$devices, '-iscsi', "initiator-name=$initiator";
3926     }
3927
3928     PVE::QemuConfig->foreach_volume($conf, sub {
3929         my ($ds, $drive) = @_;
3930
3931         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3932             check_volume_storage_type($storecfg, $drive->{file});
3933             push @$vollist, $drive->{file};
3934         }
3935
3936         # ignore efidisk here, already added in bios/fw handling code above
3937         return if $drive->{interface} eq 'efidisk';
3938         # similar for TPM
3939         return if $drive->{interface} eq 'tpmstate';
3940
3941         $use_virtio = 1 if $ds =~ m/^virtio/;
3942
3943         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3944
3945         if ($drive->{interface} eq 'virtio'){
3946            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3947         }
3948
3949         if ($drive->{interface} eq 'scsi') {
3950
3951             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3952
3953             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3954                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3955
3956             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3957             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3958
3959             my $iothread = '';
3960             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3961                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3962                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3963             } elsif ($drive->{iothread}) {
3964                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3965             }
3966
3967             my $queues = '';
3968             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3969                 $queues = ",num_queues=$drive->{queues}";
3970             }
3971
3972             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3973                 if !$scsicontroller->{$controller};
3974             $scsicontroller->{$controller}=1;
3975         }
3976
3977         if ($drive->{interface} eq 'sata') {
3978             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3979             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3980             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3981                 if !$ahcicontroller->{$controller};
3982             $ahcicontroller->{$controller}=1;
3983         }
3984
3985         my $pbs_conf = $pbs_backing->{$ds};
3986         my $pbs_name = undef;
3987         if ($pbs_conf) {
3988             $pbs_name = "drive-$ds-pbs";
3989             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3990         }
3991
3992         my $drive_cmd = print_drive_commandline_full(
3993             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3994
3995         # extra protection for templates, but SATA and IDE don't support it..
3996         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3997
3998         push @$devices, '-drive',$drive_cmd;
3999         push @$devices, '-device', print_drivedevice_full(
4000             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4001     });
4002
4003     for (my $i = 0; $i < $MAX_NETS; $i++) {
4004         my $netname = "net$i";
4005
4006         next if !$conf->{$netname};
4007         my $d = parse_net($conf->{$netname});
4008         next if !$d;
4009
4010         $use_virtio = 1 if $d->{model} eq 'virtio';
4011
4012         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4013
4014         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4015         push @$devices, '-netdev', $netdevfull;
4016
4017         my $netdevicefull = print_netdevice_full(
4018             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4019
4020         push @$devices, '-device', $netdevicefull;
4021     }
4022
4023     if ($conf->{ivshmem}) {
4024         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4025
4026         my $bus;
4027         if ($q35) {
4028             $bus = print_pcie_addr("ivshmem");
4029         } else {
4030             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4031         }
4032
4033         my $ivshmem_name = $ivshmem->{name} // $vmid;
4034         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4035
4036         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4037         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4038             .",size=$ivshmem->{size}M";
4039     }
4040
4041     # pci.4 is nested in pci.1
4042     $bridges->{1} = 1 if $bridges->{4};
4043
4044     if (!$q35) { # add pci bridges
4045         if (min_version($machine_version, 2, 3)) {
4046            $bridges->{1} = 1;
4047            $bridges->{2} = 1;
4048         }
4049         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4050     }
4051
4052     for my $k (sort {$b cmp $a} keys %$bridges) {
4053         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4054
4055         my $k_name = $k;
4056         if ($k == 2 && $legacy_igd) {
4057             $k_name = "$k-igd";
4058         }
4059         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4060         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4061
4062         if ($q35) { # add after -readconfig pve-q35.cfg
4063             splice @$devices, 2, 0, '-device', $devstr;
4064         } else {
4065             unshift @$devices, '-device', $devstr if $k > 0;
4066         }
4067     }
4068
4069     if (!$kvm) {
4070         push @$machineFlags, 'accel=tcg';
4071     }
4072
4073     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4074
4075     my $machine_type_min = $machine_type;
4076     if ($add_pve_version) {
4077         $machine_type_min =~ s/\+pve\d+$//;
4078         $machine_type_min .= "+pve$required_pve_version";
4079     }
4080     push @$machineFlags, "type=${machine_type_min}";
4081
4082     push @$cmd, @$devices;
4083     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4084     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4085     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4086
4087     if (my $vmstate = $conf->{vmstate}) {
4088         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4089         push @$vollist, $vmstate;
4090         push @$cmd, '-loadstate', $statepath;
4091         print "activating and using '$vmstate' as vmstate\n";
4092     }
4093
4094     if (PVE::QemuConfig->is_template($conf)) {
4095         # needed to workaround base volumes being read-only
4096         push @$cmd, '-snapshot';
4097     }
4098
4099     # add custom args
4100     if ($conf->{args}) {
4101         my $aa = PVE::Tools::split_args($conf->{args});
4102         push @$cmd, @$aa;
4103     }
4104
4105     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4106 }
4107
4108 sub check_rng_source {
4109     my ($source) = @_;
4110
4111     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4112     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4113         if ! -e $source;
4114
4115     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4116     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4117         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4118         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4119         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4120             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4121             ." to the host.\n";
4122     }
4123 }
4124
4125 sub spice_port {
4126     my ($vmid) = @_;
4127
4128     my $res = mon_cmd($vmid, 'query-spice');
4129
4130     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4131 }
4132
4133 sub vm_devices_list {
4134     my ($vmid) = @_;
4135
4136     my $res = mon_cmd($vmid, 'query-pci');
4137     my $devices_to_check = [];
4138     my $devices = {};
4139     foreach my $pcibus (@$res) {
4140         push @$devices_to_check, @{$pcibus->{devices}},
4141     }
4142
4143     while (@$devices_to_check) {
4144         my $to_check = [];
4145         for my $d (@$devices_to_check) {
4146             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4147             next if !$d->{'pci_bridge'};
4148
4149             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4150             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4151         }
4152         $devices_to_check = $to_check;
4153     }
4154
4155     my $resblock = mon_cmd($vmid, 'query-block');
4156     foreach my $block (@$resblock) {
4157         if($block->{device} =~ m/^drive-(\S+)/){
4158                 $devices->{$1} = 1;
4159         }
4160     }
4161
4162     my $resmice = mon_cmd($vmid, 'query-mice');
4163     foreach my $mice (@$resmice) {
4164         if ($mice->{name} eq 'QEMU HID Tablet') {
4165             $devices->{tablet} = 1;
4166             last;
4167         }
4168     }
4169
4170     # for usb devices there is no query-usb
4171     # but we can iterate over the entries in
4172     # qom-list path=/machine/peripheral
4173     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4174     foreach my $per (@$resperipheral) {
4175         if ($per->{name} =~ m/^usb\d+$/) {
4176             $devices->{$per->{name}} = 1;
4177         }
4178     }
4179
4180     return $devices;
4181 }
4182
4183 sub vm_deviceplug {
4184     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4185
4186     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4187
4188     my $devices_list = vm_devices_list($vmid);
4189     return 1 if defined($devices_list->{$deviceid});
4190
4191     # add PCI bridge if we need it for the device
4192     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4193
4194     if ($deviceid eq 'tablet') {
4195         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4196     } elsif ($deviceid eq 'keyboard') {
4197         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4198     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4199         die "usb hotplug currently not reliable\n";
4200         # since we can't reliably hot unplug all added usb devices and usb
4201         # passthrough breaks live migration we disable usb hotplugging for now
4202         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4203     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4204         qemu_iothread_add($vmid, $deviceid, $device);
4205
4206         qemu_driveadd($storecfg, $vmid, $device);
4207         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4208
4209         qemu_deviceadd($vmid, $devicefull);
4210         eval { qemu_deviceaddverify($vmid, $deviceid); };
4211         if (my $err = $@) {
4212             eval { qemu_drivedel($vmid, $deviceid); };
4213             warn $@ if $@;
4214             die $err;
4215         }
4216     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4217         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4218         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4219         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4220
4221         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4222
4223         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4224             qemu_iothread_add($vmid, $deviceid, $device);
4225             $devicefull .= ",iothread=iothread-$deviceid";
4226         }
4227
4228         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4229             $devicefull .= ",num_queues=$device->{queues}";
4230         }
4231
4232         qemu_deviceadd($vmid, $devicefull);
4233         qemu_deviceaddverify($vmid, $deviceid);
4234     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4235         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4236         qemu_driveadd($storecfg, $vmid, $device);
4237
4238         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4239         eval { qemu_deviceadd($vmid, $devicefull); };
4240         if (my $err = $@) {
4241             eval { qemu_drivedel($vmid, $deviceid); };
4242             warn $@ if $@;
4243             die $err;
4244         }
4245     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4246         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4247
4248         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4249         my $use_old_bios_files = undef;
4250         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4251
4252         my $netdevicefull = print_netdevice_full(
4253             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4254         qemu_deviceadd($vmid, $netdevicefull);
4255         eval {
4256             qemu_deviceaddverify($vmid, $deviceid);
4257             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4258         };
4259         if (my $err = $@) {
4260             eval { qemu_netdevdel($vmid, $deviceid); };
4261             warn $@ if $@;
4262             die $err;
4263         }
4264     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4265         my $bridgeid = $2;
4266         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4267         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4268
4269         qemu_deviceadd($vmid, $devicefull);
4270         qemu_deviceaddverify($vmid, $deviceid);
4271     } else {
4272         die "can't hotplug device '$deviceid'\n";
4273     }
4274
4275     return 1;
4276 }
4277
4278 # fixme: this should raise exceptions on error!
4279 sub vm_deviceunplug {
4280     my ($vmid, $conf, $deviceid) = @_;
4281
4282     my $devices_list = vm_devices_list($vmid);
4283     return 1 if !defined($devices_list->{$deviceid});
4284
4285     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4286     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4287
4288     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4289         qemu_devicedel($vmid, $deviceid);
4290     } elsif ($deviceid =~ m/^usb\d+$/) {
4291         die "usb hotplug currently not reliable\n";
4292         # when unplugging usb devices this way, there may be remaining usb
4293         # controllers/hubs so we disable it for now
4294         #qemu_devicedel($vmid, $deviceid);
4295         #qemu_devicedelverify($vmid, $deviceid);
4296     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4297         my $device = parse_drive($deviceid, $conf->{$deviceid});
4298
4299         qemu_devicedel($vmid, $deviceid);
4300         qemu_devicedelverify($vmid, $deviceid);
4301         qemu_drivedel($vmid, $deviceid);
4302         qemu_iothread_del($vmid, $deviceid, $device);
4303     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4304         qemu_devicedel($vmid, $deviceid);
4305         qemu_devicedelverify($vmid, $deviceid);
4306     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4307         my $device = parse_drive($deviceid, $conf->{$deviceid});
4308
4309         qemu_devicedel($vmid, $deviceid);
4310         qemu_devicedelverify($vmid, $deviceid);
4311         qemu_drivedel($vmid, $deviceid);
4312         qemu_deletescsihw($conf, $vmid, $deviceid);
4313
4314         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4315             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4316     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4317         qemu_devicedel($vmid, $deviceid);
4318         qemu_devicedelverify($vmid, $deviceid);
4319         qemu_netdevdel($vmid, $deviceid);
4320     } else {
4321         die "can't unplug device '$deviceid'\n";
4322     }
4323
4324     return 1;
4325 }
4326
4327 sub qemu_deviceadd {
4328     my ($vmid, $devicefull) = @_;
4329
4330     $devicefull = "driver=".$devicefull;
4331     my %options =  split(/[=,]/, $devicefull);
4332
4333     mon_cmd($vmid, "device_add" , %options);
4334 }
4335
4336 sub qemu_devicedel {
4337     my ($vmid, $deviceid) = @_;
4338
4339     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4340 }
4341
4342 sub qemu_iothread_add {
4343     my ($vmid, $deviceid, $device) = @_;
4344
4345     if ($device->{iothread}) {
4346         my $iothreads = vm_iothreads_list($vmid);
4347         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4348     }
4349 }
4350
4351 sub qemu_iothread_del {
4352     my ($vmid, $deviceid, $device) = @_;
4353
4354     if ($device->{iothread}) {
4355         my $iothreads = vm_iothreads_list($vmid);
4356         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4357     }
4358 }
4359
4360 sub qemu_objectadd {
4361     my ($vmid, $objectid, $qomtype) = @_;
4362
4363     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4364
4365     return 1;
4366 }
4367
4368 sub qemu_objectdel {
4369     my ($vmid, $objectid) = @_;
4370
4371     mon_cmd($vmid, "object-del", id => $objectid);
4372
4373     return 1;
4374 }
4375
4376 sub qemu_driveadd {
4377     my ($storecfg, $vmid, $device) = @_;
4378
4379     my $kvmver = get_running_qemu_version($vmid);
4380     my $io_uring = min_version($kvmver, 6, 0);
4381     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4382     $drive =~ s/\\/\\\\/g;
4383     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4384
4385     # If the command succeeds qemu prints: "OK"
4386     return 1 if $ret =~ m/OK/s;
4387
4388     die "adding drive failed: $ret\n";
4389 }
4390
4391 sub qemu_drivedel {
4392     my ($vmid, $deviceid) = @_;
4393
4394     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4395     $ret =~ s/^\s+//;
4396
4397     return 1 if $ret eq "";
4398
4399     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4400     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4401
4402     die "deleting drive $deviceid failed : $ret\n";
4403 }
4404
4405 sub qemu_deviceaddverify {
4406     my ($vmid, $deviceid) = @_;
4407
4408     for (my $i = 0; $i <= 5; $i++) {
4409          my $devices_list = vm_devices_list($vmid);
4410          return 1 if defined($devices_list->{$deviceid});
4411          sleep 1;
4412     }
4413
4414     die "error on hotplug device '$deviceid'\n";
4415 }
4416
4417
4418 sub qemu_devicedelverify {
4419     my ($vmid, $deviceid) = @_;
4420
4421     # need to verify that the device is correctly removed as device_del
4422     # is async and empty return is not reliable
4423
4424     for (my $i = 0; $i <= 5; $i++) {
4425          my $devices_list = vm_devices_list($vmid);
4426          return 1 if !defined($devices_list->{$deviceid});
4427          sleep 1;
4428     }
4429
4430     die "error on hot-unplugging device '$deviceid'\n";
4431 }
4432
4433 sub qemu_findorcreatescsihw {
4434     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4435
4436     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4437
4438     my $scsihwid="$controller_prefix$controller";
4439     my $devices_list = vm_devices_list($vmid);
4440
4441     if (!defined($devices_list->{$scsihwid})) {
4442         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4443     }
4444
4445     return 1;
4446 }
4447
4448 sub qemu_deletescsihw {
4449     my ($conf, $vmid, $opt) = @_;
4450
4451     my $device = parse_drive($opt, $conf->{$opt});
4452
4453     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4454         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4455         return 1;
4456     }
4457
4458     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4459
4460     my $devices_list = vm_devices_list($vmid);
4461     foreach my $opt (keys %{$devices_list}) {
4462         if (is_valid_drivename($opt)) {
4463             my $drive = parse_drive($opt, $conf->{$opt});
4464             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4465                 return 1;
4466             }
4467         }
4468     }
4469
4470     my $scsihwid="scsihw$controller";
4471
4472     vm_deviceunplug($vmid, $conf, $scsihwid);
4473
4474     return 1;
4475 }
4476
4477 sub qemu_add_pci_bridge {
4478     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4479
4480     my $bridges = {};
4481
4482     my $bridgeid;
4483
4484     print_pci_addr($device, $bridges, $arch, $machine_type);
4485
4486     while (my ($k, $v) = each %$bridges) {
4487         $bridgeid = $k;
4488     }
4489     return 1 if !defined($bridgeid) || $bridgeid < 1;
4490
4491     my $bridge = "pci.$bridgeid";
4492     my $devices_list = vm_devices_list($vmid);
4493
4494     if (!defined($devices_list->{$bridge})) {
4495         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4496     }
4497
4498     return 1;
4499 }
4500
4501 sub qemu_set_link_status {
4502     my ($vmid, $device, $up) = @_;
4503
4504     mon_cmd($vmid, "set_link", name => $device,
4505                up => $up ? JSON::true : JSON::false);
4506 }
4507
4508 sub qemu_netdevadd {
4509     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4510
4511     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4512     my %options =  split(/[=,]/, $netdev);
4513
4514     if (defined(my $vhost = $options{vhost})) {
4515         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4516     }
4517
4518     if (defined(my $queues = $options{queues})) {
4519         $options{queues} = $queues + 0;
4520     }
4521
4522     mon_cmd($vmid, "netdev_add",  %options);
4523     return 1;
4524 }
4525
4526 sub qemu_netdevdel {
4527     my ($vmid, $deviceid) = @_;
4528
4529     mon_cmd($vmid, "netdev_del", id => $deviceid);
4530 }
4531
4532 sub qemu_usb_hotplug {
4533     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4534
4535     return if !$device;
4536
4537     # remove the old one first
4538     vm_deviceunplug($vmid, $conf, $deviceid);
4539
4540     # check if xhci controller is necessary and available
4541     if ($device->{usb3}) {
4542
4543         my $devicelist = vm_devices_list($vmid);
4544
4545         if (!$devicelist->{xhci}) {
4546             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4547             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4548         }
4549     }
4550     my $d = parse_usb_device($device->{host});
4551     $d->{usb3} = $device->{usb3};
4552
4553     # add the new one
4554     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4555 }
4556
4557 sub qemu_cpu_hotplug {
4558     my ($vmid, $conf, $vcpus) = @_;
4559
4560     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4561
4562     my $sockets = 1;
4563     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4564     $sockets = $conf->{sockets} if  $conf->{sockets};
4565     my $cores = $conf->{cores} || 1;
4566     my $maxcpus = $sockets * $cores;
4567
4568     $vcpus = $maxcpus if !$vcpus;
4569
4570     die "you can't add more vcpus than maxcpus\n"
4571         if $vcpus > $maxcpus;
4572
4573     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4574
4575     if ($vcpus < $currentvcpus) {
4576
4577         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4578
4579             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4580                 qemu_devicedel($vmid, "cpu$i");
4581                 my $retry = 0;
4582                 my $currentrunningvcpus = undef;
4583                 while (1) {
4584                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4585                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4586                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4587                     $retry++;
4588                     sleep 1;
4589                 }
4590                 #update conf after each succesfull cpu unplug
4591                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4592                 PVE::QemuConfig->write_config($vmid, $conf);
4593             }
4594         } else {
4595             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4596         }
4597
4598         return;
4599     }
4600
4601     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4602     die "vcpus in running vm does not match its configuration\n"
4603         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4604
4605     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4606
4607         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4608             my $cpustr = print_cpu_device($conf, $i);
4609             qemu_deviceadd($vmid, $cpustr);
4610
4611             my $retry = 0;
4612             my $currentrunningvcpus = undef;
4613             while (1) {
4614                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4615                 last if scalar(@{$currentrunningvcpus}) == $i;
4616                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4617                 sleep 1;
4618                 $retry++;
4619             }
4620             #update conf after each succesfull cpu hotplug
4621             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4622             PVE::QemuConfig->write_config($vmid, $conf);
4623         }
4624     } else {
4625
4626         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4627             mon_cmd($vmid, "cpu-add", id => int($i));
4628         }
4629     }
4630 }
4631
4632 sub qemu_block_set_io_throttle {
4633     my ($vmid, $deviceid,
4634         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4635         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4636         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4637         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4638
4639     return if !check_running($vmid) ;
4640
4641     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4642         bps => int($bps),
4643         bps_rd => int($bps_rd),
4644         bps_wr => int($bps_wr),
4645         iops => int($iops),
4646         iops_rd => int($iops_rd),
4647         iops_wr => int($iops_wr),
4648         bps_max => int($bps_max),
4649         bps_rd_max => int($bps_rd_max),
4650         bps_wr_max => int($bps_wr_max),
4651         iops_max => int($iops_max),
4652         iops_rd_max => int($iops_rd_max),
4653         iops_wr_max => int($iops_wr_max),
4654         bps_max_length => int($bps_max_length),
4655         bps_rd_max_length => int($bps_rd_max_length),
4656         bps_wr_max_length => int($bps_wr_max_length),
4657         iops_max_length => int($iops_max_length),
4658         iops_rd_max_length => int($iops_rd_max_length),
4659         iops_wr_max_length => int($iops_wr_max_length),
4660     );
4661
4662 }
4663
4664 sub qemu_block_resize {
4665     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4666
4667     my $running = check_running($vmid);
4668
4669     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4670
4671     return if !$running;
4672
4673     my $padding = (1024 - $size % 1024) % 1024;
4674     $size = $size + $padding;
4675
4676     mon_cmd(
4677         $vmid,
4678         "block_resize",
4679         device => $deviceid,
4680         size => int($size),
4681         timeout => 60,
4682     );
4683 }
4684
4685 sub qemu_volume_snapshot {
4686     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4687
4688     my $running = check_running($vmid);
4689
4690     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4691         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4692     } else {
4693         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4694     }
4695 }
4696
4697 sub qemu_volume_snapshot_delete {
4698     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4699
4700     my $running = check_running($vmid);
4701
4702     if($running) {
4703
4704         $running = undef;
4705         my $conf = PVE::QemuConfig->load_config($vmid);
4706         PVE::QemuConfig->foreach_volume($conf, sub {
4707             my ($ds, $drive) = @_;
4708             $running = 1 if $drive->{file} eq $volid;
4709         });
4710     }
4711
4712     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4713         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4714     } else {
4715         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4716     }
4717 }
4718
4719 sub set_migration_caps {
4720     my ($vmid, $savevm) = @_;
4721
4722     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4723
4724     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4725     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4726
4727     my $cap_ref = [];
4728
4729     my $enabled_cap = {
4730         "auto-converge" => 1,
4731         "xbzrle" => 1,
4732         "x-rdma-pin-all" => 0,
4733         "zero-blocks" => 0,
4734         "compress" => 0,
4735         "dirty-bitmaps" => $dirty_bitmaps,
4736     };
4737
4738     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4739
4740     for my $supported_capability (@$supported_capabilities) {
4741         push @$cap_ref, {
4742             capability => $supported_capability->{capability},
4743             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4744         };
4745     }
4746
4747     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4748 }
4749
4750 sub foreach_volid {
4751     my ($conf, $func, @param) = @_;
4752
4753     my $volhash = {};
4754
4755     my $test_volid = sub {
4756         my ($key, $drive, $snapname) = @_;
4757
4758         my $volid = $drive->{file};
4759         return if !$volid;
4760
4761         $volhash->{$volid}->{cdrom} //= 1;
4762         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4763
4764         my $replicate = $drive->{replicate} // 1;
4765         $volhash->{$volid}->{replicate} //= 0;
4766         $volhash->{$volid}->{replicate} = 1 if $replicate;
4767
4768         $volhash->{$volid}->{shared} //= 0;
4769         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4770
4771         $volhash->{$volid}->{referenced_in_config} //= 0;
4772         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4773
4774         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4775             if defined($snapname);
4776
4777         my $size = $drive->{size};
4778         $volhash->{$volid}->{size} //= $size if $size;
4779
4780         $volhash->{$volid}->{is_vmstate} //= 0;
4781         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4782
4783         $volhash->{$volid}->{is_tpmstate} //= 0;
4784         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4785
4786         $volhash->{$volid}->{is_unused} //= 0;
4787         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4788
4789         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4790     };
4791
4792     my $include_opts = {
4793         extra_keys => ['vmstate'],
4794         include_unused => 1,
4795     };
4796
4797     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4798     foreach my $snapname (keys %{$conf->{snapshots}}) {
4799         my $snap = $conf->{snapshots}->{$snapname};
4800         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4801     }
4802
4803     foreach my $volid (keys %$volhash) {
4804         &$func($volid, $volhash->{$volid}, @param);
4805     }
4806 }
4807
4808 my $fast_plug_option = {
4809     'lock' => 1,
4810     'name' => 1,
4811     'onboot' => 1,
4812     'shares' => 1,
4813     'startup' => 1,
4814     'description' => 1,
4815     'protection' => 1,
4816     'vmstatestorage' => 1,
4817     'hookscript' => 1,
4818     'tags' => 1,
4819 };
4820
4821 # hotplug changes in [PENDING]
4822 # $selection hash can be used to only apply specified options, for
4823 # example: { cores => 1 } (only apply changed 'cores')
4824 # $errors ref is used to return error messages
4825 sub vmconfig_hotplug_pending {
4826     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4827
4828     my $defaults = load_defaults();
4829     my $arch = get_vm_arch($conf);
4830     my $machine_type = get_vm_machine($conf, undef, $arch);
4831
4832     # commit values which do not have any impact on running VM first
4833     # Note: those option cannot raise errors, we we do not care about
4834     # $selection and always apply them.
4835
4836     my $add_error = sub {
4837         my ($opt, $msg) = @_;
4838         $errors->{$opt} = "hotplug problem - $msg";
4839     };
4840
4841     my $changes = 0;
4842     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4843         if ($fast_plug_option->{$opt}) {
4844             $conf->{$opt} = $conf->{pending}->{$opt};
4845             delete $conf->{pending}->{$opt};
4846             $changes = 1;
4847         }
4848     }
4849
4850     if ($changes) {
4851         PVE::QemuConfig->write_config($vmid, $conf);
4852     }
4853
4854     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4855
4856     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4857     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4858     foreach my $opt (sort keys %$pending_delete_hash) {
4859         next if $selection && !$selection->{$opt};
4860         my $force = $pending_delete_hash->{$opt}->{force};
4861         eval {
4862             if ($opt eq 'hotplug') {
4863                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4864             } elsif ($opt eq 'tablet') {
4865                 die "skip\n" if !$hotplug_features->{usb};
4866                 if ($defaults->{tablet}) {
4867                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4868                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4869                         if $arch eq 'aarch64';
4870                 } else {
4871                     vm_deviceunplug($vmid, $conf, 'tablet');
4872                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4873                 }
4874             } elsif ($opt =~ m/^usb\d+/) {
4875                 die "skip\n";
4876                 # since we cannot reliably hot unplug usb devices we are disabling it
4877                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4878                 #vm_deviceunplug($vmid, $conf, $opt);
4879             } elsif ($opt eq 'vcpus') {
4880                 die "skip\n" if !$hotplug_features->{cpu};
4881                 qemu_cpu_hotplug($vmid, $conf, undef);
4882             } elsif ($opt eq 'balloon') {
4883                 # enable balloon device is not hotpluggable
4884                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4885                 # here we reset the ballooning value to memory
4886                 my $balloon = $conf->{memory} || $defaults->{memory};
4887                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4888             } elsif ($fast_plug_option->{$opt}) {
4889                 # do nothing
4890             } elsif ($opt =~ m/^net(\d+)$/) {
4891                 die "skip\n" if !$hotplug_features->{network};
4892                 vm_deviceunplug($vmid, $conf, $opt);
4893             } elsif (is_valid_drivename($opt)) {
4894                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4895                 vm_deviceunplug($vmid, $conf, $opt);
4896                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4897             } elsif ($opt =~ m/^memory$/) {
4898                 die "skip\n" if !$hotplug_features->{memory};
4899                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4900             } elsif ($opt eq 'cpuunits') {
4901                 $cgroup->change_cpu_shares(undef, 1024);
4902             } elsif ($opt eq 'cpulimit') {
4903                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4904             } else {
4905                 die "skip\n";
4906             }
4907         };
4908         if (my $err = $@) {
4909             &$add_error($opt, $err) if $err ne "skip\n";
4910         } else {
4911             delete $conf->{$opt};
4912             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4913         }
4914     }
4915
4916     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4917     $apply_pending_cloudinit = sub {
4918         return if $apply_pending_cloudinit_done; # once is enough
4919         $apply_pending_cloudinit_done = 1; # once is enough
4920
4921         my ($key, $value) = @_;
4922
4923         my @cloudinit_opts = keys %$confdesc_cloudinit;
4924         foreach my $opt (keys %{$conf->{pending}}) {
4925             next if !grep { $_ eq $opt } @cloudinit_opts;
4926             $conf->{$opt} = delete $conf->{pending}->{$opt};
4927         }
4928
4929         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4930         foreach my $opt (sort keys %$pending_delete_hash) {
4931             next if !grep { $_ eq $opt } @cloudinit_opts;
4932             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4933             delete $conf->{$opt};
4934         }
4935
4936         my $new_conf = { %$conf };
4937         $new_conf->{$key} = $value;
4938         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4939     };
4940
4941     foreach my $opt (keys %{$conf->{pending}}) {
4942         next if $selection && !$selection->{$opt};
4943         my $value = $conf->{pending}->{$opt};
4944         eval {
4945             if ($opt eq 'hotplug') {
4946                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4947             } elsif ($opt eq 'tablet') {
4948                 die "skip\n" if !$hotplug_features->{usb};
4949                 if ($value == 1) {
4950                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4951                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4952                         if $arch eq 'aarch64';
4953                 } elsif ($value == 0) {
4954                     vm_deviceunplug($vmid, $conf, 'tablet');
4955                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4956                 }
4957             } elsif ($opt =~ m/^usb\d+$/) {
4958                 die "skip\n";
4959                 # since we cannot reliably hot unplug usb devices we disable it for now
4960                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4961                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4962                 #die "skip\n" if !$d;
4963                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4964             } elsif ($opt eq 'vcpus') {
4965                 die "skip\n" if !$hotplug_features->{cpu};
4966                 qemu_cpu_hotplug($vmid, $conf, $value);
4967             } elsif ($opt eq 'balloon') {
4968                 # enable/disable balloning device is not hotpluggable
4969                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4970                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4971                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4972
4973                 # allow manual ballooning if shares is set to zero
4974                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4975                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4976                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4977                 }
4978             } elsif ($opt =~ m/^net(\d+)$/) {
4979                 # some changes can be done without hotplug
4980                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4981                                     $vmid, $opt, $value, $arch, $machine_type);
4982             } elsif (is_valid_drivename($opt)) {
4983                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4984                 # some changes can be done without hotplug
4985                 my $drive = parse_drive($opt, $value);
4986                 if (drive_is_cloudinit($drive)) {
4987                     &$apply_pending_cloudinit($opt, $value);
4988                 }
4989                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4990                                      $vmid, $opt, $value, $arch, $machine_type);
4991             } elsif ($opt =~ m/^memory$/) { #dimms
4992                 die "skip\n" if !$hotplug_features->{memory};
4993                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4994             } elsif ($opt eq 'cpuunits') {
4995                 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4996                 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4997             } elsif ($opt eq 'cpulimit') {
4998                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4999                 $cgroup->change_cpu_quota($cpulimit, 100000);
5000             } elsif ($opt eq 'agent') {
5001                 vmconfig_update_agent($conf, $opt, $value);
5002             } else {
5003                 die "skip\n";  # skip non-hot-pluggable options
5004             }
5005         };
5006         if (my $err = $@) {
5007             &$add_error($opt, $err) if $err ne "skip\n";
5008         } else {
5009             $conf->{$opt} = $value;
5010             delete $conf->{pending}->{$opt};
5011         }
5012     }
5013
5014     PVE::QemuConfig->write_config($vmid, $conf);
5015 }
5016
5017 sub try_deallocate_drive {
5018     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5019
5020     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5021         my $volid = $drive->{file};
5022         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5023             my $sid = PVE::Storage::parse_volume_id($volid);
5024             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5025
5026             # check if the disk is really unused
5027             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5028                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5029             PVE::Storage::vdisk_free($storecfg, $volid);
5030             return 1;
5031         } else {
5032             # If vm is not owner of this disk remove from config
5033             return 1;
5034         }
5035     }
5036
5037     return;
5038 }
5039
5040 sub vmconfig_delete_or_detach_drive {
5041     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5042
5043     my $drive = parse_drive($opt, $conf->{$opt});
5044
5045     my $rpcenv = PVE::RPCEnvironment::get();
5046     my $authuser = $rpcenv->get_user();
5047
5048     if ($force) {
5049         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5050         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5051     } else {
5052         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5053     }
5054 }
5055
5056
5057
5058 sub vmconfig_apply_pending {
5059     my ($vmid, $conf, $storecfg, $errors) = @_;
5060
5061     return if !scalar(keys %{$conf->{pending}});
5062
5063     my $add_apply_error = sub {
5064         my ($opt, $msg) = @_;
5065         my $err_msg = "unable to apply pending change $opt : $msg";
5066         $errors->{$opt} = $err_msg;
5067         warn $err_msg;
5068     };
5069
5070     # cold plug
5071
5072     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5073     foreach my $opt (sort keys %$pending_delete_hash) {
5074         my $force = $pending_delete_hash->{$opt}->{force};
5075         eval {
5076             if ($opt =~ m/^unused/) {
5077                 die "internal error";
5078             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5079                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5080             }
5081         };
5082         if (my $err = $@) {
5083             $add_apply_error->($opt, $err);
5084         } else {
5085             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5086             delete $conf->{$opt};
5087         }
5088     }
5089
5090     PVE::QemuConfig->cleanup_pending($conf);
5091
5092     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5093         next if $opt eq 'delete'; # just to be sure
5094         eval {
5095             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5096                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5097             }
5098         };
5099         if (my $err = $@) {
5100             $add_apply_error->($opt, $err);
5101         } else {
5102             $conf->{$opt} = delete $conf->{pending}->{$opt};
5103         }
5104     }
5105
5106     # write all changes at once to avoid unnecessary i/o
5107     PVE::QemuConfig->write_config($vmid, $conf);
5108 }
5109
5110 sub vmconfig_update_net {
5111     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5112
5113     my $newnet = parse_net($value);
5114
5115     if ($conf->{$opt}) {
5116         my $oldnet = parse_net($conf->{$opt});
5117
5118         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5119             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5120             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5121             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5122
5123             # for non online change, we try to hot-unplug
5124             die "skip\n" if !$hotplug;
5125             vm_deviceunplug($vmid, $conf, $opt);
5126         } else {
5127
5128             die "internal error" if $opt !~ m/net(\d+)/;
5129             my $iface = "tap${vmid}i$1";
5130
5131             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5132                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5133                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5134                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5135                 PVE::Network::tap_unplug($iface);
5136
5137                 if ($have_sdn) {
5138                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5139                 } else {
5140                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5141                 }
5142             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5143                 # Rate can be applied on its own but any change above needs to
5144                 # include the rate in tap_plug since OVS resets everything.
5145                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5146             }
5147
5148             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5149                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5150             }
5151
5152             return 1;
5153         }
5154     }
5155
5156     if ($hotplug) {
5157         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5158     } else {
5159         die "skip\n";
5160     }
5161 }
5162
5163 sub vmconfig_update_agent {
5164     my ($conf, $opt, $value) = @_;
5165
5166     die "skip\n" if !$conf->{$opt};
5167
5168     my $hotplug_options = { fstrim_cloned_disks => 1 };
5169
5170     my $old_agent = parse_guest_agent($conf);
5171     my $agent = parse_guest_agent({$opt => $value});
5172
5173     for my $option (keys %$agent) { # added/changed options
5174         next if defined($hotplug_options->{$option});
5175         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5176     }
5177
5178     for my $option (keys %$old_agent) { # removed options
5179         next if defined($hotplug_options->{$option});
5180         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5181     }
5182
5183     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5184 }
5185
5186 sub vmconfig_update_disk {
5187     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5188
5189     my $drive = parse_drive($opt, $value);
5190
5191     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5192         my $media = $drive->{media} || 'disk';
5193         my $oldmedia = $old_drive->{media} || 'disk';
5194         die "unable to change media type\n" if $media ne $oldmedia;
5195
5196         if (!drive_is_cdrom($old_drive)) {
5197
5198             if ($drive->{file} ne $old_drive->{file}) {
5199
5200                 die "skip\n" if !$hotplug;
5201
5202                 # unplug and register as unused
5203                 vm_deviceunplug($vmid, $conf, $opt);
5204                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5205
5206             } else {
5207                 # update existing disk
5208
5209                 # skip non hotpluggable value
5210                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5211                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5212                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5213                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5214                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5215                     die "skip\n";
5216                 }
5217
5218                 # apply throttle
5219                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5220                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5221                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5222                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5223                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5224                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5225                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5226                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5227                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5228                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5229                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5230                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5231                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5232                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5233                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5234                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5235                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5236                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5237
5238                     qemu_block_set_io_throttle(
5239                         $vmid,"drive-$opt",
5240                         ($drive->{mbps} || 0)*1024*1024,
5241                         ($drive->{mbps_rd} || 0)*1024*1024,
5242                         ($drive->{mbps_wr} || 0)*1024*1024,
5243                          $drive->{iops} || 0,
5244                          $drive->{iops_rd} || 0,
5245                          $drive->{iops_wr} || 0,
5246                         ($drive->{mbps_max} || 0)*1024*1024,
5247                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5248                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5249                          $drive->{iops_max} || 0,
5250                          $drive->{iops_rd_max} || 0,
5251                          $drive->{iops_wr_max} || 0,
5252                          $drive->{bps_max_length} || 1,
5253                          $drive->{bps_rd_max_length} || 1,
5254                          $drive->{bps_wr_max_length} || 1,
5255                          $drive->{iops_max_length} || 1,
5256                          $drive->{iops_rd_max_length} || 1,
5257                          $drive->{iops_wr_max_length} || 1,
5258                     );
5259
5260                 }
5261
5262                 return 1;
5263             }
5264
5265         } else { # cdrom
5266
5267             if ($drive->{file} eq 'none') {
5268                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5269                 if (drive_is_cloudinit($old_drive)) {
5270                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5271                 }
5272             } else {
5273                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5274
5275                 # force eject if locked
5276                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5277
5278                 if ($path) {
5279                     mon_cmd($vmid, "blockdev-change-medium",
5280                         id => "$opt", filename => "$path");
5281                 }
5282             }
5283
5284             return 1;
5285         }
5286     }
5287
5288     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5289     # hotplug new disks
5290     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5291     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5292 }
5293
5294 # called in locked context by incoming migration
5295 sub vm_migrate_get_nbd_disks {
5296     my ($storecfg, $conf, $replicated_volumes) = @_;
5297
5298     my $local_volumes = {};
5299     PVE::QemuConfig->foreach_volume($conf, sub {
5300         my ($ds, $drive) = @_;
5301
5302         return if drive_is_cdrom($drive);
5303         return if $ds eq 'tpmstate0';
5304
5305         my $volid = $drive->{file};
5306
5307         return if !$volid;
5308
5309         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5310
5311         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5312         return if $scfg->{shared};
5313
5314         # replicated disks re-use existing state via bitmap
5315         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5316         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5317     });
5318     return $local_volumes;
5319 }
5320
5321 # called in locked context by incoming migration
5322 sub vm_migrate_alloc_nbd_disks {
5323     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5324
5325     my $nbd = {};
5326     foreach my $opt (sort keys %$source_volumes) {
5327         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5328
5329         if ($use_existing) {
5330             $nbd->{$opt}->{drivestr} = print_drive($drive);
5331             $nbd->{$opt}->{volid} = $volid;
5332             $nbd->{$opt}->{replicated} = 1;
5333             next;
5334         }
5335
5336         # storage mapping + volname = regular migration
5337         # storage mapping + format = remote migration
5338         # order of precedence, filtered by whether storage supports it:
5339         # 1. explicit requested format
5340         # 2. format of current volume
5341         # 3. default format of storage
5342         if (!$storagemap->{identity}) {
5343             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5344             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5345             if (!$format || !grep { $format eq $_ } @$validFormats) {
5346                 if ($volname) {
5347                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5348                     my $fileFormat = qemu_img_format($scfg, $volname);
5349                     $format = $fileFormat
5350                         if grep { $fileFormat eq $_ } @$validFormats;
5351                 }
5352                 $format //= $defFormat;
5353             }
5354         } else {
5355             # can't happen for remote migration, so $volname is always defined
5356             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5357             $format = qemu_img_format($scfg, $volname);
5358         }
5359
5360         my $size = $drive->{size} / 1024;
5361         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5362         my $newdrive = $drive;
5363         $newdrive->{format} = $format;
5364         $newdrive->{file} = $newvolid;
5365         my $drivestr = print_drive($newdrive);
5366         $nbd->{$opt}->{drivestr} = $drivestr;
5367         $nbd->{$opt}->{volid} = $newvolid;
5368     }
5369
5370     return $nbd;
5371 }
5372
5373 # see vm_start_nolock for parameters, additionally:
5374 # migrate_opts:
5375 #   storagemap = parsed storage map for allocating NBD disks
5376 sub vm_start {
5377     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5378
5379     return PVE::QemuConfig->lock_config($vmid, sub {
5380         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5381
5382         die "you can't start a vm if it's a template\n"
5383             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5384
5385         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5386         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5387
5388         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5389
5390         if ($has_backup_lock && $running) {
5391             # a backup is currently running, attempt to start the guest in the
5392             # existing QEMU instance
5393             return vm_resume($vmid);
5394         }
5395
5396         PVE::QemuConfig->check_lock($conf)
5397             if !($params->{skiplock} || $has_suspended_lock);
5398
5399         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5400
5401         die "VM $vmid already running\n" if $running;
5402
5403         if (my $storagemap = $migrate_opts->{storagemap}) {
5404             my $replicated = $migrate_opts->{replicated_volumes};
5405             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5406             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5407
5408             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5409                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5410             }
5411         }
5412
5413         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5414     });
5415 }
5416
5417
5418 # params:
5419 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5420 #   skiplock => 0/1, skip checking for config lock
5421 #   skiptemplate => 0/1, skip checking whether VM is template
5422 #   forcemachine => to force Qemu machine (rollback/migration)
5423 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5424 #   timeout => in seconds
5425 #   paused => start VM in paused state (backup)
5426 #   resume => resume from hibernation
5427 #   pbs-backing => {
5428 #      sata0 => {
5429 #         repository
5430 #         snapshot
5431 #         keyfile
5432 #         archive
5433 #      },
5434 #      virtio2 => ...
5435 #   }
5436 # migrate_opts:
5437 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5438 #   migratedfrom => source node
5439 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5440 #   network => CIDR of migration network
5441 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5442 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5443 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5444 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5445 #       contained in config
5446 sub vm_start_nolock {
5447     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5448
5449     my $statefile = $params->{statefile};
5450     my $resume = $params->{resume};
5451
5452     my $migratedfrom = $migrate_opts->{migratedfrom};
5453     my $migration_type = $migrate_opts->{type};
5454
5455     my $res = {};
5456
5457     # clean up leftover reboot request files
5458     eval { clear_reboot_request($vmid); };
5459     warn $@ if $@;
5460
5461     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5462         vmconfig_apply_pending($vmid, $conf, $storecfg);
5463         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5464     }
5465
5466     # don't regenerate the ISO if the VM is started as part of a live migration
5467     # this way we can reuse the old ISO with the correct config
5468     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5469
5470     # override offline migrated volumes, conf is out of date still
5471     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5472         for my $key (sort keys $offline_volumes->%*) {
5473             my $parsed = parse_drive($key, $conf->{$key});
5474             $parsed->{file} = $offline_volumes->{$key};
5475             $conf->{$key} = print_drive($parsed);
5476         }
5477     }
5478
5479     my $defaults = load_defaults();
5480
5481     # set environment variable useful inside network script
5482     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5483
5484     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5485
5486     my $forcemachine = $params->{forcemachine};
5487     my $forcecpu = $params->{forcecpu};
5488     if ($resume) {
5489         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5490         $forcemachine = $conf->{runningmachine};
5491         $forcecpu = $conf->{runningcpu};
5492         print "Resuming suspended VM\n";
5493     }
5494
5495     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5496         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5497
5498     my $migration_ip;
5499     my $get_migration_ip = sub {
5500         my ($nodename) = @_;
5501
5502         return $migration_ip if defined($migration_ip);
5503
5504         my $cidr = $migrate_opts->{network};
5505
5506         if (!defined($cidr)) {
5507             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5508             $cidr = $dc_conf->{migration}->{network};
5509         }
5510
5511         if (defined($cidr)) {
5512             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5513
5514             die "could not get IP: no address configured on local " .
5515                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5516
5517             die "could not get IP: multiple addresses configured on local " .
5518                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5519
5520             $migration_ip = @$ips[0];
5521         }
5522
5523         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5524             if !defined($migration_ip);
5525
5526         return $migration_ip;
5527     };
5528
5529     my $migrate_uri;
5530     if ($statefile) {
5531         if ($statefile eq 'tcp') {
5532             my $localip = "localhost";
5533             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5534             my $nodename = nodename();
5535
5536             if (!defined($migration_type)) {
5537                 if (defined($datacenterconf->{migration}->{type})) {
5538                     $migration_type = $datacenterconf->{migration}->{type};
5539                 } else {
5540                     $migration_type = 'secure';
5541                 }
5542             }
5543
5544             if ($migration_type eq 'insecure') {
5545                 $localip = $get_migration_ip->($nodename);
5546                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5547             }
5548
5549             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5550             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5551             $migrate_uri = "tcp:${localip}:${migrate_port}";
5552             push @$cmd, '-incoming', $migrate_uri;
5553             push @$cmd, '-S';
5554
5555         } elsif ($statefile eq 'unix') {
5556             # should be default for secure migrations as a ssh TCP forward
5557             # tunnel is not deterministic reliable ready and fails regurarly
5558             # to set up in time, so use UNIX socket forwards
5559             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5560             unlink $socket_addr;
5561
5562             $migrate_uri = "unix:$socket_addr";
5563
5564             push @$cmd, '-incoming', $migrate_uri;
5565             push @$cmd, '-S';
5566
5567         } elsif (-e $statefile) {
5568             push @$cmd, '-loadstate', $statefile;
5569         } else {
5570             my $statepath = PVE::Storage::path($storecfg, $statefile);
5571             push @$vollist, $statefile;
5572             push @$cmd, '-loadstate', $statepath;
5573         }
5574     } elsif ($params->{paused}) {
5575         push @$cmd, '-S';
5576     }
5577
5578     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5579
5580     my $pci_devices = {}; # host pci devices
5581     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5582         my $dev = $conf->{"hostpci$i"} or next;
5583         $pci_devices->{$i} = parse_hostpci($dev);
5584     }
5585
5586     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5587     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5588
5589     # map to a flat list of pci ids
5590     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5591
5592     # reserve all PCI IDs before actually doing anything with them
5593     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5594
5595     eval {
5596         for my $id (sort keys %$pci_devices) {
5597             my $d = $pci_devices->{$id};
5598             for my $dev ($d->{pciid}->@*) {
5599                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5600             }
5601         }
5602     };
5603     if (my $err = $@) {
5604         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5605         warn $@ if $@;
5606         die $err;
5607     }
5608
5609     PVE::Storage::activate_volumes($storecfg, $vollist);
5610
5611     eval {
5612         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5613     };
5614     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5615     # timeout should be more than enough here...
5616     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5617
5618     my $cpuunits = get_cpuunits($conf);
5619
5620     my %run_params = (
5621         timeout => $statefile ? undef : $start_timeout,
5622         umask => 0077,
5623         noerr => 1,
5624     );
5625
5626     # when migrating, prefix QEMU output so other side can pick up any
5627     # errors that might occur and show the user
5628     if ($migratedfrom) {
5629         $run_params{quiet} = 1;
5630         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5631     }
5632
5633     my %systemd_properties = (
5634         Slice => 'qemu.slice',
5635         KillMode => 'process',
5636         SendSIGKILL => 0,
5637         TimeoutStopUSec => ULONG_MAX, # infinity
5638     );
5639
5640     if (PVE::CGroup::cgroup_mode() == 2) {
5641         $systemd_properties{CPUWeight} = $cpuunits;
5642     } else {
5643         $systemd_properties{CPUShares} = $cpuunits;
5644     }
5645
5646     if (my $cpulimit = $conf->{cpulimit}) {
5647         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5648     }
5649     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5650
5651     my $run_qemu = sub {
5652         PVE::Tools::run_fork sub {
5653             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5654
5655             my $tpmpid;
5656             if (my $tpm = $conf->{tpmstate0}) {
5657                 # start the TPM emulator so QEMU can connect on start
5658                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5659             }
5660
5661             my $exitcode = run_command($cmd, %run_params);
5662             if ($exitcode) {
5663                 if ($tpmpid) {
5664                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5665                     kill 'TERM', $tpmpid;
5666                 }
5667                 die "QEMU exited with code $exitcode\n";
5668             }
5669         };
5670     };
5671
5672     if ($conf->{hugepages}) {
5673
5674         my $code = sub {
5675             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5676             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5677
5678             PVE::QemuServer::Memory::hugepages_mount();
5679             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5680
5681             eval { $run_qemu->() };
5682             if (my $err = $@) {
5683                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5684                     if !$conf->{keephugepages};
5685                 die $err;
5686             }
5687
5688             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5689                 if !$conf->{keephugepages};
5690         };
5691         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5692
5693     } else {
5694         eval { $run_qemu->() };
5695     }
5696
5697     if (my $err = $@) {
5698         # deactivate volumes if start fails
5699         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5700         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5701
5702         die "start failed: $err";
5703     }
5704
5705     # re-reserve all PCI IDs now that we can know the actual VM PID
5706     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5707     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5708     warn $@ if $@;
5709
5710     print "migration listens on $migrate_uri\n" if $migrate_uri;
5711     $res->{migrate_uri} = $migrate_uri;
5712
5713     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5714         eval { mon_cmd($vmid, "cont"); };
5715         warn $@ if $@;
5716     }
5717
5718     #start nbd server for storage migration
5719     if (my $nbd = $migrate_opts->{nbd}) {
5720         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5721
5722         my $migrate_storage_uri;
5723         # nbd_protocol_version > 0 for unix socket support
5724         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5725             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5726             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5727             $migrate_storage_uri = "nbd:unix:$socket_path";
5728         } else {
5729             my $nodename = nodename();
5730             my $localip = $get_migration_ip->($nodename);
5731             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5732             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5733
5734             mon_cmd($vmid, "nbd-server-start", addr => {
5735                 type => 'inet',
5736                 data => {
5737                     host => "${localip}",
5738                     port => "${storage_migrate_port}",
5739                 },
5740             });
5741             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5742             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5743         }
5744
5745         $res->{migrate_storage_uri} = $migrate_storage_uri;
5746
5747         foreach my $opt (sort keys %$nbd) {
5748             my $drivestr = $nbd->{$opt}->{drivestr};
5749             my $volid = $nbd->{$opt}->{volid};
5750             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5751             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5752             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5753             print "re-using replicated volume: $opt - $volid\n"
5754                 if $nbd->{$opt}->{replicated};
5755
5756             $res->{drives}->{$opt} = $nbd->{$opt};
5757             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5758         }
5759     }
5760
5761     if ($migratedfrom) {
5762         eval {
5763             set_migration_caps($vmid);
5764         };
5765         warn $@ if $@;
5766
5767         if ($spice_port) {
5768             print "spice listens on port $spice_port\n";
5769             $res->{spice_port} = $spice_port;
5770             if ($migrate_opts->{spice_ticket}) {
5771                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5772                     $migrate_opts->{spice_ticket});
5773                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5774             }
5775         }
5776
5777     } else {
5778         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5779             if !$statefile && $conf->{balloon};
5780
5781         foreach my $opt (keys %$conf) {
5782             next if $opt !~  m/^net\d+$/;
5783             my $nicconf = parse_net($conf->{$opt});
5784             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5785         }
5786     }
5787
5788     mon_cmd($vmid, 'qom-set',
5789                 path => "machine/peripheral/balloon0",
5790                 property => "guest-stats-polling-interval",
5791                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5792
5793     if ($resume) {
5794         print "Resumed VM, removing state\n";
5795         if (my $vmstate = $conf->{vmstate}) {
5796             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5797             PVE::Storage::vdisk_free($storecfg, $vmstate);
5798         }
5799         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5800         PVE::QemuConfig->write_config($vmid, $conf);
5801     }
5802
5803     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5804
5805     return $res;
5806 }
5807
5808 sub vm_commandline {
5809     my ($storecfg, $vmid, $snapname) = @_;
5810
5811     my $conf = PVE::QemuConfig->load_config($vmid);
5812
5813     my ($forcemachine, $forcecpu);
5814     if ($snapname) {
5815         my $snapshot = $conf->{snapshots}->{$snapname};
5816         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5817
5818         # check for machine or CPU overrides in snapshot
5819         $forcemachine = $snapshot->{runningmachine};
5820         $forcecpu = $snapshot->{runningcpu};
5821
5822         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5823
5824         $conf = $snapshot;
5825     }
5826
5827     my $defaults = load_defaults();
5828
5829     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5830
5831     return PVE::Tools::cmd2string($cmd);
5832 }
5833
5834 sub vm_reset {
5835     my ($vmid, $skiplock) = @_;
5836
5837     PVE::QemuConfig->lock_config($vmid, sub {
5838
5839         my $conf = PVE::QemuConfig->load_config($vmid);
5840
5841         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5842
5843         mon_cmd($vmid, "system_reset");
5844     });
5845 }
5846
5847 sub get_vm_volumes {
5848     my ($conf) = @_;
5849
5850     my $vollist = [];
5851     foreach_volid($conf, sub {
5852         my ($volid, $attr) = @_;
5853
5854         return if $volid =~ m|^/|;
5855
5856         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5857         return if !$sid;
5858
5859         push @$vollist, $volid;
5860     });
5861
5862     return $vollist;
5863 }
5864
5865 sub vm_stop_cleanup {
5866     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5867
5868     eval {
5869
5870         if (!$keepActive) {
5871             my $vollist = get_vm_volumes($conf);
5872             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5873
5874             if (my $tpmdrive = $conf->{tpmstate0}) {
5875                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5876                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5877                 if ($storeid) {
5878                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5879                 }
5880             }
5881         }
5882
5883         foreach my $ext (qw(mon qmp pid vnc qga)) {
5884             unlink "/var/run/qemu-server/${vmid}.$ext";
5885         }
5886
5887         if ($conf->{ivshmem}) {
5888             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5889             # just delete it for now, VMs which have this already open do not
5890             # are affected, but new VMs will get a separated one. If this
5891             # becomes an issue we either add some sort of ref-counting or just
5892             # add a "don't delete on stop" flag to the ivshmem format.
5893             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5894         }
5895
5896         my $ids = [];
5897         foreach my $key (keys %$conf) {
5898             next if $key !~ m/^hostpci(\d+)$/;
5899             my $hostpciindex = $1;
5900             my $d = parse_hostpci($conf->{$key});
5901             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5902
5903             foreach my $pci (@{$d->{pciid}}) {
5904                 my $pciid = $pci->{id};
5905                 push @$ids, $pci->{id};
5906                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5907             }
5908         }
5909         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5910
5911         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5912     };
5913     warn $@ if $@; # avoid errors - just warn
5914 }
5915
5916 # call only in locked context
5917 sub _do_vm_stop {
5918     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5919
5920     my $pid = check_running($vmid, $nocheck);
5921     return if !$pid;
5922
5923     my $conf;
5924     if (!$nocheck) {
5925         $conf = PVE::QemuConfig->load_config($vmid);
5926         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5927         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5928             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5929             $timeout = $opts->{down} if $opts->{down};
5930         }
5931         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5932     }
5933
5934     eval {
5935         if ($shutdown) {
5936             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5937                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5938             } else {
5939                 mon_cmd($vmid, "system_powerdown");
5940             }
5941         } else {
5942             mon_cmd($vmid, "quit");
5943         }
5944     };
5945     my $err = $@;
5946
5947     if (!$err) {
5948         $timeout = 60 if !defined($timeout);
5949
5950         my $count = 0;
5951         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5952             $count++;
5953             sleep 1;
5954         }
5955
5956         if ($count >= $timeout) {
5957             if ($force) {
5958                 warn "VM still running - terminating now with SIGTERM\n";
5959                 kill 15, $pid;
5960             } else {
5961                 die "VM quit/powerdown failed - got timeout\n";
5962             }
5963         } else {
5964             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5965             return;
5966         }
5967     } else {
5968        if (!check_running($vmid, $nocheck)) {
5969             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5970             return;
5971        }
5972        if ($force) {
5973             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5974             kill 15, $pid;
5975         } else {
5976             die "VM quit/powerdown failed\n";
5977         }
5978     }
5979
5980     # wait again
5981     $timeout = 10;
5982
5983     my $count = 0;
5984     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5985         $count++;
5986         sleep 1;
5987     }
5988
5989     if ($count >= $timeout) {
5990         warn "VM still running - terminating now with SIGKILL\n";
5991         kill 9, $pid;
5992         sleep 1;
5993     }
5994
5995     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5996 }
5997
5998 # Note: use $nocheck to skip tests if VM configuration file exists.
5999 # We need that when migration VMs to other nodes (files already moved)
6000 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6001 sub vm_stop {
6002     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6003
6004     $force = 1 if !defined($force) && !$shutdown;
6005
6006     if ($migratedfrom){
6007         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6008         kill 15, $pid if $pid;
6009         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6010         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6011         return;
6012     }
6013
6014     PVE::QemuConfig->lock_config($vmid, sub {
6015         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6016    });
6017 }
6018
6019 sub vm_reboot {
6020     my ($vmid, $timeout) = @_;
6021
6022     PVE::QemuConfig->lock_config($vmid, sub {
6023         eval {
6024
6025             # only reboot if running, as qmeventd starts it again on a stop event
6026             return if !check_running($vmid);
6027
6028             create_reboot_request($vmid);
6029
6030             my $storecfg = PVE::Storage::config();
6031             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6032
6033         };
6034         if (my $err = $@) {
6035             # avoid that the next normal shutdown will be confused for a reboot
6036             clear_reboot_request($vmid);
6037             die $err;
6038         }
6039    });
6040 }
6041
6042 # note: if using the statestorage parameter, the caller has to check privileges
6043 sub vm_suspend {
6044     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6045
6046     my $conf;
6047     my $path;
6048     my $storecfg;
6049     my $vmstate;
6050
6051     PVE::QemuConfig->lock_config($vmid, sub {
6052
6053         $conf = PVE::QemuConfig->load_config($vmid);
6054
6055         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6056         PVE::QemuConfig->check_lock($conf)
6057             if !($skiplock || $is_backing_up);
6058
6059         die "cannot suspend to disk during backup\n"
6060             if $is_backing_up && $includestate;
6061
6062         if ($includestate) {
6063             $conf->{lock} = 'suspending';
6064             my $date = strftime("%Y-%m-%d", localtime(time()));
6065             $storecfg = PVE::Storage::config();
6066             if (!$statestorage) {
6067                 $statestorage = find_vmstate_storage($conf, $storecfg);
6068                 # check permissions for the storage
6069                 my $rpcenv = PVE::RPCEnvironment::get();
6070                 if ($rpcenv->{type} ne 'cli') {
6071                     my $authuser = $rpcenv->get_user();
6072                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6073                 }
6074             }
6075
6076
6077             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6078                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6079             $path = PVE::Storage::path($storecfg, $vmstate);
6080             PVE::QemuConfig->write_config($vmid, $conf);
6081         } else {
6082             mon_cmd($vmid, "stop");
6083         }
6084     });
6085
6086     if ($includestate) {
6087         # save vm state
6088         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6089
6090         eval {
6091             set_migration_caps($vmid, 1);
6092             mon_cmd($vmid, "savevm-start", statefile => $path);
6093             for(;;) {
6094                 my $state = mon_cmd($vmid, "query-savevm");
6095                 if (!$state->{status}) {
6096                     die "savevm not active\n";
6097                 } elsif ($state->{status} eq 'active') {
6098                     sleep(1);
6099                     next;
6100                 } elsif ($state->{status} eq 'completed') {
6101                     print "State saved, quitting\n";
6102                     last;
6103                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6104                     die "query-savevm failed with error '$state->{error}'\n"
6105                 } else {
6106                     die "query-savevm returned status '$state->{status}'\n";
6107                 }
6108             }
6109         };
6110         my $err = $@;
6111
6112         PVE::QemuConfig->lock_config($vmid, sub {
6113             $conf = PVE::QemuConfig->load_config($vmid);
6114             if ($err) {
6115                 # cleanup, but leave suspending lock, to indicate something went wrong
6116                 eval {
6117                     mon_cmd($vmid, "savevm-end");
6118                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6119                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6120                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6121                     PVE::QemuConfig->write_config($vmid, $conf);
6122                 };
6123                 warn $@ if $@;
6124                 die $err;
6125             }
6126
6127             die "lock changed unexpectedly\n"
6128                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6129
6130             mon_cmd($vmid, "quit");
6131             $conf->{lock} = 'suspended';
6132             PVE::QemuConfig->write_config($vmid, $conf);
6133         });
6134     }
6135 }
6136
6137 sub vm_resume {
6138     my ($vmid, $skiplock, $nocheck) = @_;
6139
6140     PVE::QemuConfig->lock_config($vmid, sub {
6141         my $res = mon_cmd($vmid, 'query-status');
6142         my $resume_cmd = 'cont';
6143         my $reset = 0;
6144
6145         if ($res->{status}) {
6146             return if $res->{status} eq 'running'; # job done, go home
6147             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6148             $reset = 1 if $res->{status} eq 'shutdown';
6149         }
6150
6151         if (!$nocheck) {
6152
6153             my $conf = PVE::QemuConfig->load_config($vmid);
6154
6155             PVE::QemuConfig->check_lock($conf)
6156                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6157         }
6158
6159         if ($reset) {
6160             # required if a VM shuts down during a backup and we get a resume
6161             # request before the backup finishes for example
6162             mon_cmd($vmid, "system_reset");
6163         }
6164         mon_cmd($vmid, $resume_cmd);
6165     });
6166 }
6167
6168 sub vm_sendkey {
6169     my ($vmid, $skiplock, $key) = @_;
6170
6171     PVE::QemuConfig->lock_config($vmid, sub {
6172
6173         my $conf = PVE::QemuConfig->load_config($vmid);
6174
6175         # there is no qmp command, so we use the human monitor command
6176         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6177         die $res if $res ne '';
6178     });
6179 }
6180
6181 # vzdump restore implementaion
6182
6183 sub tar_archive_read_firstfile {
6184     my $archive = shift;
6185
6186     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6187
6188     # try to detect archive type first
6189     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6190         die "unable to open file '$archive'\n";
6191     my $firstfile = <$fh>;
6192     kill 15, $pid;
6193     close $fh;
6194
6195     die "ERROR: archive contaions no data\n" if !$firstfile;
6196     chomp $firstfile;
6197
6198     return $firstfile;
6199 }
6200
6201 sub tar_restore_cleanup {
6202     my ($storecfg, $statfile) = @_;
6203
6204     print STDERR "starting cleanup\n";
6205
6206     if (my $fd = IO::File->new($statfile, "r")) {
6207         while (defined(my $line = <$fd>)) {
6208             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6209                 my $volid = $2;
6210                 eval {
6211                     if ($volid =~ m|^/|) {
6212                         unlink $volid || die 'unlink failed\n';
6213                     } else {
6214                         PVE::Storage::vdisk_free($storecfg, $volid);
6215                     }
6216                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6217                 };
6218                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6219             } else {
6220                 print STDERR "unable to parse line in statfile - $line";
6221             }
6222         }
6223         $fd->close();
6224     }
6225 }
6226
6227 sub restore_file_archive {
6228     my ($archive, $vmid, $user, $opts) = @_;
6229
6230     return restore_vma_archive($archive, $vmid, $user, $opts)
6231         if $archive eq '-';
6232
6233     my $info = PVE::Storage::archive_info($archive);
6234     my $format = $opts->{format} // $info->{format};
6235     my $comp = $info->{compression};
6236
6237     # try to detect archive format
6238     if ($format eq 'tar') {
6239         return restore_tar_archive($archive, $vmid, $user, $opts);
6240     } else {
6241         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6242     }
6243 }
6244
6245 # hepler to remove disks that will not be used after restore
6246 my $restore_cleanup_oldconf = sub {
6247     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6248
6249     my $kept_disks = {};
6250
6251     PVE::QemuConfig->foreach_volume($oldconf, sub {
6252         my ($ds, $drive) = @_;
6253
6254         return if drive_is_cdrom($drive, 1);
6255
6256         my $volid = $drive->{file};
6257         return if !$volid || $volid =~ m|^/|;
6258
6259         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6260         return if !$path || !$owner || ($owner != $vmid);
6261
6262         # Note: only delete disk we want to restore
6263         # other volumes will become unused
6264         if ($virtdev_hash->{$ds}) {
6265             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6266             if (my $err = $@) {
6267                 warn $err;
6268             }
6269         } else {
6270             $kept_disks->{$volid} = 1;
6271         }
6272     });
6273
6274     # after the restore we have no snapshots anymore
6275     for my $snapname (keys $oldconf->{snapshots}->%*) {
6276         my $snap = $oldconf->{snapshots}->{$snapname};
6277         if ($snap->{vmstate}) {
6278             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6279             if (my $err = $@) {
6280                 warn $err;
6281             }
6282         }
6283
6284         for my $volid (keys $kept_disks->%*) {
6285             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6286             warn $@ if $@;
6287         }
6288     }
6289 };
6290
6291 # Helper to parse vzdump backup device hints
6292 #
6293 # $rpcenv: Environment, used to ckeck storage permissions
6294 # $user: User ID, to check storage permissions
6295 # $storecfg: Storage configuration
6296 # $fh: the file handle for reading the configuration
6297 # $devinfo: should contain device sizes for all backu-up'ed devices
6298 # $options: backup options (pool, default storage)
6299 #
6300 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6301 my $parse_backup_hints = sub {
6302     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6303
6304     my $check_storage = sub { # assert if an image can be allocate
6305         my ($storeid, $scfg) = @_;
6306         die "Content type 'images' is not available on storage '$storeid'\n"
6307             if !$scfg->{content}->{images};
6308         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6309             if $user ne 'root@pam';
6310     };
6311
6312     my $virtdev_hash = {};
6313     while (defined(my $line = <$fh>)) {
6314         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6315             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6316             die "archive does not contain data for drive '$virtdev'\n"
6317                 if !$devinfo->{$devname};
6318
6319             if (defined($options->{storage})) {
6320                 $storeid = $options->{storage} || 'local';
6321             } elsif (!$storeid) {
6322                 $storeid = 'local';
6323             }
6324             $format = 'raw' if !$format;
6325             $devinfo->{$devname}->{devname} = $devname;
6326             $devinfo->{$devname}->{virtdev} = $virtdev;
6327             $devinfo->{$devname}->{format} = $format;
6328             $devinfo->{$devname}->{storeid} = $storeid;
6329
6330             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6331             $check_storage->($storeid, $scfg); # permission and content type check
6332
6333             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6334         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6335             my $virtdev = $1;
6336             my $drive = parse_drive($virtdev, $2);
6337
6338             if (drive_is_cloudinit($drive)) {
6339                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6340                 $storeid = $options->{storage} if defined ($options->{storage});
6341                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6342                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6343
6344                 $check_storage->($storeid, $scfg); # permission and content type check
6345
6346                 $virtdev_hash->{$virtdev} = {
6347                     format => $format,
6348                     storeid => $storeid,
6349                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6350                     is_cloudinit => 1,
6351                 };
6352             }
6353         }
6354     }
6355
6356     return $virtdev_hash;
6357 };
6358
6359 # Helper to allocate and activate all volumes required for a restore
6360 #
6361 # $storecfg: Storage configuration
6362 # $virtdev_hash: as returned by parse_backup_hints()
6363 #
6364 # Returns: { $virtdev => $volid }
6365 my $restore_allocate_devices = sub {
6366     my ($storecfg, $virtdev_hash, $vmid) = @_;
6367
6368     my $map = {};
6369     foreach my $virtdev (sort keys %$virtdev_hash) {
6370         my $d = $virtdev_hash->{$virtdev};
6371         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6372         my $storeid = $d->{storeid};
6373         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6374
6375         # test if requested format is supported
6376         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6377         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6378         $d->{format} = $defFormat if !$supported;
6379
6380         my $name;
6381         if ($d->{is_cloudinit}) {
6382             $name = "vm-$vmid-cloudinit";
6383             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6384             if ($scfg->{path}) {
6385                 $name .= ".$d->{format}";
6386             }
6387         }
6388
6389         my $volid = PVE::Storage::vdisk_alloc(
6390             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6391
6392         print STDERR "new volume ID is '$volid'\n";
6393         $d->{volid} = $volid;
6394
6395         PVE::Storage::activate_volumes($storecfg, [$volid]);
6396
6397         $map->{$virtdev} = $volid;
6398     }
6399
6400     return $map;
6401 };
6402
6403 sub restore_update_config_line {
6404     my ($cookie, $map, $line, $unique) = @_;
6405
6406     return '' if $line =~ m/^\#qmdump\#/;
6407     return '' if $line =~ m/^\#vzdump\#/;
6408     return '' if $line =~ m/^lock:/;
6409     return '' if $line =~ m/^unused\d+:/;
6410     return '' if $line =~ m/^parent:/;
6411
6412     my $res = '';
6413
6414     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6415     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6416         # try to convert old 1.X settings
6417         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6418         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6419             my ($model, $macaddr) = split(/\=/, $devconfig);
6420             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6421             my $net = {
6422                 model => $model,
6423                 bridge => "vmbr$ind",
6424                 macaddr => $macaddr,
6425             };
6426             my $netstr = print_net($net);
6427
6428             $res .= "net$cookie->{netcount}: $netstr\n";
6429             $cookie->{netcount}++;
6430         }
6431     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6432         my ($id, $netstr) = ($1, $2);
6433         my $net = parse_net($netstr);
6434         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6435         $netstr = print_net($net);
6436         $res .= "$id: $netstr\n";
6437     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6438         my $virtdev = $1;
6439         my $value = $3;
6440         my $di = parse_drive($virtdev, $value);
6441         if (defined($di->{backup}) && !$di->{backup}) {
6442             $res .= "#$line";
6443         } elsif ($map->{$virtdev}) {
6444             delete $di->{format}; # format can change on restore
6445             $di->{file} = $map->{$virtdev};
6446             $value = print_drive($di);
6447             $res .= "$virtdev: $value\n";
6448         } else {
6449             $res .= $line;
6450         }
6451     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6452         my $vmgenid = $1;
6453         if ($vmgenid ne '0') {
6454             # always generate a new vmgenid if there was a valid one setup
6455             $vmgenid = generate_uuid();
6456         }
6457         $res .= "vmgenid: $vmgenid\n";
6458     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6459         my ($uuid, $uuid_str);
6460         UUID::generate($uuid);
6461         UUID::unparse($uuid, $uuid_str);
6462         my $smbios1 = parse_smbios1($2);
6463         $smbios1->{uuid} = $uuid_str;
6464         $res .= $1.print_smbios1($smbios1)."\n";
6465     } else {
6466         $res .= $line;
6467     }
6468
6469     return $res;
6470 }
6471
6472 my $restore_deactivate_volumes = sub {
6473     my ($storecfg, $virtdev_hash) = @_;
6474
6475     my $vollist = [];
6476     for my $dev (values $virtdev_hash->%*) {
6477         push $vollist->@*, $dev->{volid} if $dev->{volid};
6478     }
6479
6480     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6481     print STDERR $@ if $@;
6482 };
6483
6484 my $restore_destroy_volumes = sub {
6485     my ($storecfg, $virtdev_hash) = @_;
6486
6487     for my $dev (values $virtdev_hash->%*) {
6488         my $volid = $dev->{volid} or next;
6489         eval {
6490             PVE::Storage::vdisk_free($storecfg, $volid);
6491             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6492         };
6493         print STDERR "unable to cleanup '$volid' - $@" if $@;
6494     }
6495 };
6496
6497 my $restore_merge_config = sub {
6498     my ($filename, $backup_conf_raw, $override_conf) = @_;
6499
6500     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6501     for my $key (keys $override_conf->%*) {
6502         $backup_conf->{$key} = $override_conf->{$key};
6503     }
6504
6505     return $backup_conf;
6506 };
6507
6508 sub scan_volids {
6509     my ($cfg, $vmid) = @_;
6510
6511     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6512
6513     my $volid_hash = {};
6514     foreach my $storeid (keys %$info) {
6515         foreach my $item (@{$info->{$storeid}}) {
6516             next if !($item->{volid} && $item->{size});
6517             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6518             $volid_hash->{$item->{volid}} = $item;
6519         }
6520     }
6521
6522     return $volid_hash;
6523 }
6524
6525 sub update_disk_config {
6526     my ($vmid, $conf, $volid_hash) = @_;
6527
6528     my $changes;
6529     my $prefix = "VM $vmid";
6530
6531     # used and unused disks
6532     my $referenced = {};
6533
6534     # Note: it is allowed to define multiple storages with same path (alias), so
6535     # we need to check both 'volid' and real 'path' (two different volid can point
6536     # to the same path).
6537
6538     my $referencedpath = {};
6539
6540     # update size info
6541     PVE::QemuConfig->foreach_volume($conf, sub {
6542         my ($opt, $drive) = @_;
6543
6544         my $volid = $drive->{file};
6545         return if !$volid;
6546         my $volume = $volid_hash->{$volid};
6547
6548         # mark volid as "in-use" for next step
6549         $referenced->{$volid} = 1;
6550         if ($volume && (my $path = $volume->{path})) {
6551             $referencedpath->{$path} = 1;
6552         }
6553
6554         return if drive_is_cdrom($drive);
6555         return if !$volume;
6556
6557         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6558         if (defined($updated)) {
6559             $changes = 1;
6560             $conf->{$opt} = print_drive($updated);
6561             print "$prefix ($opt): $msg\n";
6562         }
6563     });
6564
6565     # remove 'unusedX' entry if volume is used
6566     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6567         my ($opt, $drive) = @_;
6568
6569         my $volid = $drive->{file};
6570         return if !$volid;
6571
6572         my $path;
6573         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6574         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6575             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6576             $changes = 1;
6577             delete $conf->{$opt};
6578         }
6579
6580         $referenced->{$volid} = 1;
6581         $referencedpath->{$path} = 1 if $path;
6582     });
6583
6584     foreach my $volid (sort keys %$volid_hash) {
6585         next if $volid =~ m/vm-$vmid-state-/;
6586         next if $referenced->{$volid};
6587         my $path = $volid_hash->{$volid}->{path};
6588         next if !$path; # just to be sure
6589         next if $referencedpath->{$path};
6590         $changes = 1;
6591         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6592         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6593         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6594     }
6595
6596     return $changes;
6597 }
6598
6599 sub rescan {
6600     my ($vmid, $nolock, $dryrun) = @_;
6601
6602     my $cfg = PVE::Storage::config();
6603
6604     print "rescan volumes...\n";
6605     my $volid_hash = scan_volids($cfg, $vmid);
6606
6607     my $updatefn =  sub {
6608         my ($vmid) = @_;
6609
6610         my $conf = PVE::QemuConfig->load_config($vmid);
6611
6612         PVE::QemuConfig->check_lock($conf);
6613
6614         my $vm_volids = {};
6615         foreach my $volid (keys %$volid_hash) {
6616             my $info = $volid_hash->{$volid};
6617             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6618         }
6619
6620         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6621
6622         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6623     };
6624
6625     if (defined($vmid)) {
6626         if ($nolock) {
6627             &$updatefn($vmid);
6628         } else {
6629             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6630         }
6631     } else {
6632         my $vmlist = config_list();
6633         foreach my $vmid (keys %$vmlist) {
6634             if ($nolock) {
6635                 &$updatefn($vmid);
6636             } else {
6637                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6638             }
6639         }
6640     }
6641 }
6642
6643 sub restore_proxmox_backup_archive {
6644     my ($archive, $vmid, $user, $options) = @_;
6645
6646     my $storecfg = PVE::Storage::config();
6647
6648     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6649     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6650
6651     my $fingerprint = $scfg->{fingerprint};
6652     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6653
6654     my $repo = PVE::PBSClient::get_repository($scfg);
6655
6656     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6657     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6658     local $ENV{PBS_PASSWORD} = $password;
6659     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6660
6661     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6662         PVE::Storage::parse_volname($storecfg, $archive);
6663
6664     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6665
6666     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6667
6668     my $tmpdir = "/var/tmp/vzdumptmp$$";
6669     rmtree $tmpdir;
6670     mkpath $tmpdir;
6671
6672     my $conffile = PVE::QemuConfig->config_file($vmid);
6673      # disable interrupts (always do cleanups)
6674     local $SIG{INT} =
6675         local $SIG{TERM} =
6676         local $SIG{QUIT} =
6677         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6678
6679     # Note: $oldconf is undef if VM does not exists
6680     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6681     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6682     my $new_conf_raw = '';
6683
6684     my $rpcenv = PVE::RPCEnvironment::get();
6685     my $devinfo = {}; # info about drives included in backup
6686     my $virtdev_hash = {}; # info about allocated drives
6687
6688     eval {
6689         # enable interrupts
6690         local $SIG{INT} =
6691             local $SIG{TERM} =
6692             local $SIG{QUIT} =
6693             local $SIG{HUP} =
6694             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6695
6696         my $cfgfn = "$tmpdir/qemu-server.conf";
6697         my $firewall_config_fn = "$tmpdir/fw.conf";
6698         my $index_fn = "$tmpdir/index.json";
6699
6700         my $cmd = "restore";
6701
6702         my $param = [$pbs_backup_name, "index.json", $index_fn];
6703         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6704         my $index = PVE::Tools::file_get_contents($index_fn);
6705         $index = decode_json($index);
6706
6707         foreach my $info (@{$index->{files}}) {
6708             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6709                 my $devname = $1;
6710                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6711                     $devinfo->{$devname}->{size} = $1;
6712                 } else {
6713                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6714                 }
6715             }
6716         }
6717
6718         my $is_qemu_server_backup = scalar(
6719             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6720         );
6721         if (!$is_qemu_server_backup) {
6722             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6723         }
6724         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6725
6726         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6727         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6728
6729         if ($has_firewall_config) {
6730             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6731             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6732
6733             my $pve_firewall_dir = '/etc/pve/firewall';
6734             mkdir $pve_firewall_dir; # make sure the dir exists
6735             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6736         }
6737
6738         my $fh = IO::File->new($cfgfn, "r") ||
6739             die "unable to read qemu-server.conf - $!\n";
6740
6741         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6742
6743         # fixme: rate limit?
6744
6745         # create empty/temp config
6746         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6747
6748         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6749
6750         # allocate volumes
6751         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6752
6753         foreach my $virtdev (sort keys %$virtdev_hash) {
6754             my $d = $virtdev_hash->{$virtdev};
6755             next if $d->{is_cloudinit}; # no need to restore cloudinit
6756
6757             # this fails if storage is unavailable
6758             my $volid = $d->{volid};
6759             my $path = PVE::Storage::path($storecfg, $volid);
6760
6761             # for live-restore we only want to preload the efidisk and TPM state
6762             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6763
6764             my $pbs_restore_cmd = [
6765                 '/usr/bin/pbs-restore',
6766                 '--repository', $repo,
6767                 $pbs_backup_name,
6768                 "$d->{devname}.img.fidx",
6769                 $path,
6770                 '--verbose',
6771                 ];
6772
6773             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6774             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6775
6776             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6777                 push @$pbs_restore_cmd, '--skip-zero';
6778             }
6779
6780             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6781             print "restore proxmox backup image: $dbg_cmdstring\n";
6782             run_command($pbs_restore_cmd);
6783         }
6784
6785         $fh->seek(0, 0) || die "seek failed - $!\n";
6786
6787         my $cookie = { netcount => 0 };
6788         while (defined(my $line = <$fh>)) {
6789             $new_conf_raw .= restore_update_config_line(
6790                 $cookie,
6791                 $map,
6792                 $line,
6793                 $options->{unique},
6794             );
6795         }
6796
6797         $fh->close();
6798     };
6799     my $err = $@;
6800
6801     if ($err || !$options->{live}) {
6802         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6803     }
6804
6805     rmtree $tmpdir;
6806
6807     if ($err) {
6808         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6809         die $err;
6810     }
6811
6812     if ($options->{live}) {
6813         # keep lock during live-restore
6814         $new_conf_raw .= "\nlock: create";
6815     }
6816
6817     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6818     PVE::QemuConfig->write_config($vmid, $new_conf);
6819
6820     eval { rescan($vmid, 1); };
6821     warn $@ if $@;
6822
6823     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6824
6825     if ($options->{live}) {
6826         # enable interrupts
6827         local $SIG{INT} =
6828             local $SIG{TERM} =
6829             local $SIG{QUIT} =
6830             local $SIG{HUP} =
6831             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6832
6833         my $conf = PVE::QemuConfig->load_config($vmid);
6834         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6835
6836         # these special drives are already restored before start
6837         delete $devinfo->{'drive-efidisk0'};
6838         delete $devinfo->{'drive-tpmstate0-backup'};
6839         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6840
6841         PVE::QemuConfig->remove_lock($vmid, "create");
6842     }
6843 }
6844
6845 sub pbs_live_restore {
6846     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6847
6848     print "starting VM for live-restore\n";
6849     print "repository: '$repo', snapshot: '$snap'\n";
6850
6851     my $pbs_backing = {};
6852     for my $ds (keys %$restored_disks) {
6853         $ds =~ m/^drive-(.*)$/;
6854         my $confname = $1;
6855         $pbs_backing->{$confname} = {
6856             repository => $repo,
6857             snapshot => $snap,
6858             archive => "$ds.img.fidx",
6859         };
6860         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6861
6862         my $drive = parse_drive($confname, $conf->{$confname});
6863         print "restoring '$ds' to '$drive->{file}'\n";
6864     }
6865
6866     my $drives_streamed = 0;
6867     eval {
6868         # make sure HA doesn't interrupt our restore by stopping the VM
6869         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6870             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6871         }
6872
6873         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6874         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6875         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6876
6877         my $qmeventd_fd = register_qmeventd_handle($vmid);
6878
6879         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6880         # this will effectively collapse the backing image chain consisting of
6881         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6882         # removes itself once all backing images vanish with 'auto-remove=on')
6883         my $jobs = {};
6884         for my $ds (sort keys %$restored_disks) {
6885             my $job_id = "restore-$ds";
6886             mon_cmd($vmid, 'block-stream',
6887                 'job-id' => $job_id,
6888                 device => "$ds",
6889             );
6890             $jobs->{$job_id} = {};
6891         }
6892
6893         mon_cmd($vmid, 'cont');
6894         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6895
6896         print "restore-drive jobs finished successfully, removing all tracking block devices"
6897             ." to disconnect from Proxmox Backup Server\n";
6898
6899         for my $ds (sort keys %$restored_disks) {
6900             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6901         }
6902
6903         close($qmeventd_fd);
6904     };
6905
6906     my $err = $@;
6907
6908     if ($err) {
6909         warn "An error occurred during live-restore: $err\n";
6910         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6911         die "live-restore failed\n";
6912     }
6913 }
6914
6915 sub restore_vma_archive {
6916     my ($archive, $vmid, $user, $opts, $comp) = @_;
6917
6918     my $readfrom = $archive;
6919
6920     my $cfg = PVE::Storage::config();
6921     my $commands = [];
6922     my $bwlimit = $opts->{bwlimit};
6923
6924     my $dbg_cmdstring = '';
6925     my $add_pipe = sub {
6926         my ($cmd) = @_;
6927         push @$commands, $cmd;
6928         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6929         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6930         $readfrom = '-';
6931     };
6932
6933     my $input = undef;
6934     if ($archive eq '-') {
6935         $input = '<&STDIN';
6936     } else {
6937         # If we use a backup from a PVE defined storage we also consider that
6938         # storage's rate limit:
6939         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6940         if (defined($volid)) {
6941             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6942             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6943             if ($readlimit) {
6944                 print STDERR "applying read rate limit: $readlimit\n";
6945                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6946                 $add_pipe->($cstream);
6947             }
6948         }
6949     }
6950
6951     if ($comp) {
6952         my $info = PVE::Storage::decompressor_info('vma', $comp);
6953         my $cmd = $info->{decompressor};
6954         push @$cmd, $readfrom;
6955         $add_pipe->($cmd);
6956     }
6957
6958     my $tmpdir = "/var/tmp/vzdumptmp$$";
6959     rmtree $tmpdir;
6960
6961     # disable interrupts (always do cleanups)
6962     local $SIG{INT} =
6963         local $SIG{TERM} =
6964         local $SIG{QUIT} =
6965         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6966
6967     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6968     POSIX::mkfifo($mapfifo, 0600);
6969     my $fifofh;
6970     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6971
6972     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6973
6974     my $oldtimeout;
6975     my $timeout = 5;
6976
6977     my $devinfo = {}; # info about drives included in backup
6978     my $virtdev_hash = {}; # info about allocated drives
6979
6980     my $rpcenv = PVE::RPCEnvironment::get();
6981
6982     my $conffile = PVE::QemuConfig->config_file($vmid);
6983
6984     # Note: $oldconf is undef if VM does not exist
6985     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6986     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6987     my $new_conf_raw = '';
6988
6989     my %storage_limits;
6990
6991     my $print_devmap = sub {
6992         my $cfgfn = "$tmpdir/qemu-server.conf";
6993
6994         # we can read the config - that is already extracted
6995         my $fh = IO::File->new($cfgfn, "r") ||
6996             die "unable to read qemu-server.conf - $!\n";
6997
6998         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6999         if (-f $fwcfgfn) {
7000             my $pve_firewall_dir = '/etc/pve/firewall';
7001             mkdir $pve_firewall_dir; # make sure the dir exists
7002             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7003         }
7004
7005         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7006
7007         foreach my $info (values %{$virtdev_hash}) {
7008             my $storeid = $info->{storeid};
7009             next if defined($storage_limits{$storeid});
7010
7011             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7012             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7013             $storage_limits{$storeid} = $limit * 1024;
7014         }
7015
7016         foreach my $devname (keys %$devinfo) {
7017             die "found no device mapping information for device '$devname'\n"
7018                 if !$devinfo->{$devname}->{virtdev};
7019         }
7020
7021         # create empty/temp config
7022         if ($oldconf) {
7023             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7024             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7025         }
7026
7027         # allocate volumes
7028         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7029
7030         # print restore information to $fifofh
7031         foreach my $virtdev (sort keys %$virtdev_hash) {
7032             my $d = $virtdev_hash->{$virtdev};
7033             next if $d->{is_cloudinit}; # no need to restore cloudinit
7034
7035             my $storeid = $d->{storeid};
7036             my $volid = $d->{volid};
7037
7038             my $map_opts = '';
7039             if (my $limit = $storage_limits{$storeid}) {
7040                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7041             }
7042
7043             my $write_zeros = 1;
7044             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7045                 $write_zeros = 0;
7046             }
7047
7048             my $path = PVE::Storage::path($cfg, $volid);
7049
7050             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7051
7052             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7053         }
7054
7055         $fh->seek(0, 0) || die "seek failed - $!\n";
7056
7057         my $cookie = { netcount => 0 };
7058         while (defined(my $line = <$fh>)) {
7059             $new_conf_raw .= restore_update_config_line(
7060                 $cookie,
7061                 $map,
7062                 $line,
7063                 $opts->{unique},
7064             );
7065         }
7066
7067         $fh->close();
7068     };
7069
7070     eval {
7071         # enable interrupts
7072         local $SIG{INT} =
7073             local $SIG{TERM} =
7074             local $SIG{QUIT} =
7075             local $SIG{HUP} =
7076             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7077         local $SIG{ALRM} = sub { die "got timeout\n"; };
7078
7079         $oldtimeout = alarm($timeout);
7080
7081         my $parser = sub {
7082             my $line = shift;
7083
7084             print "$line\n";
7085
7086             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7087                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7088                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7089             } elsif ($line =~ m/^CTIME: /) {
7090                 # we correctly received the vma config, so we can disable
7091                 # the timeout now for disk allocation (set to 10 minutes, so
7092                 # that we always timeout if something goes wrong)
7093                 alarm(600);
7094                 &$print_devmap();
7095                 print $fifofh "done\n";
7096                 my $tmp = $oldtimeout || 0;
7097                 $oldtimeout = undef;
7098                 alarm($tmp);
7099                 close($fifofh);
7100                 $fifofh = undef;
7101             }
7102         };
7103
7104         print "restore vma archive: $dbg_cmdstring\n";
7105         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7106     };
7107     my $err = $@;
7108
7109     alarm($oldtimeout) if $oldtimeout;
7110
7111     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7112
7113     close($fifofh) if $fifofh;
7114     unlink $mapfifo;
7115     rmtree $tmpdir;
7116
7117     if ($err) {
7118         $restore_destroy_volumes->($cfg, $virtdev_hash);
7119         die $err;
7120     }
7121
7122     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7123     PVE::QemuConfig->write_config($vmid, $new_conf);
7124
7125     eval { rescan($vmid, 1); };
7126     warn $@ if $@;
7127
7128     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7129 }
7130
7131 sub restore_tar_archive {
7132     my ($archive, $vmid, $user, $opts) = @_;
7133
7134     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7135         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7136         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7137     }
7138
7139     if ($archive ne '-') {
7140         my $firstfile = tar_archive_read_firstfile($archive);
7141         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7142             if $firstfile ne 'qemu-server.conf';
7143     }
7144
7145     my $storecfg = PVE::Storage::config();
7146
7147     # avoid zombie disks when restoring over an existing VM -> cleanup first
7148     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7149     # skiplock=1 because qmrestore has set the 'create' lock itself already
7150     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7151     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7152
7153     my $tocmd = "/usr/lib/qemu-server/qmextract";
7154
7155     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7156     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7157     $tocmd .= ' --prealloc' if $opts->{prealloc};
7158     $tocmd .= ' --info' if $opts->{info};
7159
7160     # tar option "xf" does not autodetect compression when read from STDIN,
7161     # so we pipe to zcat
7162     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7163         PVE::Tools::shellquote("--to-command=$tocmd");
7164
7165     my $tmpdir = "/var/tmp/vzdumptmp$$";
7166     mkpath $tmpdir;
7167
7168     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7169     local $ENV{VZDUMP_VMID} = $vmid;
7170     local $ENV{VZDUMP_USER} = $user;
7171
7172     my $conffile = PVE::QemuConfig->config_file($vmid);
7173     my $new_conf_raw = '';
7174
7175     # disable interrupts (always do cleanups)
7176     local $SIG{INT} =
7177         local $SIG{TERM} =
7178         local $SIG{QUIT} =
7179         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7180
7181     eval {
7182         # enable interrupts
7183         local $SIG{INT} =
7184             local $SIG{TERM} =
7185             local $SIG{QUIT} =
7186             local $SIG{HUP} =
7187             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7188
7189         if ($archive eq '-') {
7190             print "extracting archive from STDIN\n";
7191             run_command($cmd, input => "<&STDIN");
7192         } else {
7193             print "extracting archive '$archive'\n";
7194             run_command($cmd);
7195         }
7196
7197         return if $opts->{info};
7198
7199         # read new mapping
7200         my $map = {};
7201         my $statfile = "$tmpdir/qmrestore.stat";
7202         if (my $fd = IO::File->new($statfile, "r")) {
7203             while (defined (my $line = <$fd>)) {
7204                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7205                     $map->{$1} = $2 if $1;
7206                 } else {
7207                     print STDERR "unable to parse line in statfile - $line\n";
7208                 }
7209             }
7210             $fd->close();
7211         }
7212
7213         my $confsrc = "$tmpdir/qemu-server.conf";
7214
7215         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7216
7217         my $cookie = { netcount => 0 };
7218         while (defined (my $line = <$srcfd>)) {
7219             $new_conf_raw .= restore_update_config_line(
7220                 $cookie,
7221                 $map,
7222                 $line,
7223                 $opts->{unique},
7224             );
7225         }
7226
7227         $srcfd->close();
7228     };
7229     if (my $err = $@) {
7230         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7231         die $err;
7232     }
7233
7234     rmtree $tmpdir;
7235
7236     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7237
7238     PVE::Cluster::cfs_update(); # make sure we read new file
7239
7240     eval { rescan($vmid, 1); };
7241     warn $@ if $@;
7242 };
7243
7244 sub foreach_storage_used_by_vm {
7245     my ($conf, $func) = @_;
7246
7247     my $sidhash = {};
7248
7249     PVE::QemuConfig->foreach_volume($conf, sub {
7250         my ($ds, $drive) = @_;
7251         return if drive_is_cdrom($drive);
7252
7253         my $volid = $drive->{file};
7254
7255         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7256         $sidhash->{$sid} = $sid if $sid;
7257     });
7258
7259     foreach my $sid (sort keys %$sidhash) {
7260         &$func($sid);
7261     }
7262 }
7263
7264 my $qemu_snap_storage = {
7265     rbd => 1,
7266 };
7267 sub do_snapshots_with_qemu {
7268     my ($storecfg, $volid, $deviceid) = @_;
7269
7270     return if $deviceid =~ m/tpmstate0/;
7271
7272     my $storage_name = PVE::Storage::parse_volume_id($volid);
7273     my $scfg = $storecfg->{ids}->{$storage_name};
7274     die "could not find storage '$storage_name'\n" if !defined($scfg);
7275
7276     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7277         return 1;
7278     }
7279
7280     if ($volid =~ m/\.(qcow2|qed)$/){
7281         return 1;
7282     }
7283
7284     return;
7285 }
7286
7287 sub qga_check_running {
7288     my ($vmid, $nowarn) = @_;
7289
7290     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7291     if ($@) {
7292         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7293         return 0;
7294     }
7295     return 1;
7296 }
7297
7298 sub template_create {
7299     my ($vmid, $conf, $disk) = @_;
7300
7301     my $storecfg = PVE::Storage::config();
7302
7303     PVE::QemuConfig->foreach_volume($conf, sub {
7304         my ($ds, $drive) = @_;
7305
7306         return if drive_is_cdrom($drive);
7307         return if $disk && $ds ne $disk;
7308
7309         my $volid = $drive->{file};
7310         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7311
7312         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7313         $drive->{file} = $voliddst;
7314         $conf->{$ds} = print_drive($drive);
7315         PVE::QemuConfig->write_config($vmid, $conf);
7316     });
7317 }
7318
7319 sub convert_iscsi_path {
7320     my ($path) = @_;
7321
7322     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7323         my $portal = $1;
7324         my $target = $2;
7325         my $lun = $3;
7326
7327         my $initiator_name = get_initiator_name();
7328
7329         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7330                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7331     }
7332
7333     die "cannot convert iscsi path '$path', unkown format\n";
7334 }
7335
7336 sub qemu_img_convert {
7337     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7338
7339     my $storecfg = PVE::Storage::config();
7340     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7341     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7342
7343     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7344
7345     my $cachemode;
7346     my $src_path;
7347     my $src_is_iscsi = 0;
7348     my $src_format;
7349
7350     if ($src_storeid) {
7351         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7352         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7353         $src_format = qemu_img_format($src_scfg, $src_volname);
7354         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7355         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7356         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7357     } elsif (-f $src_volid || -b $src_volid) {
7358         $src_path = $src_volid;
7359         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7360             $src_format = $1;
7361         }
7362     }
7363
7364     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7365
7366     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7367     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7368     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7369     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7370
7371     my $cmd = [];
7372     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7373     push @$cmd, '-l', "snapshot.name=$snapname"
7374         if $snapname && $src_format && $src_format eq "qcow2";
7375     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7376     push @$cmd, '-T', $cachemode if defined($cachemode);
7377
7378     if ($src_is_iscsi) {
7379         push @$cmd, '--image-opts';
7380         $src_path = convert_iscsi_path($src_path);
7381     } elsif ($src_format) {
7382         push @$cmd, '-f', $src_format;
7383     }
7384
7385     if ($dst_is_iscsi) {
7386         push @$cmd, '--target-image-opts';
7387         $dst_path = convert_iscsi_path($dst_path);
7388     } else {
7389         push @$cmd, '-O', $dst_format;
7390     }
7391
7392     push @$cmd, $src_path;
7393
7394     if (!$dst_is_iscsi && $is_zero_initialized) {
7395         push @$cmd, "zeroinit:$dst_path";
7396     } else {
7397         push @$cmd, $dst_path;
7398     }
7399
7400     my $parser = sub {
7401         my $line = shift;
7402         if($line =~ m/\((\S+)\/100\%\)/){
7403             my $percent = $1;
7404             my $transferred = int($size * $percent / 100);
7405             my $total_h = render_bytes($size, 1);
7406             my $transferred_h = render_bytes($transferred, 1);
7407
7408             print "transferred $transferred_h of $total_h ($percent%)\n";
7409         }
7410
7411     };
7412
7413     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7414     my $err = $@;
7415     die "copy failed: $err" if $err;
7416 }
7417
7418 sub qemu_img_format {
7419     my ($scfg, $volname) = @_;
7420
7421     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7422         return $1;
7423     } else {
7424         return "raw";
7425     }
7426 }
7427
7428 sub qemu_drive_mirror {
7429     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7430
7431     $jobs = {} if !$jobs;
7432
7433     my $qemu_target;
7434     my $format;
7435     $jobs->{"drive-$drive"} = {};
7436
7437     if ($dst_volid =~ /^nbd:/) {
7438         $qemu_target = $dst_volid;
7439         $format = "nbd";
7440     } else {
7441         my $storecfg = PVE::Storage::config();
7442         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7443
7444         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7445
7446         $format = qemu_img_format($dst_scfg, $dst_volname);
7447
7448         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7449
7450         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7451     }
7452
7453     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7454     $opts->{format} = $format if $format;
7455
7456     if (defined($src_bitmap)) {
7457         $opts->{sync} = 'incremental';
7458         $opts->{bitmap} = $src_bitmap;
7459         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7460     }
7461
7462     if (defined($bwlimit)) {
7463         $opts->{speed} = $bwlimit * 1024;
7464         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7465     } else {
7466         print "drive mirror is starting for drive-$drive\n";
7467     }
7468
7469     # if a job already runs for this device we get an error, catch it for cleanup
7470     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7471     if (my $err = $@) {
7472         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7473         warn "$@\n" if $@;
7474         die "mirroring error: $err\n";
7475     }
7476
7477     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7478 }
7479
7480 # $completion can be either
7481 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7482 # 'cancel': wait until all jobs are ready, block-job-cancel them
7483 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7484 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7485 sub qemu_drive_mirror_monitor {
7486     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7487
7488     $completion //= 'complete';
7489     $op //= "mirror";
7490
7491     eval {
7492         my $err_complete = 0;
7493
7494         my $starttime = time ();
7495         while (1) {
7496             die "block job ('$op') timed out\n" if $err_complete > 300;
7497
7498             my $stats = mon_cmd($vmid, "query-block-jobs");
7499             my $ctime = time();
7500
7501             my $running_jobs = {};
7502             for my $stat (@$stats) {
7503                 next if $stat->{type} ne $op;
7504                 $running_jobs->{$stat->{device}} = $stat;
7505             }
7506
7507             my $readycounter = 0;
7508
7509             for my $job_id (sort keys %$jobs) {
7510                 my $job = $running_jobs->{$job_id};
7511
7512                 my $vanished = !defined($job);
7513                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7514                 if($complete || ($vanished && $completion eq 'auto')) {
7515                     print "$job_id: $op-job finished\n";
7516                     delete $jobs->{$job_id};
7517                     next;
7518                 }
7519
7520                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7521
7522                 my $busy = $job->{busy};
7523                 my $ready = $job->{ready};
7524                 if (my $total = $job->{len}) {
7525                     my $transferred = $job->{offset} || 0;
7526                     my $remaining = $total - $transferred;
7527                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7528
7529                     my $duration = $ctime - $starttime;
7530                     my $total_h = render_bytes($total, 1);
7531                     my $transferred_h = render_bytes($transferred, 1);
7532
7533                     my $status = sprintf(
7534                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7535                         render_duration($duration),
7536                     );
7537
7538                     if ($ready) {
7539                         if ($busy) {
7540                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7541                         } else {
7542                             $status .= ", ready";
7543                         }
7544                     }
7545                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7546                     $jobs->{$job_id}->{ready} = $ready;
7547                 }
7548
7549                 $readycounter++ if $job->{ready};
7550             }
7551
7552             last if scalar(keys %$jobs) == 0;
7553
7554             if ($readycounter == scalar(keys %$jobs)) {
7555                 print "all '$op' jobs are ready\n";
7556
7557                 # do the complete later (or has already been done)
7558                 last if $completion eq 'skip' || $completion eq 'auto';
7559
7560                 if ($vmiddst && $vmiddst != $vmid) {
7561                     my $agent_running = $qga && qga_check_running($vmid);
7562                     if ($agent_running) {
7563                         print "freeze filesystem\n";
7564                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7565                         warn $@ if $@;
7566                     } else {
7567                         print "suspend vm\n";
7568                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7569                         warn $@ if $@;
7570                     }
7571
7572                     # if we clone a disk for a new target vm, we don't switch the disk
7573                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7574
7575                     if ($agent_running) {
7576                         print "unfreeze filesystem\n";
7577                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7578                         warn $@ if $@;
7579                     } else {
7580                         print "resume vm\n";
7581                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7582                         warn $@ if $@;
7583                     }
7584
7585                     last;
7586                 } else {
7587
7588                     for my $job_id (sort keys %$jobs) {
7589                         # try to switch the disk if source and destination are on the same guest
7590                         print "$job_id: Completing block job_id...\n";
7591
7592                         my $op;
7593                         if ($completion eq 'complete') {
7594                             $op = 'block-job-complete';
7595                         } elsif ($completion eq 'cancel') {
7596                             $op = 'block-job-cancel';
7597                         } else {
7598                             die "invalid completion value: $completion\n";
7599                         }
7600                         eval { mon_cmd($vmid, $op, device => $job_id) };
7601                         if ($@ =~ m/cannot be completed/) {
7602                             print "$job_id: block job cannot be completed, trying again.\n";
7603                             $err_complete++;
7604                         }else {
7605                             print "$job_id: Completed successfully.\n";
7606                             $jobs->{$job_id}->{complete} = 1;
7607                         }
7608                     }
7609                 }
7610             }
7611             sleep 1;
7612         }
7613     };
7614     my $err = $@;
7615
7616     if ($err) {
7617         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7618         die "block job ($op) error: $err";
7619     }
7620 }
7621
7622 sub qemu_blockjobs_cancel {
7623     my ($vmid, $jobs) = @_;
7624
7625     foreach my $job (keys %$jobs) {
7626         print "$job: Cancelling block job\n";
7627         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7628         $jobs->{$job}->{cancel} = 1;
7629     }
7630
7631     while (1) {
7632         my $stats = mon_cmd($vmid, "query-block-jobs");
7633
7634         my $running_jobs = {};
7635         foreach my $stat (@$stats) {
7636             $running_jobs->{$stat->{device}} = $stat;
7637         }
7638
7639         foreach my $job (keys %$jobs) {
7640
7641             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7642                 print "$job: Done.\n";
7643                 delete $jobs->{$job};
7644             }
7645         }
7646
7647         last if scalar(keys %$jobs) == 0;
7648
7649         sleep 1;
7650     }
7651 }
7652
7653 sub clone_disk {
7654     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7655
7656     my ($vmid, $running) = $source->@{qw(vmid running)};
7657     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7658
7659     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7660     my ($storage, $format) = $dest->@{qw(storage format)};
7661
7662     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7663
7664     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7665         die "cloning from/to EFI disk requires EFI disk\n"
7666             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7667         die "cloning from/to TPM state requires TPM state\n"
7668             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7669
7670         # This would lead to two device nodes in QEMU pointing to the same backing image!
7671         die "cannot change drive name when cloning disk from/to the same VM\n"
7672             if $use_drive_mirror && $vmid == $newvmid;
7673     }
7674
7675     die "cannot move TPM state while VM is running\n"
7676         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7677
7678     my $newvolid;
7679
7680     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7681     print "$src_drivename " if $src_drivename;
7682     print "($drive->{file})\n";
7683
7684     if (!$full) {
7685         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7686         push @$newvollist, $newvolid;
7687     } else {
7688
7689         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7690         $storeid = $storage if $storage;
7691
7692         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7693
7694         my $name = undef;
7695         my $size = undef;
7696         if (drive_is_cloudinit($drive)) {
7697             $name = "vm-$newvmid-cloudinit";
7698             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7699             if ($scfg->{path}) {
7700                 $name .= ".$dst_format";
7701             }
7702             $snapname = undef;
7703             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7704         } elsif ($dst_drivename eq 'efidisk0') {
7705             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7706         } elsif ($dst_drivename eq 'tpmstate0') {
7707             $dst_format = 'raw';
7708             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7709         } else {
7710             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7711         }
7712         $newvolid = PVE::Storage::vdisk_alloc(
7713             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7714         );
7715         push @$newvollist, $newvolid;
7716
7717         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7718
7719         if (drive_is_cloudinit($drive)) {
7720             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7721             # if this is the case, we have to complete any block-jobs still there from
7722             # previous drive-mirrors
7723             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7724                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7725             }
7726             goto no_data_clone;
7727         }
7728
7729         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7730         if ($use_drive_mirror) {
7731             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7732                 $completion, $qga, $bwlimit);
7733         } else {
7734             # TODO: handle bwlimits
7735             if ($dst_drivename eq 'efidisk0') {
7736                 # the relevant data on the efidisk may be smaller than the source
7737                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7738                 # that is given by the OVMF_VARS.fd
7739                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7740                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7741
7742                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7743
7744                 # better for Ceph if block size is not too small, see bug #3324
7745                 my $bs = 1024*1024;
7746
7747                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7748
7749                 if ($src_format eq 'qcow2' && $snapname) {
7750                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7751                         if !min_version(kvm_user_version(), 6, 2);
7752                     push $cmd->@*, '-l', $snapname;
7753                 }
7754                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7755                 run_command($cmd);
7756             } else {
7757                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7758             }
7759         }
7760     }
7761
7762 no_data_clone:
7763     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7764
7765     my $disk = dclone($drive);
7766     delete $disk->{format};
7767     $disk->{file} = $newvolid;
7768     $disk->{size} = $size if defined($size);
7769
7770     return $disk;
7771 }
7772
7773 sub get_running_qemu_version {
7774     my ($vmid) = @_;
7775     my $res = mon_cmd($vmid, "query-version");
7776     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7777 }
7778
7779 sub qemu_use_old_bios_files {
7780     my ($machine_type) = @_;
7781
7782     return if !$machine_type;
7783
7784     my $use_old_bios_files = undef;
7785
7786     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7787         $machine_type = $1;
7788         $use_old_bios_files = 1;
7789     } else {
7790         my $version = extract_version($machine_type, kvm_user_version());
7791         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7792         # load new efi bios files on migration. So this hack is required to allow
7793         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7794         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7795         $use_old_bios_files = !min_version($version, 2, 4);
7796     }
7797
7798     return ($use_old_bios_files, $machine_type);
7799 }
7800
7801 sub get_efivars_size {
7802     my ($conf, $efidisk) = @_;
7803
7804     my $arch = get_vm_arch($conf);
7805     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7806     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7807     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7808     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7809     return -s $ovmf_vars;
7810 }
7811
7812 sub update_efidisk_size {
7813     my ($conf) = @_;
7814
7815     return if !defined($conf->{efidisk0});
7816
7817     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7818     $disk->{size} = get_efivars_size($conf);
7819     $conf->{efidisk0} = print_drive($disk);
7820
7821     return;
7822 }
7823
7824 sub update_tpmstate_size {
7825     my ($conf) = @_;
7826
7827     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7828     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7829     $conf->{tpmstate0} = print_drive($disk);
7830 }
7831
7832 sub create_efidisk($$$$$$$) {
7833     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7834
7835     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7836     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7837
7838     my $vars_size_b = -s $ovmf_vars;
7839     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7840     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7841     PVE::Storage::activate_volumes($storecfg, [$volid]);
7842
7843     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7844     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7845
7846     return ($volid, $size/1024);
7847 }
7848
7849 sub vm_iothreads_list {
7850     my ($vmid) = @_;
7851
7852     my $res = mon_cmd($vmid, 'query-iothreads');
7853
7854     my $iothreads = {};
7855     foreach my $iothread (@$res) {
7856         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7857     }
7858
7859     return $iothreads;
7860 }
7861
7862 sub scsihw_infos {
7863     my ($conf, $drive) = @_;
7864
7865     my $maxdev = 0;
7866
7867     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7868         $maxdev = 7;
7869     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7870         $maxdev = 1;
7871     } else {
7872         $maxdev = 256;
7873     }
7874
7875     my $controller = int($drive->{index} / $maxdev);
7876     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7877         ? "virtioscsi"
7878         : "scsihw";
7879
7880     return ($maxdev, $controller, $controller_prefix);
7881 }
7882
7883 sub windows_version {
7884     my ($ostype) = @_;
7885
7886     return 0 if !$ostype;
7887
7888     my $winversion = 0;
7889
7890     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7891         $winversion = 5;
7892     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7893         $winversion = 6;
7894     } elsif ($ostype =~ m/^win(\d+)$/) {
7895         $winversion = $1;
7896     }
7897
7898     return $winversion;
7899 }
7900
7901 sub resolve_dst_disk_format {
7902         my ($storecfg, $storeid, $src_volname, $format) = @_;
7903         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7904
7905         if (!$format) {
7906             # if no target format is specified, use the source disk format as hint
7907             if ($src_volname) {
7908                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7909                 $format = qemu_img_format($scfg, $src_volname);
7910             } else {
7911                 return $defFormat;
7912             }
7913         }
7914
7915         # test if requested format is supported - else use default
7916         my $supported = grep { $_ eq $format } @$validFormats;
7917         $format = $defFormat if !$supported;
7918         return $format;
7919 }
7920
7921 # NOTE: if this logic changes, please update docs & possibly gui logic
7922 sub find_vmstate_storage {
7923     my ($conf, $storecfg) = @_;
7924
7925     # first, return storage from conf if set
7926     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7927
7928     my ($target, $shared, $local);
7929
7930     foreach_storage_used_by_vm($conf, sub {
7931         my ($sid) = @_;
7932         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7933         my $dst = $scfg->{shared} ? \$shared : \$local;
7934         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7935     });
7936
7937     # second, use shared storage where VM has at least one disk
7938     # third, use local storage where VM has at least one disk
7939     # fall back to local storage
7940     $target = $shared // $local // 'local';
7941
7942     return $target;
7943 }
7944
7945 sub generate_uuid {
7946     my ($uuid, $uuid_str);
7947     UUID::generate($uuid);
7948     UUID::unparse($uuid, $uuid_str);
7949     return $uuid_str;
7950 }
7951
7952 sub generate_smbios1_uuid {
7953     return "uuid=".generate_uuid();
7954 }
7955
7956 sub nbd_stop {
7957     my ($vmid) = @_;
7958
7959     mon_cmd($vmid, 'nbd-server-stop');
7960 }
7961
7962 sub create_reboot_request {
7963     my ($vmid) = @_;
7964     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7965         or die "failed to create reboot trigger file: $!\n";
7966     close($fh);
7967 }
7968
7969 sub clear_reboot_request {
7970     my ($vmid) = @_;
7971     my $path = "/run/qemu-server/$vmid.reboot";
7972     my $res = 0;
7973
7974     $res = unlink($path);
7975     die "could not remove reboot request for $vmid: $!"
7976         if !$res && $! != POSIX::ENOENT;
7977
7978     return $res;
7979 }
7980
7981 sub bootorder_from_legacy {
7982     my ($conf, $bootcfg) = @_;
7983
7984     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7985     my $bootindex_hash = {};
7986     my $i = 1;
7987     foreach my $o (split(//, $boot)) {
7988         $bootindex_hash->{$o} = $i*100;
7989         $i++;
7990     }
7991
7992     my $bootorder = {};
7993
7994     PVE::QemuConfig->foreach_volume($conf, sub {
7995         my ($ds, $drive) = @_;
7996
7997         if (drive_is_cdrom ($drive, 1)) {
7998             if ($bootindex_hash->{d}) {
7999                 $bootorder->{$ds} = $bootindex_hash->{d};
8000                 $bootindex_hash->{d} += 1;
8001             }
8002         } elsif ($bootindex_hash->{c}) {
8003             $bootorder->{$ds} = $bootindex_hash->{c}
8004                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8005             $bootindex_hash->{c} += 1;
8006         }
8007     });
8008
8009     if ($bootindex_hash->{n}) {
8010         for (my $i = 0; $i < $MAX_NETS; $i++) {
8011             my $netname = "net$i";
8012             next if !$conf->{$netname};
8013             $bootorder->{$netname} = $bootindex_hash->{n};
8014             $bootindex_hash->{n} += 1;
8015         }
8016     }
8017
8018     return $bootorder;
8019 }
8020
8021 # Generate default device list for 'boot: order=' property. Matches legacy
8022 # default boot order, but with explicit device names. This is important, since
8023 # the fallback for when neither 'order' nor the old format is specified relies
8024 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8025 sub get_default_bootdevices {
8026     my ($conf) = @_;
8027
8028     my @ret = ();
8029
8030     # harddisk
8031     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8032     push @ret, $first if $first;
8033
8034     # cdrom
8035     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8036     push @ret, $first if $first;
8037
8038     # network
8039     for (my $i = 0; $i < $MAX_NETS; $i++) {
8040         my $netname = "net$i";
8041         next if !$conf->{$netname};
8042         push @ret, $netname;
8043         last;
8044     }
8045
8046     return \@ret;
8047 }
8048
8049 sub device_bootorder {
8050     my ($conf) = @_;
8051
8052     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8053
8054     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8055
8056     my $bootorder = {};
8057     if (!defined($boot) || $boot->{legacy}) {
8058         $bootorder = bootorder_from_legacy($conf, $boot);
8059     } elsif ($boot->{order}) {
8060         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8061         for my $dev (PVE::Tools::split_list($boot->{order})) {
8062             $bootorder->{$dev} = $i++;
8063         }
8064     }
8065
8066     return $bootorder;
8067 }
8068
8069 sub register_qmeventd_handle {
8070     my ($vmid) = @_;
8071
8072     my $fh;
8073     my $peer = "/var/run/qmeventd.sock";
8074     my $count = 0;
8075
8076     for (;;) {
8077         $count++;
8078         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8079         last if $fh;
8080         if ($! != EINTR && $! != EAGAIN) {
8081             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8082         }
8083         if ($count > 4) {
8084             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8085               . "after $count retries\n";
8086         }
8087         usleep(25000);
8088     }
8089
8090     # send handshake to mark VM as backing up
8091     print $fh to_json({vzdump => {vmid => "$vmid"}});
8092
8093     # return handle to be closed later when inhibit is no longer required
8094     return $fh;
8095 }
8096
8097 # bash completion helper
8098
8099 sub complete_backup_archives {
8100     my ($cmdname, $pname, $cvalue) = @_;
8101
8102     my $cfg = PVE::Storage::config();
8103
8104     my $storeid;
8105
8106     if ($cvalue =~ m/^([^:]+):/) {
8107         $storeid = $1;
8108     }
8109
8110     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8111
8112     my $res = [];
8113     foreach my $id (keys %$data) {
8114         foreach my $item (@{$data->{$id}}) {
8115             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8116             push @$res, $item->{volid} if defined($item->{volid});
8117         }
8118     }
8119
8120     return $res;
8121 }
8122
8123 my $complete_vmid_full = sub {
8124     my ($running) = @_;
8125
8126     my $idlist = vmstatus();
8127
8128     my $res = [];
8129
8130     foreach my $id (keys %$idlist) {
8131         my $d = $idlist->{$id};
8132         if (defined($running)) {
8133             next if $d->{template};
8134             next if $running && $d->{status} ne 'running';
8135             next if !$running && $d->{status} eq 'running';
8136         }
8137         push @$res, $id;
8138
8139     }
8140     return $res;
8141 };
8142
8143 sub complete_vmid {
8144     return &$complete_vmid_full();
8145 }
8146
8147 sub complete_vmid_stopped {
8148     return &$complete_vmid_full(0);
8149 }
8150
8151 sub complete_vmid_running {
8152     return &$complete_vmid_full(1);
8153 }
8154
8155 sub complete_storage {
8156
8157     my $cfg = PVE::Storage::config();
8158     my $ids = $cfg->{ids};
8159
8160     my $res = [];
8161     foreach my $sid (keys %$ids) {
8162         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8163         next if !$ids->{$sid}->{content}->{images};
8164         push @$res, $sid;
8165     }
8166
8167     return $res;
8168 }
8169
8170 sub complete_migration_storage {
8171     my ($cmd, $param, $current_value, $all_args) = @_;
8172
8173     my $targetnode = @$all_args[1];
8174
8175     my $cfg = PVE::Storage::config();
8176     my $ids = $cfg->{ids};
8177
8178     my $res = [];
8179     foreach my $sid (keys %$ids) {
8180         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8181         next if !$ids->{$sid}->{content}->{images};
8182         push @$res, $sid;
8183     }
8184
8185     return $res;
8186 }
8187
8188 sub vm_is_paused {
8189     my ($vmid) = @_;
8190     my $qmpstatus = eval {
8191         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8192         mon_cmd($vmid, "query-status");
8193     };
8194     warn "$@\n" if $@;
8195     return $qmpstatus && $qmpstatus->{status} eq "paused";
8196 }
8197
8198 sub check_volume_storage_type {
8199     my ($storecfg, $vol) = @_;
8200
8201     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8202     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8203     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8204
8205     die "storage '$storeid' does not support content-type '$vtype'\n"
8206         if !$scfg->{content}->{$vtype};
8207
8208     return 1;
8209 }
8210
8211 1;