PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 123     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 124     type => 'string',
 125     format => 'storage-pair-list',
 126     optional => 1,
 127 });
 128
 129 #no warnings 'redefine';
 130
 131 my $nodename_cache;
 132 sub nodename {
 133     $nodename_cache //= PVE::INotify::nodename();
 134     return $nodename_cache;
 135 }
 136
 137 my $watchdog_fmt = {
 138     model => {
 139         default_key => 1,
 140         type => 'string',
 141         enum => [qw(i6300esb ib700)],
 142         description => "Watchdog type to emulate.",
 143         default => 'i6300esb',
 144         optional => 1,
 145     },
 146     action => {
 147         type => 'string',
 148         enum => [qw(reset shutdown poweroff pause debug none)],
 149         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 150         optional => 1,
 151     },
 152 };
 153 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 154
 155 my $agent_fmt = {
 156     enabled => {
 157         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 158         type => 'boolean',
 159         default => 0,
 160         default_key => 1,
 161     },
 162     fstrim_cloned_disks => {
 163         description => "Run fstrim after moving a disk or migrating the VM.",
 164         type => 'boolean',
 165         optional => 1,
 166         default => 0
 167     },
 168     type => {
 169         description => "Select the agent type",
 170         type => 'string',
 171         default => 'virtio',
 172         optional => 1,
 173         enum => [qw(virtio isa)],
 174     },
 175 };
 176
 177 my $vga_fmt = {
 178     type => {
 179         description => "Select the VGA type.",
 180         type => 'string',
 181         default => 'std',
 182         optional => 1,
 183         default_key => 1,
 184         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio virtio-gl vmware)],
 185     },
 186     memory => {
 187         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 188         type => 'integer',
 189         optional => 1,
 190         minimum => 4,
 191         maximum => 512,
 192     },
 193 };
 194
 195 my $ivshmem_fmt = {
 196     size => {
 197         type => 'integer',
 198         minimum => 1,
 199         description => "The size of the file in MB.",
 200     },
 201     name => {
 202         type => 'string',
 203         pattern => '[a-zA-Z0-9\-]+',
 204         optional => 1,
 205         format_description => 'string',
 206         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 207     },
 208 };
 209
 210 my $audio_fmt = {
 211     device => {
 212         type => 'string',
 213         enum => [qw(ich9-intel-hda intel-hda AC97)],
 214         description =>  "Configure an audio device."
 215     },
 216     driver =>  {
 217         type => 'string',
 218         enum => ['spice', 'none'],
 219         default => 'spice',
 220         optional => 1,
 221         description => "Driver backend for the audio device."
 222     },
 223 };
 224
 225 my $spice_enhancements_fmt = {
 226     foldersharing => {
 227         type => 'boolean',
 228         optional => 1,
 229         default => '0',
 230         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 231     },
 232     videostreaming =>  {
 233         type => 'string',
 234         enum => ['off', 'all', 'filter'],
 235         default => 'off',
 236         optional => 1,
 237         description => "Enable video streaming. Uses compression for detected video streams."
 238     },
 239 };
 240
 241 my $rng_fmt = {
 242     source => {
 243         type => 'string',
 244         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 245         default_key => 1,
 246         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 247             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 248             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 249             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 250             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 251             ." a hardware RNG from the host.",
 252     },
 253     max_bytes => {
 254         type => 'integer',
 255         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 256             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 257             ." `0` to disable limiting (potentially dangerous!).",
 258         optional => 1,
 259
 260         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 261         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 262         # reading from /dev/urandom
 263         default => 1024,
 264     },
 265     period => {
 266         type => 'integer',
 267         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 268             ." the guest to retrieve another 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $meta_info_fmt = {
 275     'ctime' => {
 276         type => 'integer',
 277         description => "The guest creation timestamp as UNIX epoch time",
 278         minimum => 0,
 279         optional => 1,
 280     },
 281     'creation-qemu' => {
 282         type => 'string',
 283         description => "The QEMU (machine) version from the time this VM was created.",
 284         pattern => '\d+(\.\d+)+',
 285         optional => 1,
 286     },
 287 };
 288
 289 my $confdesc = {
 290     onboot => {
 291         optional => 1,
 292         type => 'boolean',
 293         description => "Specifies whether a VM will be started during system bootup.",
 294         default => 0,
 295     },
 296     autostart => {
 297         optional => 1,
 298         type => 'boolean',
 299         description => "Automatic restart after crash (currently ignored).",
 300         default => 0,
 301     },
 302     hotplug => {
 303         optional => 1,
 304         type => 'string', format => 'pve-hotplug-features',
 305         description => "Selectively enable hotplug features. This is a comma separated list of"
 306             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 307             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 308         default => 'network,disk,usb',
 309     },
 310     reboot => {
 311         optional => 1,
 312         type => 'boolean',
 313         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 314         default => 1,
 315     },
 316     lock => {
 317         optional => 1,
 318         type => 'string',
 319         description => "Lock/unlock the VM.",
 320         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 321     },
 322     cpulimit => {
 323         optional => 1,
 324         type => 'number',
 325         description => "Limit of CPU usage.",
 326         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 327             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 328         minimum => 0,
 329         maximum => 128,
 330         default => 0,
 331     },
 332     cpuunits => {
 333         optional => 1,
 334         type => 'integer',
 335         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 336         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 337             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 338             ." weights of all the other running VMs.",
 339         minimum => 1,
 340         maximum => 262144,
 341         default => 'cgroup v1: 1024, cgroup v2: 100',
 342     },
 343     memory => {
 344         optional => 1,
 345         type => 'integer',
 346         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 347             ." you use the balloon device.",
 348         minimum => 16,
 349         default => 512,
 350     },
 351     balloon => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 355         minimum => 0,
 356     },
 357     shares => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 361             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 362             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 363         minimum => 0,
 364         maximum => 50000,
 365         default => 1000,
 366     },
 367     keyboard => {
 368         optional => 1,
 369         type => 'string',
 370         description => "Keyboard layout for VNC server. This option is generally not required and"
 371         ." is often better handled from within the guest OS.",
 372         enum => PVE::Tools::kvmkeymaplist(),
 373         default => undef,
 374     },
 375     name => {
 376         optional => 1,
 377         type => 'string', format => 'dns-name',
 378         description => "Set a name for the VM. Only used on the configuration web interface.",
 379     },
 380     scsihw => {
 381         optional => 1,
 382         type => 'string',
 383         description => "SCSI controller model",
 384         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 385         default => 'lsi',
 386     },
 387     description => {
 388         optional => 1,
 389         type => 'string',
 390         description => "Description for the VM. Shown in the web-interface VM's summary."
 391             ." This is saved as comment inside the configuration file.",
 392         maxLength => 1024 * 8,
 393     },
 394     ostype => {
 395         optional => 1,
 396         type => 'string',
 397         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 398         description => "Specify guest operating system.",
 399         verbose_description => <<EODESC,
 400 Specify guest operating system. This is used to enable special
 401 optimization/features for specific operating systems:
 402
 403 [horizontal]
 404 other;; unspecified OS
 405 wxp;; Microsoft Windows XP
 406 w2k;; Microsoft Windows 2000
 407 w2k3;; Microsoft Windows 2003
 408 w2k8;; Microsoft Windows 2008
 409 wvista;; Microsoft Windows Vista
 410 win7;; Microsoft Windows 7
 411 win8;; Microsoft Windows 8/2012/2012r2
 412 win10;; Microsoft Windows 10/2016/2019
 413 win11;; Microsoft Windows 11/2022
 414 l24;; Linux 2.4 Kernel
 415 l26;; Linux 2.6 - 5.X Kernel
 416 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 417 EODESC
 418     },
 419     boot => {
 420         optional => 1,
 421         type => 'string', format => 'pve-qm-boot',
 422         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 423             ." key or 'legacy=' is deprecated.",
 424     },
 425     bootdisk => {
 426         optional => 1,
 427         type => 'string', format => 'pve-qm-bootdisk',
 428         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 429         pattern => '(ide|sata|scsi|virtio)\d+',
 430     },
 431     smp => {
 432         optional => 1,
 433         type => 'integer',
 434         description => "The number of CPUs. Please use option -sockets instead.",
 435         minimum => 1,
 436         default => 1,
 437     },
 438     sockets => {
 439         optional => 1,
 440         type => 'integer',
 441         description => "The number of CPU sockets.",
 442         minimum => 1,
 443         default => 1,
 444     },
 445     cores => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of cores per socket.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     numa => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable NUMA.",
 456         default => 0,
 457     },
 458     hugepages => {
 459         optional => 1,
 460         type => 'string',
 461         description => "Enable/disable hugepages memory.",
 462         enum => [qw(any 2 1024)],
 463     },
 464     keephugepages => {
 465         optional => 1,
 466         type => 'boolean',
 467         default => 0,
 468         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 469             ." after VM shutdown and can be used for subsequent starts.",
 470     },
 471     vcpus => {
 472         optional => 1,
 473         type => 'integer',
 474         description => "Number of hotplugged vcpus.",
 475         minimum => 1,
 476         default => 0,
 477     },
 478     acpi => {
 479         optional => 1,
 480         type => 'boolean',
 481         description => "Enable/disable ACPI.",
 482         default => 1,
 483     },
 484     agent => {
 485         optional => 1,
 486         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 487         type => 'string',
 488         format => $agent_fmt,
 489     },
 490     kvm => {
 491         optional => 1,
 492         type => 'boolean',
 493         description => "Enable/disable KVM hardware virtualization.",
 494         default => 1,
 495     },
 496     tdf => {
 497         optional => 1,
 498         type => 'boolean',
 499         description => "Enable/disable time drift fix.",
 500         default => 0,
 501     },
 502     localtime => {
 503         optional => 1,
 504         type => 'boolean',
 505         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 506             ." the `ostype` indicates a Microsoft Windows OS.",
 507     },
 508     freeze => {
 509         optional => 1,
 510         type => 'boolean',
 511         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 512     },
 513     vga => {
 514         optional => 1,
 515         type => 'string', format => $vga_fmt,
 516         description => "Configure the VGA hardware.",
 517         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 518             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 519             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 520             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 521             ." display server. For win* OS you can select how many independent displays you want,"
 522             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 523             ." using a serial device as terminal.",
 524     },
 525     watchdog => {
 526         optional => 1,
 527         type => 'string', format => 'pve-qm-watchdog',
 528         description => "Create a virtual hardware watchdog device.",
 529         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 530             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 531             ." else the watchdog will reset the guest (or execute the respective action specified)",
 532     },
 533     startdate => {
 534         optional => 1,
 535         type => 'string',
 536         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 537         description => "Set the initial date of the real time clock. Valid format for date are:"
 538             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 539         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 540         default => 'now',
 541     },
 542     startup =>  get_standard_option('pve-startup-order'),
 543     template => {
 544         optional => 1,
 545         type => 'boolean',
 546         description => "Enable/disable Template.",
 547         default => 0,
 548     },
 549     args => {
 550         optional => 1,
 551         type => 'string',
 552         description => "Arbitrary arguments passed to kvm.",
 553         verbose_description => <<EODESCR,
 554 Arbitrary arguments passed to kvm, for example:
 555
 556 args: -no-reboot -no-hpet
 557
 558 NOTE: this option is for experts only.
 559 EODESCR
 560     },
 561     tablet => {
 562         optional => 1,
 563         type => 'boolean',
 564         default => 1,
 565         description => "Enable/disable the USB tablet device.",
 566         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 567             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 568             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 569             ." may consider disabling this to save some context switches. This is turned off by"
 570             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 571     },
 572     migrate_speed => {
 573         optional => 1,
 574         type => 'integer',
 575         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 576         minimum => 0,
 577         default => 0,
 578     },
 579     migrate_downtime => {
 580         optional => 1,
 581         type => 'number',
 582         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 583         minimum => 0,
 584         default => 0.1,
 585     },
 586     cdrom => {
 587         optional => 1,
 588         type => 'string', format => 'pve-qm-ide',
 589         typetext => '<volume>',
 590         description => "This is an alias for option -ide2",
 591     },
 592     cpu => {
 593         optional => 1,
 594         description => "Emulated CPU type.",
 595         type => 'string',
 596         format => 'pve-vm-cpu-conf',
 597     },
 598     parent => get_standard_option('pve-snapshot-name', {
 599         optional => 1,
 600         description => "Parent snapshot name. This is used internally, and should not be modified.",
 601     }),
 602     snaptime => {
 603         optional => 1,
 604         description => "Timestamp for snapshots.",
 605         type => 'integer',
 606         minimum => 0,
 607     },
 608     vmstate => {
 609         optional => 1,
 610         type => 'string', format => 'pve-volume-id',
 611         description => "Reference to a volume which stores the VM state. This is used internally"
 612             ." for snapshots.",
 613     },
 614     vmstatestorage => get_standard_option('pve-storage-id', {
 615         description => "Default storage for VM state volumes/files.",
 616         optional => 1,
 617     }),
 618     runningmachine => get_standard_option('pve-qemu-machine', {
 619         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 620             ." for snapshots.",
 621     }),
 622     runningcpu => {
 623         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 624             ." internally for snapshots.",
 625         optional => 1,
 626         type => 'string',
 627         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 628         format_description => 'QEMU -cpu parameter'
 629     },
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 647             ." remove disk operations.",
 648         default => 0,
 649     },
 650     bios => {
 651         optional => 1,
 652         type => 'string',
 653         enum => [ qw(seabios ovmf) ],
 654         description => "Select BIOS implementation.",
 655         default => 'seabios',
 656     },
 657     vmgenid => {
 658         type => 'string',
 659         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 660         format_description => 'UUID',
 661         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 662             ." to disable explicitly.",
 663         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 664             ." value identifier to the guest OS. This allows to notify the guest operating system"
 665             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 666             ." execution or creation from a template). The guest operating system notices the"
 667             ." change, and is then able to react as appropriate by marking its copies of"
 668             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 669             ."Note that auto-creation only works when done through API/CLI create or update methods"
 670             .", but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 684             ." the host.",
 685         optional => 1,
 686     },
 687     audio0 => {
 688         type => 'string',
 689         format => $audio_fmt,
 690         description => "Configure a audio device, useful in combination with QXL/Spice.",
 691         optional => 1
 692     },
 693     spice_enhancements => {
 694         type => 'string',
 695         format => $spice_enhancements_fmt,
 696         description => "Configure additional enhancements for SPICE.",
 697         optional => 1
 698     },
 699     tags => {
 700         type => 'string', format => 'pve-tag-list',
 701         description => 'Tags of the VM. This is only meta information.',
 702         optional => 1,
 703     },
 704     rng0 => {
 705         type => 'string',
 706         format => $rng_fmt,
 707         description => "Configure a VirtIO-based Random Number Generator.",
 708         optional => 1,
 709     },
 710     meta => {
 711         type => 'string',
 712         format => $meta_info_fmt,
 713         description => "Some (read-only) meta-information about this guest.",
 714         optional => 1,
 715     },
 716 };
 717
 718 my $cicustom_fmt = {
 719     meta => {
 720         type => 'string',
 721         optional => 1,
 722         description => 'Specify a custom file containing all meta data passed to the VM via"
 723             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 724         format => 'pve-volume-id',
 725         format_description => 'volume',
 726     },
 727     network => {
 728         type => 'string',
 729         optional => 1,
 730         description => 'Specify a custom file containing all network data passed to the VM via'
 731             .' cloud-init.',
 732         format => 'pve-volume-id',
 733         format_description => 'volume',
 734     },
 735     user => {
 736         type => 'string',
 737         optional => 1,
 738         description => 'Specify a custom file containing all user data passed to the VM via'
 739             .' cloud-init.',
 740         format => 'pve-volume-id',
 741         format_description => 'volume',
 742     },
 743     vendor => {
 744     type => 'string',
 745     optional => 1,
 746     description => 'Specify a custom file containing all vendor data passed to the VM via'
 747      .' cloud-init.',
 748     format => 'pve-volume-id',
 749     format_description => 'volume',
 750     },
 751 };
 752 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 753
 754 my $confdesc_cloudinit = {
 755     citype => {
 756         optional => 1,
 757         type => 'string',
 758         description => 'Specifies the cloud-init configuration format. The default depends on the'
 759             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 760             .' and `configdrive2` for windows.',
 761         enum => ['configdrive2', 'nocloud', 'opennebula'],
 762     },
 763     ciuser => {
 764         optional => 1,
 765         type => 'string',
 766         description => "cloud-init: User name to change ssh keys and password for instead of the"
 767             ." image's configured default user.",
 768     },
 769     cipassword => {
 770         optional => 1,
 771         type => 'string',
 772         description => 'cloud-init: Password to assign the user. Using this is generally not'
 773             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 774             .' support hashed passwords.',
 775     },
 776     cicustom => {
 777         optional => 1,
 778         type => 'string',
 779         description => 'cloud-init: Specify custom files to replace the automatically generated'
 780             .' ones at start.',
 781         format => 'pve-qm-cicustom',
 782     },
 783     searchdomain => {
 784         optional => 1,
 785         type => 'string',
 786         description => 'cloud-init: Sets DNS search domains for a container. Create will'
 787             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 788             .' are set.',
 789     },
 790     nameserver => {
 791         optional => 1,
 792         type => 'string', format => 'address-list',
 793         description => 'cloud-init: Sets DNS server IP address for a container. Create will'
 794             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 795             .' are set.',
 796     },
 797     sshkeys => {
 798         optional => 1,
 799         type => 'string',
 800         format => 'urlencoded',
 801         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 802     },
 803 };
 804
 805 # what about other qemu settings ?
 806 #cpu => 'string',
 807 #machine => 'string',
 808 #fda => 'file',
 809 #fdb => 'file',
 810 #mtdblock => 'file',
 811 #sd => 'file',
 812 #pflash => 'file',
 813 #snapshot => 'bool',
 814 #bootp => 'file',
 815 ##tftp => 'dir',
 816 ##smb => 'dir',
 817 #kernel => 'file',
 818 #append => 'string',
 819 #initrd => 'file',
 820 ##soundhw => 'string',
 821
 822 while (my ($k, $v) = each %$confdesc) {
 823     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 824 }
 825
 826 my $MAX_USB_DEVICES = 5;
 827 my $MAX_NETS = 32;
 828 my $MAX_SERIAL_PORTS = 4;
 829 my $MAX_PARALLEL_PORTS = 3;
 830 my $MAX_NUMA = 8;
 831
 832 my $numa_fmt = {
 833     cpus => {
 834         type => "string",
 835         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 836         description => "CPUs accessing this NUMA node.",
 837         format_description => "id[-id];...",
 838     },
 839     memory => {
 840         type => "number",
 841         description => "Amount of memory this NUMA node provides.",
 842         optional => 1,
 843     },
 844     hostnodes => {
 845         type => "string",
 846         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 847         description => "Host NUMA nodes to use.",
 848         format_description => "id[-id];...",
 849         optional => 1,
 850     },
 851     policy => {
 852         type => 'string',
 853         enum => [qw(preferred bind interleave)],
 854         description => "NUMA allocation policy.",
 855         optional => 1,
 856     },
 857 };
 858 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 859 my $numadesc = {
 860     optional => 1,
 861     type => 'string', format => $numa_fmt,
 862     description => "NUMA topology.",
 863 };
 864 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 865
 866 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 867     $confdesc->{"numa$i"} = $numadesc;
 868 }
 869
 870 my $nic_model_list = [
 871     'e1000',
 872     'e1000-82540em',
 873     'e1000-82544gc',
 874     'e1000-82545em',
 875     'e1000e',
 876     'i82551',
 877     'i82557b',
 878     'i82559er',
 879     'ne2k_isa',
 880     'ne2k_pci',
 881     'pcnet',
 882     'rtl8139',
 883     'virtio',
 884     'vmxnet3',
 885 ];
 886 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 887
 888 my $net_fmt_bridge_descr = <<__EOD__;
 889 Bridge to attach the network device to. The Proxmox VE standard bridge
 890 is called 'vmbr0'.
 891
 892 If you do not specify a bridge, we create a kvm user (NATed) network
 893 device, which provides DHCP and DNS services. The following addresses
 894 are used:
 895
 896  10.0.2.2   Gateway
 897  10.0.2.3   DNS Server
 898  10.0.2.4   SMB Server
 899
 900 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 901 __EOD__
 902
 903 my $net_fmt = {
 904     macaddr  => get_standard_option('mac-addr', {
 905         description => "MAC address. That address must be unique withing your network. This is"
 906             ." automatically generated if not specified.",
 907     }),
 908     model => {
 909         type => 'string',
 910         description => "Network Card Model. The 'virtio' model provides the best performance with"
 911             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 912             ." best to use 'e1000'.",
 913         enum => $nic_model_list,
 914         default_key => 1,
 915     },
 916     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 917     bridge => get_standard_option('pve-bridge-id', {
 918         description => $net_fmt_bridge_descr,
 919         optional => 1,
 920     }),
 921     queues => {
 922         type => 'integer',
 923         minimum => 0, maximum => 16,
 924         description => 'Number of packet queues to be used on the device.',
 925         optional => 1,
 926     },
 927     rate => {
 928         type => 'number',
 929         minimum => 0,
 930         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 931         optional => 1,
 932     },
 933     tag => {
 934         type => 'integer',
 935         minimum => 1, maximum => 4094,
 936         description => 'VLAN tag to apply to packets on this interface.',
 937         optional => 1,
 938     },
 939     trunks => {
 940         type => 'string',
 941         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 942         description => 'VLAN trunks to pass through this interface.',
 943         format_description => 'vlanid[;vlanid...]',
 944         optional => 1,
 945     },
 946     firewall => {
 947         type => 'boolean',
 948         description => 'Whether this interface should be protected by the firewall.',
 949         optional => 1,
 950     },
 951     link_down => {
 952         type => 'boolean',
 953         description => 'Whether this interface should be disconnected (like pulling the plug).',
 954         optional => 1,
 955     },
 956     mtu => {
 957         type => 'integer',
 958         minimum => 1, maximum => 65520,
 959         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 960         optional => 1,
 961     },
 962 };
 963
 964 my $netdesc = {
 965     optional => 1,
 966     type => 'string', format => $net_fmt,
 967     description => "Specify network devices.",
 968 };
 969
 970 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 971
 972 my $ipconfig_fmt = {
 973     ip => {
 974         type => 'string',
 975         format => 'pve-ipv4-config',
 976         format_description => 'IPv4Format/CIDR',
 977         description => 'IPv4 address in CIDR format.',
 978         optional => 1,
 979         default => 'dhcp',
 980     },
 981     gw => {
 982         type => 'string',
 983         format => 'ipv4',
 984         format_description => 'GatewayIPv4',
 985         description => 'Default gateway for IPv4 traffic.',
 986         optional => 1,
 987         requires => 'ip',
 988     },
 989     ip6 => {
 990         type => 'string',
 991         format => 'pve-ipv6-config',
 992         format_description => 'IPv6Format/CIDR',
 993         description => 'IPv6 address in CIDR format.',
 994         optional => 1,
 995         default => 'dhcp',
 996     },
 997     gw6 => {
 998         type => 'string',
 999         format => 'ipv6',
1000         format_description => 'GatewayIPv6',
1001         description => 'Default gateway for IPv6 traffic.',
1002         optional => 1,
1003         requires => 'ip6',
1004     },
1005 };
1006 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1007 my $ipconfigdesc = {
1008     optional => 1,
1009     type => 'string', format => 'pve-qm-ipconfig',
1010     description => <<'EODESCR',
1011 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1012
1013 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1014
1015 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1016 gateway should be provided.
1017 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1018 cloud-init 19.4 or newer.
1019
1020 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1021 dhcp on IPv4.
1022 EODESCR
1023 };
1024 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1025
1026 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1027     $confdesc->{"net$i"} = $netdesc;
1028     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1029 }
1030
1031 foreach my $key (keys %$confdesc_cloudinit) {
1032     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1033 }
1034
1035 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1036 sub verify_volume_id_or_qm_path {
1037     my ($volid, $noerr) = @_;
1038
1039     return $volid if $volid eq 'none' || $volid eq 'cdrom';
1040
1041     return verify_volume_id_or_absolute_path($volid, $noerr);
1042 }
1043
1044 PVE::JSONSchema::register_format('pve-volume-id-or-absolute-path', \&verify_volume_id_or_absolute_path);
1045 sub verify_volume_id_or_absolute_path {
1046     my ($volid, $noerr) = @_;
1047
1048     return $volid if $volid =~ m|^/|;
1049
1050     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1051     if ($@) {
1052         return if $noerr;
1053         die $@;
1054     }
1055     return $volid;
1056 }
1057
1058 my $usb_fmt = {
1059     host => {
1060         default_key => 1,
1061         type => 'string', format => 'pve-qm-usb-device',
1062         format_description => 'HOSTUSBDEVICE|spice',
1063         description => <<EODESCR,
1064 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1065
1066  'bus-port(.port)*' (decimal numbers) or
1067  'vendor_id:product_id' (hexadeciaml numbers) or
1068  'spice'
1069
1070 You can use the 'lsusb -t' command to list existing usb devices.
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1073 machines - use with special care.
1074
1075 The value 'spice' can be used to add a usb redirection devices for spice.
1076 EODESCR
1077     },
1078     usb3 => {
1079         optional => 1,
1080         type => 'boolean',
1081         description => "Specifies whether if given host option is a USB3 device or port.",
1082         default => 0,
1083     },
1084 };
1085
1086 my $usbdesc = {
1087     optional => 1,
1088     type => 'string', format => $usb_fmt,
1089     description => "Configure an USB device (n is 0 to 4).",
1090 };
1091 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1092
1093 my $serialdesc = {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '(/dev/.+|socket)',
1097         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1098         verbose_description =>  <<EODESCR,
1099 Create a serial device inside the VM (n is 0 to 3), and pass through a
1100 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1101 host side (use 'qm terminal' to open a terminal connection).
1102
1103 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1104 use with special care.
1105
1106 CAUTION: Experimental! User reported problems with this option.
1107 EODESCR
1108 };
1109
1110 my $paralleldesc= {
1111         optional => 1,
1112         type => 'string',
1113         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1114         description =>  "Map host parallel devices (n is 0 to 2).",
1115         verbose_description =>  <<EODESCR,
1116 Map host parallel devices (n is 0 to 2).
1117
1118 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1119 machines - use with special care.
1120
1121 CAUTION: Experimental! User reported problems with this option.
1122 EODESCR
1123 };
1124
1125 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1126     $confdesc->{"parallel$i"} = $paralleldesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1130     $confdesc->{"serial$i"} = $serialdesc;
1131 }
1132
1133 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1134     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1135 }
1136
1137 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1138     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1139 }
1140
1141 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1142     $confdesc->{"usb$i"} = $usbdesc;
1143 }
1144
1145 my $boot_fmt = {
1146     legacy => {
1147         optional => 1,
1148         default_key => 1,
1149         type => 'string',
1150         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1151                      . " Deprecated, use 'order=' instead.",
1152         pattern => '[acdn]{1,4}',
1153         format_description => "[acdn]{1,4}",
1154
1155         # note: this is also the fallback if boot: is not given at all
1156         default => 'cdn',
1157     },
1158     order => {
1159         optional => 1,
1160         type => 'string',
1161         format => 'pve-qm-bootdev-list',
1162         format_description => "device[;device...]",
1163         description => <<EODESC,
1164 The guest will attempt to boot from devices in the order they appear here.
1165
1166 Disks, optical drives and passed-through storage USB devices will be directly
1167 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1168 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1169
1170 Note that only devices in this list will be marked as bootable and thus loaded
1171 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1172 (e.g. software-raid), you need to specify all of them here.
1173
1174 Overrides the deprecated 'legacy=[acdn]*' value when given.
1175 EODESC
1176     },
1177 };
1178 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1179
1180 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1181 sub verify_bootdev {
1182     my ($dev, $noerr) = @_;
1183
1184     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1185     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1186
1187     my $check = sub {
1188         my ($base) = @_;
1189         return 0 if $dev !~ m/^$base\d+$/;
1190         return 0 if !$confdesc->{$dev};
1191         return 1;
1192     };
1193
1194     return $dev if $check->("net");
1195     return $dev if $check->("usb");
1196     return $dev if $check->("hostpci");
1197
1198     return if $noerr;
1199     die "invalid boot device '$dev'\n";
1200 }
1201
1202 sub print_bootorder {
1203     my ($devs) = @_;
1204     return "" if !@$devs;
1205     my $data = { order => join(';', @$devs) };
1206     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1207 }
1208
1209 my $kvm_api_version = 0;
1210
1211 sub kvm_version {
1212     return $kvm_api_version if $kvm_api_version;
1213
1214     open my $fh, '<', '/dev/kvm' or return;
1215
1216     # 0xae00 => KVM_GET_API_VERSION
1217     $kvm_api_version = ioctl($fh, 0xae00, 0);
1218     close($fh);
1219
1220     return $kvm_api_version;
1221 }
1222
1223 my $kvm_user_version = {};
1224 my $kvm_mtime = {};
1225
1226 sub kvm_user_version {
1227     my ($binary) = @_;
1228
1229     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1230     my $st = stat($binary);
1231
1232     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1233     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1234         $cachedmtime == $st->mtime;
1235
1236     $kvm_user_version->{$binary} = 'unknown';
1237     $kvm_mtime->{$binary} = $st->mtime;
1238
1239     my $code = sub {
1240         my $line = shift;
1241         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1242             $kvm_user_version->{$binary} = $2;
1243         }
1244     };
1245
1246     eval { run_command([$binary, '--version'], outfunc => $code); };
1247     warn $@ if $@;
1248
1249     return $kvm_user_version->{$binary};
1250
1251 }
1252 my sub extract_version {
1253     my ($machine_type, $version) = @_;
1254     $version = kvm_user_version() if !defined($version);
1255     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1256 }
1257
1258 sub kernel_has_vhost_net {
1259     return -c '/dev/vhost-net';
1260 }
1261
1262 sub option_exists {
1263     my $key = shift;
1264     return defined($confdesc->{$key});
1265 }
1266
1267 my $cdrom_path;
1268 sub get_cdrom_path {
1269
1270     return  $cdrom_path if $cdrom_path;
1271
1272     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1273     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1274     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1275 }
1276
1277 sub get_iso_path {
1278     my ($storecfg, $vmid, $cdrom) = @_;
1279
1280     if ($cdrom eq 'cdrom') {
1281         return get_cdrom_path();
1282     } elsif ($cdrom eq 'none') {
1283         return '';
1284     } elsif ($cdrom =~ m|^/|) {
1285         return $cdrom;
1286     } else {
1287         return PVE::Storage::path($storecfg, $cdrom);
1288     }
1289 }
1290
1291 # try to convert old style file names to volume IDs
1292 sub filename_to_volume_id {
1293     my ($vmid, $file, $media) = @_;
1294
1295      if (!($file eq 'none' || $file eq 'cdrom' ||
1296           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1297
1298         return if $file =~ m|/|;
1299
1300         if ($media && $media eq 'cdrom') {
1301             $file = "local:iso/$file";
1302         } else {
1303             $file = "local:$vmid/$file";
1304         }
1305     }
1306
1307     return $file;
1308 }
1309
1310 sub verify_media_type {
1311     my ($opt, $vtype, $media) = @_;
1312
1313     return if !$media;
1314
1315     my $etype;
1316     if ($media eq 'disk') {
1317         $etype = 'images';
1318     } elsif ($media eq 'cdrom') {
1319         $etype = 'iso';
1320     } else {
1321         die "internal error";
1322     }
1323
1324     return if ($vtype eq $etype);
1325
1326     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1327 }
1328
1329 sub cleanup_drive_path {
1330     my ($opt, $storecfg, $drive) = @_;
1331
1332     # try to convert filesystem paths to volume IDs
1333
1334     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1335         ($drive->{file} !~ m|^/dev/.+|) &&
1336         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1337         ($drive->{file} !~ m/^\d+$/)) {
1338         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1339         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1340             if !$vtype;
1341         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1342         verify_media_type($opt, $vtype, $drive->{media});
1343         $drive->{file} = $volid;
1344     }
1345
1346     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1347 }
1348
1349 sub parse_hotplug_features {
1350     my ($data) = @_;
1351
1352     my $res = {};
1353
1354     return $res if $data eq '0';
1355
1356     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1357
1358     foreach my $feature (PVE::Tools::split_list($data)) {
1359         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1360             $res->{$1} = 1;
1361         } else {
1362             die "invalid hotplug feature '$feature'\n";
1363         }
1364     }
1365     return $res;
1366 }
1367
1368 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1369 sub pve_verify_hotplug_features {
1370     my ($value, $noerr) = @_;
1371
1372     return $value if parse_hotplug_features($value);
1373
1374     return if $noerr;
1375
1376     die "unable to parse hotplug option\n";
1377 }
1378
1379 sub scsi_inquiry {
1380     my($fh, $noerr) = @_;
1381
1382     my $SG_IO = 0x2285;
1383     my $SG_GET_VERSION_NUM = 0x2282;
1384
1385     my $versionbuf = "\x00" x 8;
1386     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1387     if (!$ret) {
1388         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1389         return;
1390     }
1391     my $version = unpack("I", $versionbuf);
1392     if ($version < 30000) {
1393         die "scsi generic interface too old\n"  if !$noerr;
1394         return;
1395     }
1396
1397     my $buf = "\x00" x 36;
1398     my $sensebuf = "\x00" x 8;
1399     my $cmd = pack("C x3 C x1", 0x12, 36);
1400
1401     # see /usr/include/scsi/sg.h
1402     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1403
1404     my $packet = pack(
1405         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1406     );
1407
1408     $ret = ioctl($fh, $SG_IO, $packet);
1409     if (!$ret) {
1410         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1411         return;
1412     }
1413
1414     my @res = unpack($sg_io_hdr_t, $packet);
1415     if ($res[17] || $res[18]) {
1416         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1417         return;
1418     }
1419
1420     my $res = {};
1421     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1422
1423     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1424     $res->{type} &= 0x1F;
1425
1426     return $res;
1427 }
1428
1429 sub path_is_scsi {
1430     my ($path) = @_;
1431
1432     my $fh = IO::File->new("+<$path") || return;
1433     my $res = scsi_inquiry($fh, 1);
1434     close($fh);
1435
1436     return $res;
1437 }
1438
1439 sub print_tabletdevice_full {
1440     my ($conf, $arch) = @_;
1441
1442     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1443
1444     # we use uhci for old VMs because tablet driver was buggy in older qemu
1445     my $usbbus;
1446     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1447         $usbbus = 'ehci';
1448     } else {
1449         $usbbus = 'uhci';
1450     }
1451
1452     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1453 }
1454
1455 sub print_keyboarddevice_full {
1456     my ($conf, $arch) = @_;
1457
1458     return if $arch ne 'aarch64';
1459
1460     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1461 }
1462
1463 my sub get_drive_id {
1464     my ($drive) = @_;
1465     return "$drive->{interface}$drive->{index}";
1466 }
1467
1468 sub print_drivedevice_full {
1469     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1470
1471     my $device = '';
1472     my $maxdev = 0;
1473
1474     my $drive_id = get_drive_id($drive);
1475     if ($drive->{interface} eq 'virtio') {
1476         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1477         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1478         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1479     } elsif ($drive->{interface} eq 'scsi') {
1480
1481         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1482         my $unit = $drive->{index} % $maxdev;
1483         my $devicetype = 'hd';
1484         my $path = '';
1485         if (drive_is_cdrom($drive)) {
1486             $devicetype = 'cd';
1487         } else {
1488             if ($drive->{file} =~ m|^/|) {
1489                 $path = $drive->{file};
1490                 if (my $info = path_is_scsi($path)) {
1491                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1492                         $devicetype = 'block';
1493                     } elsif ($info->{type} == 1) { # tape
1494                         $devicetype = 'generic';
1495                     }
1496                 }
1497             } else {
1498                  $path = PVE::Storage::path($storecfg, $drive->{file});
1499             }
1500
1501             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1502             my $version = extract_version($machine_type, kvm_user_version());
1503             if ($path =~ m/^iscsi\:\/\// &&
1504                !min_version($version, 4, 1)) {
1505                 $devicetype = 'generic';
1506             }
1507         }
1508
1509         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1510             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1511         } else {
1512             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1513                 .",lun=$drive->{index}";
1514         }
1515         $device .= ",drive=drive-$drive_id,id=$drive_id";
1516
1517         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1518             $device .= ",rotation_rate=1";
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521
1522     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1523         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1524         my $controller = int($drive->{index} / $maxdev);
1525         my $unit = $drive->{index} % $maxdev;
1526         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1527
1528         $device = "ide-$devicetype";
1529         if ($drive->{interface} eq 'ide') {
1530             $device .= ",bus=ide.$controller,unit=$unit";
1531         } else {
1532             $device .= ",bus=ahci$controller.$unit";
1533         }
1534         $device .= ",drive=drive-$drive_id,id=$drive_id";
1535
1536         if ($devicetype eq 'hd') {
1537             if (my $model = $drive->{model}) {
1538                 $model = URI::Escape::uri_unescape($model);
1539                 $device .= ",model=$model";
1540             }
1541             if ($drive->{ssd}) {
1542                 $device .= ",rotation_rate=1";
1543             }
1544         }
1545         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1546     } elsif ($drive->{interface} eq 'usb') {
1547         die "implement me";
1548         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1549     } else {
1550         die "unsupported interface type";
1551     }
1552
1553     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1554
1555     if (my $serial = $drive->{serial}) {
1556         $serial = URI::Escape::uri_unescape($serial);
1557         $device .= ",serial=$serial";
1558     }
1559
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_commandline_full {
1579     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format = $drive->{format};
1584     my $drive_id = get_drive_id($drive);
1585
1586     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1587     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1588
1589     if (drive_is_cdrom($drive)) {
1590         $path = get_iso_path($storecfg, $vmid, $volid);
1591         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1592     } else {
1593         if ($storeid) {
1594             $path = PVE::Storage::path($storecfg, $volid);
1595             $format //= qemu_img_format($scfg, $volname);
1596         } else {
1597             $path = $volid;
1598             $format //= "raw";
1599         }
1600    }
1601
1602    my $is_rbd = $path =~ m/^rbd:/;
1603
1604     my $opts = '';
1605     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1606     foreach my $o (@qemu_drive_options) {
1607         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1608     }
1609
1610     # snapshot only accepts on|off
1611     if (defined($drive->{snapshot})) {
1612         my $v = $drive->{snapshot} ? 'on' : 'off';
1613         $opts .= ",snapshot=$v";
1614     }
1615
1616     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1617         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1618     }
1619
1620     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1621         my ($dir, $qmpname) = @$type;
1622         if (my $v = $drive->{"mbps$dir"}) {
1623             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1624         }
1625         if (my $v = $drive->{"mbps${dir}_max"}) {
1626             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1627         }
1628         if (my $v = $drive->{"bps${dir}_max_length"}) {
1629             $opts .= ",throttling.bps$qmpname-max-length=$v";
1630         }
1631         if (my $v = $drive->{"iops${dir}"}) {
1632             $opts .= ",throttling.iops$qmpname=$v";
1633         }
1634         if (my $v = $drive->{"iops${dir}_max"}) {
1635             $opts .= ",throttling.iops$qmpname-max=$v";
1636         }
1637         if (my $v = $drive->{"iops${dir}_max_length"}) {
1638             $opts .= ",throttling.iops$qmpname-max-length=$v";
1639         }
1640     }
1641
1642     if ($pbs_name) {
1643         $format = "rbd" if $is_rbd;
1644         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1645             if !$format;
1646         $opts .= ",format=alloc-track,file.driver=$format";
1647     } elsif ($format) {
1648         $opts .= ",format=$format";
1649     }
1650
1651     my $cache_direct = 0;
1652
1653     if (my $cache = $drive->{cache}) {
1654         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1655     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1656         $opts .= ",cache=none";
1657         $cache_direct = 1;
1658     }
1659
1660     # io_uring with cache mode writeback or writethrough on krbd will hang...
1661     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1662
1663     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1664     # sometimes, just plain disable...
1665     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1666
1667     if (!$drive->{aio}) {
1668         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1669             # io_uring supports all cache modes
1670             $opts .= ",aio=io_uring";
1671         } else {
1672             # aio native works only with O_DIRECT
1673             if($cache_direct) {
1674                 $opts .= ",aio=native";
1675             } else {
1676                 $opts .= ",aio=threads";
1677             }
1678         }
1679     }
1680
1681     if (!drive_is_cdrom($drive)) {
1682         my $detectzeroes;
1683         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1684             $detectzeroes = 'off';
1685         } elsif ($drive->{discard}) {
1686             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1687         } else {
1688             # This used to be our default with discard not being specified:
1689             $detectzeroes = 'on';
1690         }
1691
1692         # note: 'detect-zeroes' works per blockdev and we want it to persist
1693         # after the alloc-track is removed, so put it on 'file' directly
1694         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1695         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1696     }
1697
1698     if ($pbs_name) {
1699         $opts .= ",backing=$pbs_name";
1700         $opts .= ",auto-remove=on";
1701     }
1702
1703     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1704     my $file_param = "file";
1705     if ($pbs_name) {
1706         # non-rbd drivers require the underlying file to be a seperate block
1707         # node, so add a second .file indirection
1708         $file_param .= ".file" if !$is_rbd;
1709         $file_param .= ".filename";
1710     }
1711     my $pathinfo = $path ? "$file_param=$path," : '';
1712
1713     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1714 }
1715
1716 sub print_pbs_blockdev {
1717     my ($pbs_conf, $pbs_name) = @_;
1718     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1719     $blockdev .= ",repository=$pbs_conf->{repository}";
1720     $blockdev .= ",namespace=$pbs_conf->{namespace}" if $pbs_conf->{namespace};
1721     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1722     $blockdev .= ",archive=$pbs_conf->{archive}";
1723     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1724     return $blockdev;
1725 }
1726
1727 sub print_netdevice_full {
1728     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1729
1730     my $device = $net->{model};
1731     if ($net->{model} eq 'virtio') {
1732          $device = 'virtio-net-pci';
1733      };
1734
1735     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1736     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1737     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1738         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1739         # and out of each queue plus one config interrupt and control vector queue
1740         my $vectors = $net->{queues} * 2 + 2;
1741         $tmpstr .= ",vectors=$vectors,mq=on";
1742     }
1743     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1744
1745     if (my $mtu = $net->{mtu}) {
1746         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1747             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1748             if ($mtu == 1) {
1749                  $mtu = $bridge_mtu;
1750             } elsif ($mtu < 576) {
1751                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1752             } elsif ($mtu > $bridge_mtu) {
1753                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1754             }
1755             $tmpstr .= ",host_mtu=$mtu";
1756         } else {
1757             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1758         }
1759     }
1760
1761     if ($use_old_bios_files) {
1762         my $romfile;
1763         if ($device eq 'virtio-net-pci') {
1764             $romfile = 'pxe-virtio.rom';
1765         } elsif ($device eq 'e1000') {
1766             $romfile = 'pxe-e1000.rom';
1767         } elsif ($device eq 'e1000e') {
1768             $romfile = 'pxe-e1000e.rom';
1769         } elsif ($device eq 'ne2k') {
1770             $romfile = 'pxe-ne2k_pci.rom';
1771         } elsif ($device eq 'pcnet') {
1772             $romfile = 'pxe-pcnet.rom';
1773         } elsif ($device eq 'rtl8139') {
1774             $romfile = 'pxe-rtl8139.rom';
1775         }
1776         $tmpstr .= ",romfile=$romfile" if $romfile;
1777     }
1778
1779     return $tmpstr;
1780 }
1781
1782 sub print_netdev_full {
1783     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1784
1785     my $i = '';
1786     if ($netid =~ m/^net(\d+)$/) {
1787         $i = int($1);
1788     }
1789
1790     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1791
1792     my $ifname = "tap${vmid}i$i";
1793
1794     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1795     die "interface name '$ifname' is too long (max 15 character)\n"
1796         if length($ifname) >= 16;
1797
1798     my $vhostparam = '';
1799     if (is_native($arch)) {
1800         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1801     }
1802
1803     my $vmname = $conf->{name} || "vm$vmid";
1804
1805     my $netdev = "";
1806     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1807
1808     if ($net->{bridge}) {
1809         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1810             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1811     } else {
1812         $netdev = "type=user,id=$netid,hostname=$vmname";
1813     }
1814
1815     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1816
1817     return $netdev;
1818 }
1819
1820 my $vga_map = {
1821     'cirrus' => 'cirrus-vga',
1822     'std' => 'VGA',
1823     'vmware' => 'vmware-svga',
1824     'virtio' => 'virtio-vga',
1825     'virtio-gl' => 'virtio-vga-gl',
1826 };
1827
1828 sub print_vga_device {
1829     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1830
1831     my $type = $vga_map->{$vga->{type}};
1832     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1833         $type = 'virtio-gpu';
1834     }
1835     my $vgamem_mb = $vga->{memory};
1836
1837     my $max_outputs = '';
1838     if ($qxlnum) {
1839         $type = $id ? 'qxl' : 'qxl-vga';
1840
1841         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1842             # set max outputs so linux can have up to 4 qxl displays with one device
1843             if (min_version($machine_version, 4, 1)) {
1844                 $max_outputs = ",max_outputs=4";
1845             }
1846         }
1847     }
1848
1849     die "no devicetype for $vga->{type}\n" if !$type;
1850
1851     my $memory = "";
1852     if ($vgamem_mb) {
1853         if ($vga->{type} =~ /^virtio/) {
1854             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1855             $memory = ",max_hostmem=$bytes";
1856         } elsif ($qxlnum) {
1857             # from https://www.spice-space.org/multiple-monitors.html
1858             $memory = ",vgamem_mb=$vga->{memory}";
1859             my $ram = $vgamem_mb * 4;
1860             my $vram = $vgamem_mb * 2;
1861             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1862         } else {
1863             $memory = ",vgamem_mb=$vga->{memory}";
1864         }
1865     } elsif ($qxlnum && $id) {
1866         $memory = ",ram_size=67108864,vram_size=33554432";
1867     }
1868
1869     my $edidoff = "";
1870     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1871         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1872     }
1873
1874     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1875     my $vgaid = "vga" . ($id // '');
1876     my $pciaddr;
1877     if ($q35 && $vgaid eq 'vga') {
1878         # the first display uses pcie.0 bus on q35 machines
1879         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1880     } else {
1881         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1882     }
1883
1884     if ($vga->{type} eq 'virtio-gl') {
1885         my $base = '/usr/lib/x86_64-linux-gnu/lib';
1886         die "missing libraries for '$vga->{type}' detected! Please install 'libgl1' and 'libegl1'\n"
1887             if !-e "${base}EGL.so.1" || !-e "${base}GL.so.1";
1888
1889         die "no DRM render node detected (/dev/dri/renderD*), no GPU? - needed for '$vga->{type}' display\n"
1890             if !PVE::Tools::dir_glob_regex('/dev/dri/', "renderD.*");
1891     }
1892
1893     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1894 }
1895
1896 sub parse_number_sets {
1897     my ($set) = @_;
1898     my $res = [];
1899     foreach my $part (split(/;/, $set)) {
1900         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1901             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1902             push @$res, [ $1, $2 ];
1903         } else {
1904             die "invalid range: $part\n";
1905         }
1906     }
1907     return $res;
1908 }
1909
1910 sub parse_numa {
1911     my ($data) = @_;
1912
1913     my $res = parse_property_string($numa_fmt, $data);
1914     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1915     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1916     return $res;
1917 }
1918
1919 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1920 sub parse_net {
1921     my ($data) = @_;
1922
1923     my $res = eval { parse_property_string($net_fmt, $data) };
1924     if ($@) {
1925         warn $@;
1926         return;
1927     }
1928     if (!defined($res->{macaddr})) {
1929         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1930         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1931     }
1932     return $res;
1933 }
1934
1935 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1936 sub parse_ipconfig {
1937     my ($data) = @_;
1938
1939     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1940     if ($@) {
1941         warn $@;
1942         return;
1943     }
1944
1945     if ($res->{gw} && !$res->{ip}) {
1946         warn 'gateway specified without specifying an IP address';
1947         return;
1948     }
1949     if ($res->{gw6} && !$res->{ip6}) {
1950         warn 'IPv6 gateway specified without specifying an IPv6 address';
1951         return;
1952     }
1953     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1954         warn 'gateway specified together with DHCP';
1955         return;
1956     }
1957     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1958         # gw6 + auto/dhcp
1959         warn "IPv6 gateway specified together with $res->{ip6} address";
1960         return;
1961     }
1962
1963     if (!$res->{ip} && !$res->{ip6}) {
1964         return { ip => 'dhcp', ip6 => 'dhcp' };
1965     }
1966
1967     return $res;
1968 }
1969
1970 sub print_net {
1971     my $net = shift;
1972
1973     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1974 }
1975
1976 sub add_random_macs {
1977     my ($settings) = @_;
1978
1979     foreach my $opt (keys %$settings) {
1980         next if $opt !~ m/^net(\d+)$/;
1981         my $net = parse_net($settings->{$opt});
1982         next if !$net;
1983         $settings->{$opt} = print_net($net);
1984     }
1985 }
1986
1987 sub vm_is_volid_owner {
1988     my ($storecfg, $vmid, $volid) = @_;
1989
1990     if ($volid !~  m|^/|) {
1991         my ($path, $owner);
1992         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1993         if ($owner && ($owner == $vmid)) {
1994             return 1;
1995         }
1996     }
1997
1998     return;
1999 }
2000
2001 sub vmconfig_register_unused_drive {
2002     my ($storecfg, $vmid, $conf, $drive) = @_;
2003
2004     if (drive_is_cloudinit($drive)) {
2005         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2006         warn $@ if $@;
2007     } elsif (!drive_is_cdrom($drive)) {
2008         my $volid = $drive->{file};
2009         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2010             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2011         }
2012     }
2013 }
2014
2015 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2016 my $smbios1_fmt = {
2017     uuid => {
2018         type => 'string',
2019         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2020         format_description => 'UUID',
2021         description => "Set SMBIOS1 UUID.",
2022         optional => 1,
2023     },
2024     version => {
2025         type => 'string',
2026         pattern => '[A-Za-z0-9+\/]+={0,2}',
2027         format_description => 'Base64 encoded string',
2028         description => "Set SMBIOS1 version.",
2029         optional => 1,
2030     },
2031     serial => {
2032         type => 'string',
2033         pattern => '[A-Za-z0-9+\/]+={0,2}',
2034         format_description => 'Base64 encoded string',
2035         description => "Set SMBIOS1 serial number.",
2036         optional => 1,
2037     },
2038     manufacturer => {
2039         type => 'string',
2040         pattern => '[A-Za-z0-9+\/]+={0,2}',
2041         format_description => 'Base64 encoded string',
2042         description => "Set SMBIOS1 manufacturer.",
2043         optional => 1,
2044     },
2045     product => {
2046         type => 'string',
2047         pattern => '[A-Za-z0-9+\/]+={0,2}',
2048         format_description => 'Base64 encoded string',
2049         description => "Set SMBIOS1 product ID.",
2050         optional => 1,
2051     },
2052     sku => {
2053         type => 'string',
2054         pattern => '[A-Za-z0-9+\/]+={0,2}',
2055         format_description => 'Base64 encoded string',
2056         description => "Set SMBIOS1 SKU string.",
2057         optional => 1,
2058     },
2059     family => {
2060         type => 'string',
2061         pattern => '[A-Za-z0-9+\/]+={0,2}',
2062         format_description => 'Base64 encoded string',
2063         description => "Set SMBIOS1 family string.",
2064         optional => 1,
2065     },
2066     base64 => {
2067         type => 'boolean',
2068         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2069         optional => 1,
2070     },
2071 };
2072
2073 sub parse_smbios1 {
2074     my ($data) = @_;
2075
2076     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2077     warn $@ if $@;
2078     return $res;
2079 }
2080
2081 sub print_smbios1 {
2082     my ($smbios1) = @_;
2083     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2084 }
2085
2086 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2087
2088 sub parse_watchdog {
2089     my ($value) = @_;
2090
2091     return if !$value;
2092
2093     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2094     warn $@ if $@;
2095     return $res;
2096 }
2097
2098 sub parse_guest_agent {
2099     my ($conf) = @_;
2100
2101     return {} if !defined($conf->{agent});
2102
2103     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2104     warn $@ if $@;
2105
2106     # if the agent is disabled ignore the other potentially set properties
2107     return {} if !$res->{enabled};
2108     return $res;
2109 }
2110
2111 sub get_qga_key {
2112     my ($conf, $key) = @_;
2113     return undef if !defined($conf->{agent});
2114
2115     my $agent = parse_guest_agent($conf);
2116     return $agent->{$key};
2117 }
2118
2119 sub parse_vga {
2120     my ($value) = @_;
2121
2122     return {} if !$value;
2123     my $res = eval { parse_property_string($vga_fmt, $value) };
2124     warn $@ if $@;
2125     return $res;
2126 }
2127
2128 sub parse_rng {
2129     my ($value) = @_;
2130
2131     return if !$value;
2132
2133     my $res = eval { parse_property_string($rng_fmt, $value) };
2134     warn $@ if $@;
2135     return $res;
2136 }
2137
2138 sub parse_meta_info {
2139     my ($value) = @_;
2140
2141     return if !$value;
2142
2143     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2144     warn $@ if $@;
2145     return $res;
2146 }
2147
2148 sub new_meta_info_string {
2149     my () = @_; # for now do not allow to override any value
2150
2151     return PVE::JSONSchema::print_property_string(
2152         {
2153             'creation-qemu' => kvm_user_version(),
2154             ctime => "". int(time()),
2155         },
2156         $meta_info_fmt
2157     );
2158 }
2159
2160 sub qemu_created_version_fixups {
2161     my ($conf, $forcemachine, $kvmver) = @_;
2162
2163     my $meta = parse_meta_info($conf->{meta}) // {};
2164     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2165
2166     # check if we need to apply some handling for VMs that always use the latest machine version but
2167     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2168     # would be required (we do not want to pin machine version for non-windows OS type)
2169     if (
2170         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2171         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2172         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2173         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2174     ) {
2175         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2176         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2177             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2178             # and thus with the predictable interface naming of systemd
2179             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2180         }
2181     }
2182     return;
2183 }
2184
2185 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2186 sub verify_usb_device {
2187     my ($value, $noerr) = @_;
2188
2189     return $value if parse_usb_device($value);
2190
2191     return if $noerr;
2192
2193     die "unable to parse usb device\n";
2194 }
2195
2196 # add JSON properties for create and set function
2197 sub json_config_properties {
2198     my ($prop, $with_disk_alloc) = @_;
2199
2200     my $skip_json_config_opts = {
2201         parent => 1,
2202         snaptime => 1,
2203         vmstate => 1,
2204         runningmachine => 1,
2205         runningcpu => 1,
2206         meta => 1,
2207     };
2208
2209     foreach my $opt (keys %$confdesc) {
2210         next if $skip_json_config_opts->{$opt};
2211
2212         if ($with_disk_alloc && is_valid_drivename($opt)) {
2213             $prop->{$opt} = $PVE::QemuServer::Drive::drivedesc_hash_with_alloc->{$opt};
2214         } else {
2215             $prop->{$opt} = $confdesc->{$opt};
2216         }
2217     }
2218
2219     return $prop;
2220 }
2221
2222 # Properties that we can read from an OVF file
2223 sub json_ovf_properties {
2224     my $prop = {};
2225
2226     for my $device (PVE::QemuServer::Drive::valid_drive_names()) {
2227         $prop->{$device} = {
2228             type => 'string',
2229             format => 'pve-volume-id-or-absolute-path',
2230             description => "Disk image that gets imported to $device",
2231             optional => 1,
2232         };
2233     }
2234
2235     $prop->{cores} = {
2236         type => 'integer',
2237         description => "The number of CPU cores.",
2238         optional => 1,
2239     };
2240     $prop->{memory} = {
2241         type => 'integer',
2242         description => "Amount of RAM for the VM in MB.",
2243         optional => 1,
2244     };
2245     $prop->{name} = {
2246         type => 'string',
2247         description => "Name of the VM.",
2248         optional => 1,
2249     };
2250
2251     return $prop;
2252 }
2253
2254 # return copy of $confdesc_cloudinit to generate documentation
2255 sub cloudinit_config_properties {
2256
2257     return dclone($confdesc_cloudinit);
2258 }
2259
2260 sub check_type {
2261     my ($key, $value) = @_;
2262
2263     die "unknown setting '$key'\n" if !$confdesc->{$key};
2264
2265     my $type = $confdesc->{$key}->{type};
2266
2267     if (!defined($value)) {
2268         die "got undefined value\n";
2269     }
2270
2271     if ($value =~ m/[\n\r]/) {
2272         die "property contains a line feed\n";
2273     }
2274
2275     if ($type eq 'boolean') {
2276         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2277         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2278         die "type check ('boolean') failed - got '$value'\n";
2279     } elsif ($type eq 'integer') {
2280         return int($1) if $value =~ m/^(\d+)$/;
2281         die "type check ('integer') failed - got '$value'\n";
2282     } elsif ($type eq 'number') {
2283         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2284         die "type check ('number') failed - got '$value'\n";
2285     } elsif ($type eq 'string') {
2286         if (my $fmt = $confdesc->{$key}->{format}) {
2287             PVE::JSONSchema::check_format($fmt, $value);
2288             return $value;
2289         }
2290         $value =~ s/^\"(.*)\"$/$1/;
2291         return $value;
2292     } else {
2293         die "internal error"
2294     }
2295 }
2296
2297 sub destroy_vm {
2298     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2299
2300     my $conf = PVE::QemuConfig->load_config($vmid);
2301
2302     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2303
2304     if ($conf->{template}) {
2305         # check if any base image is still used by a linked clone
2306         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2307                 my ($ds, $drive) = @_;
2308                 return if drive_is_cdrom($drive);
2309
2310                 my $volid = $drive->{file};
2311                 return if !$volid || $volid =~ m|^/|;
2312
2313                 die "base volume '$volid' is still in use by linked cloned\n"
2314                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2315
2316         });
2317     }
2318
2319     my $volids = {};
2320     my $remove_owned_drive = sub {
2321         my ($ds, $drive) = @_;
2322         return if drive_is_cdrom($drive, 1);
2323
2324         my $volid = $drive->{file};
2325         return if !$volid || $volid =~ m|^/|;
2326         return if $volids->{$volid};
2327
2328         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2329         return if !$path || !$owner || ($owner != $vmid);
2330
2331         $volids->{$volid} = 1;
2332         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2333         warn "Could not remove disk '$volid', check manually: $@" if $@;
2334     };
2335
2336     # only remove disks owned by this VM (referenced in the config)
2337     my $include_opts = {
2338         include_unused => 1,
2339         extra_keys => ['vmstate'],
2340     };
2341     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2342
2343     for my $snap (values %{$conf->{snapshots}}) {
2344         next if !defined($snap->{vmstate});
2345         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2346         next if !defined($drive);
2347         $remove_owned_drive->('vmstate', $drive);
2348     }
2349
2350     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2351
2352     if ($purge_unreferenced) { # also remove unreferenced disk
2353         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2354         PVE::Storage::foreach_volid($vmdisks, sub {
2355             my ($volid, $sid, $volname, $d) = @_;
2356             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2357             warn $@ if $@;
2358         });
2359     }
2360
2361     if (defined $replacement_conf) {
2362         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2363     } else {
2364         PVE::QemuConfig->destroy_config($vmid);
2365     }
2366 }
2367
2368 sub parse_vm_config {
2369     my ($filename, $raw, $strict) = @_;
2370
2371     return if !defined($raw);
2372
2373     my $res = {
2374         digest => Digest::SHA::sha1_hex($raw),
2375         snapshots => {},
2376         pending => {},
2377     };
2378
2379     my $handle_error = sub {
2380         my ($msg) = @_;
2381
2382         if ($strict) {
2383             die $msg;
2384         } else {
2385             warn $msg;
2386         }
2387     };
2388
2389     $filename =~ m|/qemu-server/(\d+)\.conf$|
2390         || die "got strange filename '$filename'";
2391
2392     my $vmid = $1;
2393
2394     my $conf = $res;
2395     my $descr;
2396     my $section = '';
2397
2398     my @lines = split(/\n/, $raw);
2399     foreach my $line (@lines) {
2400         next if $line =~ m/^\s*$/;
2401
2402         if ($line =~ m/^\[PENDING\]\s*$/i) {
2403             $section = 'pending';
2404             if (defined($descr)) {
2405                 $descr =~ s/\s+$//;
2406                 $conf->{description} = $descr;
2407             }
2408             $descr = undef;
2409             $conf = $res->{$section} = {};
2410             next;
2411
2412         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2413             $section = $1;
2414             if (defined($descr)) {
2415                 $descr =~ s/\s+$//;
2416                 $conf->{description} = $descr;
2417             }
2418             $descr = undef;
2419             $conf = $res->{snapshots}->{$section} = {};
2420             next;
2421         }
2422
2423         if ($line =~ m/^\#(.*)$/) {
2424             $descr = '' if !defined($descr);
2425             $descr .= PVE::Tools::decode_text($1) . "\n";
2426             next;
2427         }
2428
2429         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2430             $descr = '' if !defined($descr);
2431             $descr .= PVE::Tools::decode_text($2);
2432         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2433             $conf->{snapstate} = $1;
2434         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2435             my $key = $1;
2436             my $value = $2;
2437             $conf->{$key} = $value;
2438         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2439             my $value = $1;
2440             if ($section eq 'pending') {
2441                 $conf->{delete} = $value; # we parse this later
2442             } else {
2443                 $handle_error->("vm $vmid - property 'delete' is only allowed in [PENDING]\n");
2444             }
2445         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2446             my $key = $1;
2447             my $value = $2;
2448             eval { $value = check_type($key, $value); };
2449             if ($@) {
2450                 $handle_error->("vm $vmid - unable to parse value of '$key' - $@");
2451             } else {
2452                 $key = 'ide2' if $key eq 'cdrom';
2453                 my $fmt = $confdesc->{$key}->{format};
2454                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2455                     my $v = parse_drive($key, $value);
2456                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2457                         $v->{file} = $volid;
2458                         $value = print_drive($v);
2459                     } else {
2460                         $handle_error->("vm $vmid - unable to parse value of '$key'\n");
2461                         next;
2462                     }
2463                 }
2464
2465                 $conf->{$key} = $value;
2466             }
2467         } else {
2468             $handle_error->("vm $vmid - unable to parse config: $line\n");
2469         }
2470     }
2471
2472     if (defined($descr)) {
2473         $descr =~ s/\s+$//;
2474         $conf->{description} = $descr;
2475     }
2476     delete $res->{snapstate}; # just to be sure
2477
2478     return $res;
2479 }
2480
2481 sub write_vm_config {
2482     my ($filename, $conf) = @_;
2483
2484     delete $conf->{snapstate}; # just to be sure
2485
2486     if ($conf->{cdrom}) {
2487         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2488         $conf->{ide2} = $conf->{cdrom};
2489         delete $conf->{cdrom};
2490     }
2491
2492     # we do not use 'smp' any longer
2493     if ($conf->{sockets}) {
2494         delete $conf->{smp};
2495     } elsif ($conf->{smp}) {
2496         $conf->{sockets} = $conf->{smp};
2497         delete $conf->{cores};
2498         delete $conf->{smp};
2499     }
2500
2501     my $used_volids = {};
2502
2503     my $cleanup_config = sub {
2504         my ($cref, $pending, $snapname) = @_;
2505
2506         foreach my $key (keys %$cref) {
2507             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2508                 $key eq 'snapstate' || $key eq 'pending';
2509             my $value = $cref->{$key};
2510             if ($key eq 'delete') {
2511                 die "propertry 'delete' is only allowed in [PENDING]\n"
2512                     if !$pending;
2513                 # fixme: check syntax?
2514                 next;
2515             }
2516             eval { $value = check_type($key, $value); };
2517             die "unable to parse value of '$key' - $@" if $@;
2518
2519             $cref->{$key} = $value;
2520
2521             if (!$snapname && is_valid_drivename($key)) {
2522                 my $drive = parse_drive($key, $value);
2523                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2524             }
2525         }
2526     };
2527
2528     &$cleanup_config($conf);
2529
2530     &$cleanup_config($conf->{pending}, 1);
2531
2532     foreach my $snapname (keys %{$conf->{snapshots}}) {
2533         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2534         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2535     }
2536
2537     # remove 'unusedX' settings if we re-add a volume
2538     foreach my $key (keys %$conf) {
2539         my $value = $conf->{$key};
2540         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2541             delete $conf->{$key};
2542         }
2543     }
2544
2545     my $generate_raw_config = sub {
2546         my ($conf, $pending) = @_;
2547
2548         my $raw = '';
2549
2550         # add description as comment to top of file
2551         if (defined(my $descr = $conf->{description})) {
2552             if ($descr) {
2553                 foreach my $cl (split(/\n/, $descr)) {
2554                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2555                 }
2556             } else {
2557                 $raw .= "#\n" if $pending;
2558             }
2559         }
2560
2561         foreach my $key (sort keys %$conf) {
2562             next if $key =~ /^(digest|description|pending|snapshots)$/;
2563             $raw .= "$key: $conf->{$key}\n";
2564         }
2565         return $raw;
2566     };
2567
2568     my $raw = &$generate_raw_config($conf);
2569
2570     if (scalar(keys %{$conf->{pending}})){
2571         $raw .= "\n[PENDING]\n";
2572         $raw .= &$generate_raw_config($conf->{pending}, 1);
2573     }
2574
2575     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2576         $raw .= "\n[$snapname]\n";
2577         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2578     }
2579
2580     return $raw;
2581 }
2582
2583 sub load_defaults {
2584
2585     my $res = {};
2586
2587     # we use static defaults from our JSON schema configuration
2588     foreach my $key (keys %$confdesc) {
2589         if (defined(my $default = $confdesc->{$key}->{default})) {
2590             $res->{$key} = $default;
2591         }
2592     }
2593
2594     return $res;
2595 }
2596
2597 sub config_list {
2598     my $vmlist = PVE::Cluster::get_vmlist();
2599     my $res = {};
2600     return $res if !$vmlist || !$vmlist->{ids};
2601     my $ids = $vmlist->{ids};
2602     my $nodename = nodename();
2603
2604     foreach my $vmid (keys %$ids) {
2605         my $d = $ids->{$vmid};
2606         next if !$d->{node} || $d->{node} ne $nodename;
2607         next if !$d->{type} || $d->{type} ne 'qemu';
2608         $res->{$vmid}->{exists} = 1;
2609     }
2610     return $res;
2611 }
2612
2613 # test if VM uses local resources (to prevent migration)
2614 sub check_local_resources {
2615     my ($conf, $noerr) = @_;
2616
2617     my @loc_res = ();
2618
2619     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2620     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2621
2622     push @loc_res, "ivshmem" if $conf->{ivshmem};
2623
2624     foreach my $k (keys %$conf) {
2625         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2626         # sockets are safe: they will recreated be on the target side post-migrate
2627         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2628         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2629     }
2630
2631     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2632
2633     return \@loc_res;
2634 }
2635
2636 # check if used storages are available on all nodes (use by migrate)
2637 sub check_storage_availability {
2638     my ($storecfg, $conf, $node) = @_;
2639
2640     PVE::QemuConfig->foreach_volume($conf, sub {
2641         my ($ds, $drive) = @_;
2642
2643         my $volid = $drive->{file};
2644         return if !$volid;
2645
2646         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2647         return if !$sid;
2648
2649         # check if storage is available on both nodes
2650         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2651         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2652
2653         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2654
2655         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2656             if !$scfg->{content}->{$vtype};
2657    });
2658 }
2659
2660 # list nodes where all VM images are available (used by has_feature API)
2661 sub shared_nodes {
2662     my ($conf, $storecfg) = @_;
2663
2664     my $nodelist = PVE::Cluster::get_nodelist();
2665     my $nodehash = { map { $_ => 1 } @$nodelist };
2666     my $nodename = nodename();
2667
2668     PVE::QemuConfig->foreach_volume($conf, sub {
2669         my ($ds, $drive) = @_;
2670
2671         my $volid = $drive->{file};
2672         return if !$volid;
2673
2674         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2675         if ($storeid) {
2676             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2677             if ($scfg->{disable}) {
2678                 $nodehash = {};
2679             } elsif (my $avail = $scfg->{nodes}) {
2680                 foreach my $node (keys %$nodehash) {
2681                     delete $nodehash->{$node} if !$avail->{$node};
2682                 }
2683             } elsif (!$scfg->{shared}) {
2684                 foreach my $node (keys %$nodehash) {
2685                     delete $nodehash->{$node} if $node ne $nodename
2686                 }
2687             }
2688         }
2689     });
2690
2691     return $nodehash
2692 }
2693
2694 sub check_local_storage_availability {
2695     my ($conf, $storecfg) = @_;
2696
2697     my $nodelist = PVE::Cluster::get_nodelist();
2698     my $nodehash = { map { $_ => {} } @$nodelist };
2699
2700     PVE::QemuConfig->foreach_volume($conf, sub {
2701         my ($ds, $drive) = @_;
2702
2703         my $volid = $drive->{file};
2704         return if !$volid;
2705
2706         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2707         if ($storeid) {
2708             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2709
2710             if ($scfg->{disable}) {
2711                 foreach my $node (keys %$nodehash) {
2712                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2713                 }
2714             } elsif (my $avail = $scfg->{nodes}) {
2715                 foreach my $node (keys %$nodehash) {
2716                     if (!$avail->{$node}) {
2717                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2718                     }
2719                 }
2720             }
2721         }
2722     });
2723
2724     foreach my $node (values %$nodehash) {
2725         if (my $unavail = $node->{unavailable_storages}) {
2726             $node->{unavailable_storages} = [ sort keys %$unavail ];
2727         }
2728     }
2729
2730     return $nodehash
2731 }
2732
2733 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2734 sub check_running {
2735     my ($vmid, $nocheck, $node) = @_;
2736
2737     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2738     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2739 }
2740
2741 sub vzlist {
2742
2743     my $vzlist = config_list();
2744
2745     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2746
2747     while (defined(my $de = $fd->read)) {
2748         next if $de !~ m/^(\d+)\.pid$/;
2749         my $vmid = $1;
2750         next if !defined($vzlist->{$vmid});
2751         if (my $pid = check_running($vmid)) {
2752             $vzlist->{$vmid}->{pid} = $pid;
2753         }
2754     }
2755
2756     return $vzlist;
2757 }
2758
2759 our $vmstatus_return_properties = {
2760     vmid => get_standard_option('pve-vmid'),
2761     status => {
2762         description => "Qemu process status.",
2763         type => 'string',
2764         enum => ['stopped', 'running'],
2765     },
2766     maxmem => {
2767         description => "Maximum memory in bytes.",
2768         type => 'integer',
2769         optional => 1,
2770         renderer => 'bytes',
2771     },
2772     maxdisk => {
2773         description => "Root disk size in bytes.",
2774         type => 'integer',
2775         optional => 1,
2776         renderer => 'bytes',
2777     },
2778     name => {
2779         description => "VM name.",
2780         type => 'string',
2781         optional => 1,
2782     },
2783     qmpstatus => {
2784         description => "Qemu QMP agent status.",
2785         type => 'string',
2786         optional => 1,
2787     },
2788     pid => {
2789         description => "PID of running qemu process.",
2790         type => 'integer',
2791         optional => 1,
2792     },
2793     uptime => {
2794         description => "Uptime.",
2795         type => 'integer',
2796         optional => 1,
2797         renderer => 'duration',
2798     },
2799     cpus => {
2800         description => "Maximum usable CPUs.",
2801         type => 'number',
2802         optional => 1,
2803     },
2804     lock => {
2805         description => "The current config lock, if any.",
2806         type => 'string',
2807         optional => 1,
2808     },
2809     tags => {
2810         description  => "The current configured tags, if any",
2811         type => 'string',
2812         optional => 1,
2813     },
2814     'running-machine' => {
2815         description => "The currently running machine type (if running).",
2816         type => 'string',
2817         optional => 1,
2818     },
2819     'running-qemu' => {
2820         description => "The currently running QEMU version (if running).",
2821         type => 'string',
2822         optional => 1,
2823     },
2824 };
2825
2826 my $last_proc_pid_stat;
2827
2828 # get VM status information
2829 # This must be fast and should not block ($full == false)
2830 # We only query KVM using QMP if $full == true (this can be slow)
2831 sub vmstatus {
2832     my ($opt_vmid, $full) = @_;
2833
2834     my $res = {};
2835
2836     my $storecfg = PVE::Storage::config();
2837
2838     my $list = vzlist();
2839     my $defaults = load_defaults();
2840
2841     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2842
2843     my $cpucount = $cpuinfo->{cpus} || 1;
2844
2845     foreach my $vmid (keys %$list) {
2846         next if $opt_vmid && ($vmid ne $opt_vmid);
2847
2848         my $conf = PVE::QemuConfig->load_config($vmid);
2849
2850         my $d = { vmid => int($vmid) };
2851         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2852
2853         # fixme: better status?
2854         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2855
2856         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2857         if (defined($size)) {
2858             $d->{disk} = 0; # no info available
2859             $d->{maxdisk} = $size;
2860         } else {
2861             $d->{disk} = 0;
2862             $d->{maxdisk} = 0;
2863         }
2864
2865         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2866             * ($conf->{cores} || $defaults->{cores});
2867         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2868         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2869
2870         $d->{name} = $conf->{name} || "VM $vmid";
2871         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2872             : $defaults->{memory}*(1024*1024);
2873
2874         if ($conf->{balloon}) {
2875             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2876             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2877                 : $defaults->{shares};
2878         }
2879
2880         $d->{uptime} = 0;
2881         $d->{cpu} = 0;
2882         $d->{mem} = 0;
2883
2884         $d->{netout} = 0;
2885         $d->{netin} = 0;
2886
2887         $d->{diskread} = 0;
2888         $d->{diskwrite} = 0;
2889
2890         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2891
2892         $d->{serial} = 1 if conf_has_serial($conf);
2893         $d->{lock} = $conf->{lock} if $conf->{lock};
2894         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2895
2896         $res->{$vmid} = $d;
2897     }
2898
2899     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2900     foreach my $dev (keys %$netdev) {
2901         next if $dev !~ m/^tap([1-9]\d*)i/;
2902         my $vmid = $1;
2903         my $d = $res->{$vmid};
2904         next if !$d;
2905
2906         $d->{netout} += $netdev->{$dev}->{receive};
2907         $d->{netin} += $netdev->{$dev}->{transmit};
2908
2909         if ($full) {
2910             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2911             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2912         }
2913
2914     }
2915
2916     my $ctime = gettimeofday;
2917
2918     foreach my $vmid (keys %$list) {
2919
2920         my $d = $res->{$vmid};
2921         my $pid = $d->{pid};
2922         next if !$pid;
2923
2924         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2925         next if !$pstat; # not running
2926
2927         my $used = $pstat->{utime} + $pstat->{stime};
2928
2929         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2930
2931         if ($pstat->{vsize}) {
2932             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2933         }
2934
2935         my $old = $last_proc_pid_stat->{$pid};
2936         if (!$old) {
2937             $last_proc_pid_stat->{$pid} = {
2938                 time => $ctime,
2939                 used => $used,
2940                 cpu => 0,
2941             };
2942             next;
2943         }
2944
2945         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2946
2947         if ($dtime > 1000) {
2948             my $dutime = $used -  $old->{used};
2949
2950             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2951             $last_proc_pid_stat->{$pid} = {
2952                 time => $ctime,
2953                 used => $used,
2954                 cpu => $d->{cpu},
2955             };
2956         } else {
2957             $d->{cpu} = $old->{cpu};
2958         }
2959     }
2960
2961     return $res if !$full;
2962
2963     my $qmpclient = PVE::QMPClient->new();
2964
2965     my $ballooncb = sub {
2966         my ($vmid, $resp) = @_;
2967
2968         my $info = $resp->{'return'};
2969         return if !$info->{max_mem};
2970
2971         my $d = $res->{$vmid};
2972
2973         # use memory assigned to VM
2974         $d->{maxmem} = $info->{max_mem};
2975         $d->{balloon} = $info->{actual};
2976
2977         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2978             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2979             $d->{freemem} = $info->{free_mem};
2980         }
2981
2982         $d->{ballooninfo} = $info;
2983     };
2984
2985     my $blockstatscb = sub {
2986         my ($vmid, $resp) = @_;
2987         my $data = $resp->{'return'} || [];
2988         my $totalrdbytes = 0;
2989         my $totalwrbytes = 0;
2990
2991         for my $blockstat (@$data) {
2992             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2993             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2994
2995             $blockstat->{device} =~ s/drive-//;
2996             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2997         }
2998         $res->{$vmid}->{diskread} = $totalrdbytes;
2999         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3000     };
3001
3002     my $machinecb = sub {
3003         my ($vmid, $resp) = @_;
3004         my $data = $resp->{'return'} || [];
3005
3006         $res->{$vmid}->{'running-machine'} =
3007             PVE::QemuServer::Machine::current_from_query_machines($data);
3008     };
3009
3010     my $versioncb = sub {
3011         my ($vmid, $resp) = @_;
3012         my $data = $resp->{'return'} // {};
3013         my $version = 'unknown';
3014
3015         if (my $v = $data->{qemu}) {
3016             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
3017         }
3018
3019         $res->{$vmid}->{'running-qemu'} = $version;
3020     };
3021
3022     my $statuscb = sub {
3023         my ($vmid, $resp) = @_;
3024
3025         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3026         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
3027         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
3028         # this fails if ballon driver is not loaded, so this must be
3029         # the last commnand (following command are aborted if this fails).
3030         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3031
3032         my $status = 'unknown';
3033         if (!defined($status = $resp->{'return'}->{status})) {
3034             warn "unable to get VM status\n";
3035             return;
3036         }
3037
3038         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3039     };
3040
3041     foreach my $vmid (keys %$list) {
3042         next if $opt_vmid && ($vmid ne $opt_vmid);
3043         next if !$res->{$vmid}->{pid}; # not running
3044         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3045     }
3046
3047     $qmpclient->queue_execute(undef, 2);
3048
3049     foreach my $vmid (keys %$list) {
3050         next if $opt_vmid && ($vmid ne $opt_vmid);
3051         next if !$res->{$vmid}->{pid}; #not running
3052
3053         # we can't use the $qmpclient since it might have already aborted on
3054         # 'query-balloon', but this might also fail for older versions...
3055         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3056         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3057     }
3058
3059     foreach my $vmid (keys %$list) {
3060         next if $opt_vmid && ($vmid ne $opt_vmid);
3061         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3062     }
3063
3064     return $res;
3065 }
3066
3067 sub conf_has_serial {
3068     my ($conf) = @_;
3069
3070     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3071         if ($conf->{"serial$i"}) {
3072             return 1;
3073         }
3074     }
3075
3076     return 0;
3077 }
3078
3079 sub conf_has_audio {
3080     my ($conf, $id) = @_;
3081
3082     $id //= 0;
3083     my $audio = $conf->{"audio$id"};
3084     return if !defined($audio);
3085
3086     my $audioproperties = parse_property_string($audio_fmt, $audio);
3087     my $audiodriver = $audioproperties->{driver} // 'spice';
3088
3089     return {
3090         dev => $audioproperties->{device},
3091         dev_id => "audiodev$id",
3092         backend => $audiodriver,
3093         backend_id => "$audiodriver-backend${id}",
3094     };
3095 }
3096
3097 sub audio_devs {
3098     my ($audio, $audiopciaddr, $machine_version) = @_;
3099
3100     my $devs = [];
3101
3102     my $id = $audio->{dev_id};
3103     my $audiodev = "";
3104     if (min_version($machine_version, 4, 2)) {
3105         $audiodev = ",audiodev=$audio->{backend_id}";
3106     }
3107
3108     if ($audio->{dev} eq 'AC97') {
3109         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3110     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3111         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3112         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3113         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3114     } else {
3115         die "unkown audio device '$audio->{dev}', implement me!";
3116     }
3117
3118     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3119
3120     return $devs;
3121 }
3122
3123 sub get_tpm_paths {
3124     my ($vmid) = @_;
3125     return {
3126         socket => "/var/run/qemu-server/$vmid.swtpm",
3127         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3128     };
3129 }
3130
3131 sub add_tpm_device {
3132     my ($vmid, $devices, $conf) = @_;
3133
3134     return if !$conf->{tpmstate0};
3135
3136     my $paths = get_tpm_paths($vmid);
3137
3138     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3139     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3140     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3141 }
3142
3143 sub start_swtpm {
3144     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3145
3146     return if !$tpmdrive;
3147
3148     my $state;
3149     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3150     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3151     if ($storeid) {
3152         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3153     } else {
3154         $state = $tpm->{file};
3155     }
3156
3157     my $paths = get_tpm_paths($vmid);
3158
3159     # during migration, we will get state from remote
3160     #
3161     if (!$migration) {
3162         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3163         my $setup_cmd = [
3164             "swtpm_setup",
3165             "--tpmstate",
3166             "file://$state",
3167             "--createek",
3168             "--create-ek-cert",
3169             "--create-platform-cert",
3170             "--lock-nvram",
3171             "--config",
3172             "/etc/swtpm_setup.conf", # do not use XDG configs
3173             "--runas",
3174             "0", # force creation as root, error if not possible
3175             "--not-overwrite", # ignore existing state, do not modify
3176         ];
3177
3178         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3179         # TPM 2.0 supports ECC crypto, use if possible
3180         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3181
3182         run_command($setup_cmd, outfunc => sub {
3183             print "swtpm_setup: $1\n";
3184         });
3185     }
3186
3187     my $emulator_cmd = [
3188         "swtpm",
3189         "socket",
3190         "--tpmstate",
3191         "backend-uri=file://$state,mode=0600",
3192         "--ctrl",
3193         "type=unixio,path=$paths->{socket},mode=0600",
3194         "--pid",
3195         "file=$paths->{pid}",
3196         "--terminate", # terminate on QEMU disconnect
3197         "--daemon",
3198     ];
3199     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3200     run_command($emulator_cmd, outfunc => sub { print $1; });
3201
3202     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3203     while (! -e $paths->{pid}) {
3204         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3205         usleep(50_000);
3206     }
3207
3208     # return untainted PID of swtpm daemon so it can be killed on error
3209     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3210     return $1;
3211 }
3212
3213 sub vga_conf_has_spice {
3214     my ($vga) = @_;
3215
3216     my $vgaconf = parse_vga($vga);
3217     my $vgatype = $vgaconf->{type};
3218     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3219
3220     return $1 || 1;
3221 }
3222
3223 sub is_native($) {
3224     my ($arch) = @_;
3225     return get_host_arch() eq $arch;
3226 }
3227
3228 sub get_vm_arch {
3229     my ($conf) = @_;
3230     return $conf->{arch} // get_host_arch();
3231 }
3232
3233 my $default_machines = {
3234     x86_64 => 'pc',
3235     aarch64 => 'virt',
3236 };
3237
3238 sub get_installed_machine_version {
3239     my ($kvmversion) = @_;
3240     $kvmversion = kvm_user_version() if !defined($kvmversion);
3241     $kvmversion =~ m/^(\d+\.\d+)/;
3242     return $1;
3243 }
3244
3245 sub windows_get_pinned_machine_version {
3246     my ($machine, $base_version, $kvmversion) = @_;
3247
3248     my $pin_version = $base_version;
3249     if (!defined($base_version) ||
3250         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3251     ) {
3252         $pin_version = get_installed_machine_version($kvmversion);
3253     }
3254     if (!$machine || $machine eq 'pc') {
3255         $machine = "pc-i440fx-$pin_version";
3256     } elsif ($machine eq 'q35') {
3257         $machine = "pc-q35-$pin_version";
3258     } elsif ($machine eq 'virt') {
3259         $machine = "virt-$pin_version";
3260     } else {
3261         warn "unknown machine type '$machine', not touching that!\n";
3262     }
3263
3264     return $machine;
3265 }
3266
3267 sub get_vm_machine {
3268     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3269
3270     my $machine = $forcemachine || $conf->{machine};
3271
3272     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3273         $kvmversion //= kvm_user_version();
3274         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3275         # layout which confuses windows quite a bit and may result in various regressions..
3276         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3277         if (windows_version($conf->{ostype})) {
3278             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3279         }
3280         $arch //= 'x86_64';
3281         $machine ||= $default_machines->{$arch};
3282         if ($add_pve_version) {
3283             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3284             $machine .= "+pve$pvever";
3285         }
3286     }
3287
3288     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3289         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3290         $machine = $1 if $is_pxe;
3291
3292         # for version-pinned machines that do not include a pve-version (e.g.
3293         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3294         $machine .= '+pve0';
3295
3296         $machine .= '.pxe' if $is_pxe;
3297     }
3298
3299     return $machine;
3300 }
3301
3302 sub get_ovmf_files($$$) {
3303     my ($arch, $efidisk, $smm) = @_;
3304
3305     my $types = $OVMF->{$arch}
3306         or die "no OVMF images known for architecture '$arch'\n";
3307
3308     my $type = 'default';
3309     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3310         $type = $smm ? "4m" : "4m-no-smm";
3311         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3312     }
3313
3314     return $types->{$type}->@*;
3315 }
3316
3317 my $Arch2Qemu = {
3318     aarch64 => '/usr/bin/qemu-system-aarch64',
3319     x86_64 => '/usr/bin/qemu-system-x86_64',
3320 };
3321 sub get_command_for_arch($) {
3322     my ($arch) = @_;
3323     return '/usr/bin/kvm' if is_native($arch);
3324
3325     my $cmd = $Arch2Qemu->{$arch}
3326         or die "don't know how to emulate architecture '$arch'\n";
3327     return $cmd;
3328 }
3329
3330 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3331 # to use in a QEMU command line (-cpu element), first array_intersect the result
3332 # of query_supported_ with query_understood_. This is necessary because:
3333 #
3334 # a) query_understood_ returns flags the host cannot use and
3335 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3336 #    flags, but CPU settings - with most of them being flags. Those settings
3337 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3338 #
3339 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3340 # expensive. If you need the value returned from this, you can get it much
3341 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3342 # $accel being 'kvm' or 'tcg'.
3343 #
3344 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3345 # changes, automatically populating pmxcfs.
3346 #
3347 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3348 # since kvm and tcg machines support different flags
3349 #
3350 sub query_supported_cpu_flags {
3351     my ($arch) = @_;
3352
3353     $arch //= get_host_arch();
3354     my $default_machine = $default_machines->{$arch};
3355
3356     my $flags = {};
3357
3358     # FIXME: Once this is merged, the code below should work for ARM as well:
3359     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3360     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3361         $arch eq "aarch64";
3362
3363     my $kvm_supported = defined(kvm_version());
3364     my $qemu_cmd = get_command_for_arch($arch);
3365     my $fakevmid = -1;
3366     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3367
3368     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3369     my $query_supported_run_qemu = sub {
3370         my ($kvm) = @_;
3371
3372         my $flags = {};
3373         my $cmd = [
3374             $qemu_cmd,
3375             '-machine', $default_machine,
3376             '-display', 'none',
3377             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3378             '-mon', 'chardev=qmp,mode=control',
3379             '-pidfile', $pidfile,
3380             '-S', '-daemonize'
3381         ];
3382
3383         if (!$kvm) {
3384             push @$cmd, '-accel', 'tcg';
3385         }
3386
3387         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3388         die "QEMU flag querying VM exited with code " . $rc if $rc;
3389
3390         eval {
3391             my $cmd_result = mon_cmd(
3392                 $fakevmid,
3393                 'query-cpu-model-expansion',
3394                 type => 'full',
3395                 model => { name => 'host' }
3396             );
3397
3398             my $props = $cmd_result->{model}->{props};
3399             foreach my $prop (keys %$props) {
3400                 next if $props->{$prop} ne '1';
3401                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3402                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3403                 # We only keep those with underscores, to match /proc/cpuinfo
3404                 $prop =~ s/\.|-/_/g;
3405                 $flags->{$prop} = 1;
3406             }
3407         };
3408         my $err = $@;
3409
3410         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3411         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3412
3413         die $err if $err;
3414
3415         return [ sort keys %$flags ];
3416     };
3417
3418     # We need to query QEMU twice, since KVM and TCG have different supported flags
3419     PVE::QemuConfig->lock_config($fakevmid, sub {
3420         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3421         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3422
3423         if ($kvm_supported) {
3424             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3425             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3426         }
3427     });
3428
3429     return $flags;
3430 }
3431
3432 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3433 my $understood_cpu_flag_dir = "/usr/share/kvm";
3434 sub query_understood_cpu_flags {
3435     my $arch = get_host_arch();
3436     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3437
3438     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3439         if ! -e $filepath;
3440
3441     my $raw = file_get_contents($filepath);
3442     $raw =~ s/^\s+|\s+$//g;
3443     my @flags = split(/\s+/, $raw);
3444
3445     return \@flags;
3446 }
3447
3448 my sub get_cpuunits {
3449     my ($conf) = @_;
3450     my $is_cgroupv2 = PVE::CGroup::cgroup_mode() == 2;
3451
3452     my $cpuunits = $conf->{cpuunits};
3453     return $is_cgroupv2 ? 100 : 1024 if !defined($cpuunits);
3454
3455     if ($is_cgroupv2) {
3456         $cpuunits = 10000 if $cpuunits >= 10000; # v1 can be higher, so clamp v2 there
3457     } else {
3458         $cpuunits = 2 if $cpuunits < 2; # v2 can be lower, so clamp v1 there
3459     }
3460     return $cpuunits;
3461 }
3462
3463 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3464 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3465 my sub should_disable_smm {
3466     my ($conf, $vga) = @_;
3467
3468     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3469         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3470 }
3471
3472 sub config_to_command {
3473     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3474         $pbs_backing) = @_;
3475
3476     my $cmd = [];
3477     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3478     my $devices = [];
3479     my $bridges = {};
3480     my $ostype = $conf->{ostype};
3481     my $winversion = windows_version($ostype);
3482     my $kvm = $conf->{kvm};
3483     my $nodename = nodename();
3484
3485     my $arch = get_vm_arch($conf);
3486     my $kvm_binary = get_command_for_arch($arch);
3487     my $kvmver = kvm_user_version($kvm_binary);
3488
3489     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3490         $kvmver //= "undefined";
3491         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3492     }
3493
3494     my $add_pve_version = min_version($kvmver, 4, 1);
3495
3496     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3497     my $machine_version = extract_version($machine_type, $kvmver);
3498     $kvm //= 1 if is_native($arch);
3499
3500     $machine_version =~ m/(\d+)\.(\d+)/;
3501     my ($machine_major, $machine_minor) = ($1, $2);
3502
3503     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3504         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3505     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3506         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3507             ." please upgrade node '$nodename'\n"
3508     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3509         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3510         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3511             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3512             ." node '$nodename'\n";
3513     }
3514
3515     # if a specific +pve version is required for a feature, use $version_guard
3516     # instead of min_version to allow machines to be run with the minimum
3517     # required version
3518     my $required_pve_version = 0;
3519     my $version_guard = sub {
3520         my ($major, $minor, $pve) = @_;
3521         return 0 if !min_version($machine_version, $major, $minor, $pve);
3522         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3523         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3524         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3525         return 1;
3526     };
3527
3528     if ($kvm && !defined kvm_version()) {
3529         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3530             ." or enable in BIOS.\n";
3531     }
3532
3533     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3534     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3535     my $use_old_bios_files = undef;
3536     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3537
3538     push @$cmd, $kvm_binary;
3539
3540     push @$cmd, '-id', $vmid;
3541
3542     my $vmname = $conf->{name} || "vm$vmid";
3543
3544     push @$cmd, '-name', "$vmname,debug-threads=on";
3545
3546     push @$cmd, '-no-shutdown';
3547
3548     my $use_virtio = 0;
3549
3550     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3551     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3552     push @$cmd, '-mon', "chardev=qmp,mode=control";
3553
3554     if (min_version($machine_version, 2, 12)) {
3555         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3556         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3557     }
3558
3559     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3560
3561     push @$cmd, '-daemonize';
3562
3563     if ($conf->{smbios1}) {
3564         my $smbios_conf = parse_smbios1($conf->{smbios1});
3565         if ($smbios_conf->{base64}) {
3566             # Do not pass base64 flag to qemu
3567             delete $smbios_conf->{base64};
3568             my $smbios_string = "";
3569             foreach my $key (keys %$smbios_conf) {
3570                 my $value;
3571                 if ($key eq "uuid") {
3572                     $value = $smbios_conf->{uuid}
3573                 } else {
3574                     $value = decode_base64($smbios_conf->{$key});
3575                 }
3576                 # qemu accepts any binary data, only commas need escaping by double comma
3577                 $value =~ s/,/,,/g;
3578                 $smbios_string .= "," . $key . "=" . $value if $value;
3579             }
3580             push @$cmd, '-smbios', "type=1" . $smbios_string;
3581         } else {
3582             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3583         }
3584     }
3585
3586     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3587         my $d;
3588         if (my $efidisk = $conf->{efidisk0}) {
3589             $d = parse_drive('efidisk0', $efidisk);
3590         }
3591
3592         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3593         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3594
3595         my ($path, $format);
3596         my $read_only_str = '';
3597         if ($d) {
3598             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3599             $format = $d->{format};
3600             if ($storeid) {
3601                 $path = PVE::Storage::path($storecfg, $d->{file});
3602                 if (!defined($format)) {
3603                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3604                     $format = qemu_img_format($scfg, $volname);
3605                 }
3606             } else {
3607                 $path = $d->{file};
3608                 die "efidisk format must be specified\n"
3609                     if !defined($format);
3610             }
3611
3612             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3613         } else {
3614             warn "no efidisk configured! Using temporary efivars disk.\n";
3615             $path = "/tmp/$vmid-ovmf.fd";
3616             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3617             $format = 'raw';
3618         }
3619
3620         my $size_str = "";
3621
3622         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3623             $size_str = ",size=" . (-s $ovmf_vars);
3624         }
3625
3626         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3627         my $cache = "";
3628         if ($path =~ m/^rbd:/) {
3629                 $cache = ',cache=writeback';
3630                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3631         }
3632
3633         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3634         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3635     }
3636
3637     if ($q35) { # tell QEMU to load q35 config early
3638         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3639         if (min_version($machine_version, 4, 0)) {
3640             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3641         } else {
3642             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3643         }
3644     }
3645
3646     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3647         push @$cmd, $fixups->@*;
3648     }
3649
3650     if ($conf->{vmgenid}) {
3651         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3652     }
3653
3654     # add usb controllers
3655     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3656         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3657     push @$devices, @usbcontrollers if @usbcontrollers;
3658     my $vga = parse_vga($conf->{vga});
3659
3660     my $qxlnum = vga_conf_has_spice($conf->{vga});
3661     $vga->{type} = 'qxl' if $qxlnum;
3662
3663     if (!$vga->{type}) {
3664         if ($arch eq 'aarch64') {
3665             $vga->{type} = 'virtio';
3666         } elsif (min_version($machine_version, 2, 9)) {
3667             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3668         } else {
3669             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3670         }
3671     }
3672
3673     # enable absolute mouse coordinates (needed by vnc)
3674     my $tablet = $conf->{tablet};
3675     if (!defined($tablet)) {
3676         $tablet = $defaults->{tablet};
3677         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3678         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3679     }
3680
3681     if ($tablet) {
3682         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3683         my $kbd = print_keyboarddevice_full($conf, $arch);
3684         push @$devices, '-device', $kbd if defined($kbd);
3685     }
3686
3687     my $bootorder = device_bootorder($conf);
3688
3689     # host pci device passthrough
3690     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3691         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3692
3693     # usb devices
3694     my $usb_dev_features = {};
3695     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3696
3697     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3698         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3699     push @$devices, @usbdevices if @usbdevices;
3700
3701     # serial devices
3702     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3703         my $path = $conf->{"serial$i"} or next;
3704         if ($path eq 'socket') {
3705             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3706             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3707             # On aarch64, serial0 is the UART device. Qemu only allows
3708             # connecting UART devices via the '-serial' command line, as
3709             # the device has a fixed slot on the hardware...
3710             if ($arch eq 'aarch64' && $i == 0) {
3711                 push @$devices, '-serial', "chardev:serial$i";
3712             } else {
3713                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3714             }
3715         } else {
3716             die "no such serial device\n" if ! -c $path;
3717             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3718             push @$devices, '-device', "isa-serial,chardev=serial$i";
3719         }
3720     }
3721
3722     # parallel devices
3723     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3724         if (my $path = $conf->{"parallel$i"}) {
3725             die "no such parallel device\n" if ! -c $path;
3726             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3727             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3728             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3729         }
3730     }
3731
3732     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3733         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3734         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3735         push @$devices, @$audio_devs;
3736     }
3737
3738     add_tpm_device($vmid, $devices, $conf);
3739
3740     my $sockets = 1;
3741     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3742     $sockets = $conf->{sockets} if  $conf->{sockets};
3743
3744     my $cores = $conf->{cores} || 1;
3745
3746     my $maxcpus = $sockets * $cores;
3747
3748     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3749
3750     my $allowed_vcpus = $cpuinfo->{cpus};
3751
3752     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3753
3754     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3755         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3756         for (my $i = 2; $i <= $vcpus; $i++)  {
3757             my $cpustr = print_cpu_device($conf,$i);
3758             push @$cmd, '-device', $cpustr;
3759         }
3760
3761     } else {
3762
3763         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3764     }
3765     push @$cmd, '-nodefaults';
3766
3767     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3768
3769     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3770
3771     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3772
3773     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3774         push @$devices, '-device', print_vga_device(
3775             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3776
3777         push @$cmd, '-display', 'egl-headless,gl=core' if $vga->{type} eq 'virtio-gl'; # VIRGL
3778
3779         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3780         push @$cmd,  '-vnc', "unix:$socket,password=on";
3781     } else {
3782         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3783         push @$cmd, '-nographic';
3784     }
3785
3786     # time drift fix
3787     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3788     my $useLocaltime = $conf->{localtime};
3789
3790     if ($winversion >= 5) { # windows
3791         $useLocaltime = 1 if !defined($conf->{localtime});
3792
3793         # use time drift fix when acpi is enabled
3794         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3795             $tdf = 1 if !defined($conf->{tdf});
3796         }
3797     }
3798
3799     if ($winversion >= 6) {
3800         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3801         push @$cmd, '-no-hpet';
3802     }
3803
3804     push @$rtcFlags, 'driftfix=slew' if $tdf;
3805
3806     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3807         push @$rtcFlags, "base=$conf->{startdate}";
3808     } elsif ($useLocaltime) {
3809         push @$rtcFlags, 'base=localtime';
3810     }
3811
3812     if ($forcecpu) {
3813         push @$cmd, '-cpu', $forcecpu;
3814     } else {
3815         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3816     }
3817
3818     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3819
3820     push @$cmd, '-S' if $conf->{freeze};
3821
3822     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3823
3824     my $guest_agent = parse_guest_agent($conf);
3825
3826     if ($guest_agent->{enabled}) {
3827         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3828         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3829
3830         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3831             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3832             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3833             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3834         } elsif ($guest_agent->{type} eq 'isa') {
3835             push @$devices, '-device', "isa-serial,chardev=qga0";
3836         }
3837     }
3838
3839     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3840     if ($rng && $version_guard->(4, 1, 2)) {
3841         check_rng_source($rng->{source});
3842
3843         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3844         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3845         my $limiter_str = "";
3846         if ($max_bytes) {
3847             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3848         }
3849
3850         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3851         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3852         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3853     }
3854
3855     my $spice_port;
3856
3857     if ($qxlnum || $vga->{type} =~ /^virtio/) {
3858         if ($qxlnum > 1) {
3859             if ($winversion){
3860                 for (my $i = 1; $i < $qxlnum; $i++){
3861                     push @$devices, '-device', print_vga_device(
3862                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3863                 }
3864             } else {
3865                 # assume other OS works like Linux
3866                 my ($ram, $vram) = ("134217728", "67108864");
3867                 if ($vga->{memory}) {
3868                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3869                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3870                 }
3871                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3872                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3873             }
3874         }
3875
3876         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3877
3878         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3879         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3880         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3881
3882         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3883         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3884         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3885
3886         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3887         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3888
3889         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3890         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3891         if ($spice_enhancement->{foldersharing}) {
3892             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3893             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3894         }
3895
3896         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3897         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3898             if $spice_enhancement->{videostreaming};
3899
3900         push @$devices, '-spice', "$spice_opts";
3901     }
3902
3903     # enable balloon by default, unless explicitly disabled
3904     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3905         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3906         my $ballooncmd = "virtio-balloon-pci,id=balloon0$pciaddr";
3907         $ballooncmd .= ",free-page-reporting=on" if min_version($machine_version, 6, 2);
3908         push @$devices, '-device', $ballooncmd;
3909     }
3910
3911     if ($conf->{watchdog}) {
3912         my $wdopts = parse_watchdog($conf->{watchdog});
3913         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3914         my $watchdog = $wdopts->{model} || 'i6300esb';
3915         push @$devices, '-device', "$watchdog$pciaddr";
3916         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3917     }
3918
3919     my $vollist = [];
3920     my $scsicontroller = {};
3921     my $ahcicontroller = {};
3922     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3923
3924     # Add iscsi initiator name if available
3925     if (my $initiator = get_initiator_name()) {
3926         push @$devices, '-iscsi', "initiator-name=$initiator";
3927     }
3928
3929     PVE::QemuConfig->foreach_volume($conf, sub {
3930         my ($ds, $drive) = @_;
3931
3932         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3933             check_volume_storage_type($storecfg, $drive->{file});
3934             push @$vollist, $drive->{file};
3935         }
3936
3937         # ignore efidisk here, already added in bios/fw handling code above
3938         return if $drive->{interface} eq 'efidisk';
3939         # similar for TPM
3940         return if $drive->{interface} eq 'tpmstate';
3941
3942         $use_virtio = 1 if $ds =~ m/^virtio/;
3943
3944         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3945
3946         if ($drive->{interface} eq 'virtio'){
3947            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3948         }
3949
3950         if ($drive->{interface} eq 'scsi') {
3951
3952             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3953
3954             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3955                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3956
3957             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3958             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3959
3960             my $iothread = '';
3961             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3962                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3963                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3964             } elsif ($drive->{iothread}) {
3965                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3966             }
3967
3968             my $queues = '';
3969             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3970                 $queues = ",num_queues=$drive->{queues}";
3971             }
3972
3973             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3974                 if !$scsicontroller->{$controller};
3975             $scsicontroller->{$controller}=1;
3976         }
3977
3978         if ($drive->{interface} eq 'sata') {
3979             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3980             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3981             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3982                 if !$ahcicontroller->{$controller};
3983             $ahcicontroller->{$controller}=1;
3984         }
3985
3986         my $pbs_conf = $pbs_backing->{$ds};
3987         my $pbs_name = undef;
3988         if ($pbs_conf) {
3989             $pbs_name = "drive-$ds-pbs";
3990             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3991         }
3992
3993         my $drive_cmd = print_drive_commandline_full(
3994             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3995
3996         # extra protection for templates, but SATA and IDE don't support it..
3997         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3998
3999         push @$devices, '-drive',$drive_cmd;
4000         push @$devices, '-device', print_drivedevice_full(
4001             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4002     });
4003
4004     for (my $i = 0; $i < $MAX_NETS; $i++) {
4005         my $netname = "net$i";
4006
4007         next if !$conf->{$netname};
4008         my $d = parse_net($conf->{$netname});
4009         next if !$d;
4010
4011         $use_virtio = 1 if $d->{model} eq 'virtio';
4012
4013         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
4014
4015         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
4016         push @$devices, '-netdev', $netdevfull;
4017
4018         my $netdevicefull = print_netdevice_full(
4019             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
4020
4021         push @$devices, '-device', $netdevicefull;
4022     }
4023
4024     if ($conf->{ivshmem}) {
4025         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4026
4027         my $bus;
4028         if ($q35) {
4029             $bus = print_pcie_addr("ivshmem");
4030         } else {
4031             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4032         }
4033
4034         my $ivshmem_name = $ivshmem->{name} // $vmid;
4035         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4036
4037         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4038         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
4039             .",size=$ivshmem->{size}M";
4040     }
4041
4042     # pci.4 is nested in pci.1
4043     $bridges->{1} = 1 if $bridges->{4};
4044
4045     if (!$q35) { # add pci bridges
4046         if (min_version($machine_version, 2, 3)) {
4047            $bridges->{1} = 1;
4048            $bridges->{2} = 1;
4049         }
4050         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4051     }
4052
4053     for my $k (sort {$b cmp $a} keys %$bridges) {
4054         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
4055
4056         my $k_name = $k;
4057         if ($k == 2 && $legacy_igd) {
4058             $k_name = "$k-igd";
4059         }
4060         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4061         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4062
4063         if ($q35) { # add after -readconfig pve-q35.cfg
4064             splice @$devices, 2, 0, '-device', $devstr;
4065         } else {
4066             unshift @$devices, '-device', $devstr if $k > 0;
4067         }
4068     }
4069
4070     if (!$kvm) {
4071         push @$machineFlags, 'accel=tcg';
4072     }
4073
4074     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4075
4076     my $machine_type_min = $machine_type;
4077     if ($add_pve_version) {
4078         $machine_type_min =~ s/\+pve\d+$//;
4079         $machine_type_min .= "+pve$required_pve_version";
4080     }
4081     push @$machineFlags, "type=${machine_type_min}";
4082
4083     push @$cmd, @$devices;
4084     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4085     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4086     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4087
4088     if (my $vmstate = $conf->{vmstate}) {
4089         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4090         push @$vollist, $vmstate;
4091         push @$cmd, '-loadstate', $statepath;
4092         print "activating and using '$vmstate' as vmstate\n";
4093     }
4094
4095     if (PVE::QemuConfig->is_template($conf)) {
4096         # needed to workaround base volumes being read-only
4097         push @$cmd, '-snapshot';
4098     }
4099
4100     # add custom args
4101     if ($conf->{args}) {
4102         my $aa = PVE::Tools::split_args($conf->{args});
4103         push @$cmd, @$aa;
4104     }
4105
4106     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4107 }
4108
4109 sub check_rng_source {
4110     my ($source) = @_;
4111
4112     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4113     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4114         if ! -e $source;
4115
4116     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4117     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4118         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4119         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4120         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4121             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4122             ." to the host.\n";
4123     }
4124 }
4125
4126 sub spice_port {
4127     my ($vmid) = @_;
4128
4129     my $res = mon_cmd($vmid, 'query-spice');
4130
4131     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4132 }
4133
4134 sub vm_devices_list {
4135     my ($vmid) = @_;
4136
4137     my $res = mon_cmd($vmid, 'query-pci');
4138     my $devices_to_check = [];
4139     my $devices = {};
4140     foreach my $pcibus (@$res) {
4141         push @$devices_to_check, @{$pcibus->{devices}},
4142     }
4143
4144     while (@$devices_to_check) {
4145         my $to_check = [];
4146         for my $d (@$devices_to_check) {
4147             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4148             next if !$d->{'pci_bridge'};
4149
4150             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4151             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4152         }
4153         $devices_to_check = $to_check;
4154     }
4155
4156     my $resblock = mon_cmd($vmid, 'query-block');
4157     foreach my $block (@$resblock) {
4158         if($block->{device} =~ m/^drive-(\S+)/){
4159                 $devices->{$1} = 1;
4160         }
4161     }
4162
4163     my $resmice = mon_cmd($vmid, 'query-mice');
4164     foreach my $mice (@$resmice) {
4165         if ($mice->{name} eq 'QEMU HID Tablet') {
4166             $devices->{tablet} = 1;
4167             last;
4168         }
4169     }
4170
4171     # for usb devices there is no query-usb
4172     # but we can iterate over the entries in
4173     # qom-list path=/machine/peripheral
4174     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4175     foreach my $per (@$resperipheral) {
4176         if ($per->{name} =~ m/^usb\d+$/) {
4177             $devices->{$per->{name}} = 1;
4178         }
4179     }
4180
4181     return $devices;
4182 }
4183
4184 sub vm_deviceplug {
4185     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4186
4187     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4188
4189     my $devices_list = vm_devices_list($vmid);
4190     return 1 if defined($devices_list->{$deviceid});
4191
4192     # add PCI bridge if we need it for the device
4193     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4194
4195     if ($deviceid eq 'tablet') {
4196         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4197     } elsif ($deviceid eq 'keyboard') {
4198         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4199     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4200         die "usb hotplug currently not reliable\n";
4201         # since we can't reliably hot unplug all added usb devices and usb
4202         # passthrough breaks live migration we disable usb hotplugging for now
4203         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4204     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4205         qemu_iothread_add($vmid, $deviceid, $device);
4206
4207         qemu_driveadd($storecfg, $vmid, $device);
4208         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4209
4210         qemu_deviceadd($vmid, $devicefull);
4211         eval { qemu_deviceaddverify($vmid, $deviceid); };
4212         if (my $err = $@) {
4213             eval { qemu_drivedel($vmid, $deviceid); };
4214             warn $@ if $@;
4215             die $err;
4216         }
4217     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4218         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4219         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4220         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4221
4222         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4223
4224         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4225             qemu_iothread_add($vmid, $deviceid, $device);
4226             $devicefull .= ",iothread=iothread-$deviceid";
4227         }
4228
4229         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4230             $devicefull .= ",num_queues=$device->{queues}";
4231         }
4232
4233         qemu_deviceadd($vmid, $devicefull);
4234         qemu_deviceaddverify($vmid, $deviceid);
4235     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4236         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4237         qemu_driveadd($storecfg, $vmid, $device);
4238
4239         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4240         eval { qemu_deviceadd($vmid, $devicefull); };
4241         if (my $err = $@) {
4242             eval { qemu_drivedel($vmid, $deviceid); };
4243             warn $@ if $@;
4244             die $err;
4245         }
4246     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4247         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4248
4249         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4250         my $use_old_bios_files = undef;
4251         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4252
4253         my $netdevicefull = print_netdevice_full(
4254             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4255         qemu_deviceadd($vmid, $netdevicefull);
4256         eval {
4257             qemu_deviceaddverify($vmid, $deviceid);
4258             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4259         };
4260         if (my $err = $@) {
4261             eval { qemu_netdevdel($vmid, $deviceid); };
4262             warn $@ if $@;
4263             die $err;
4264         }
4265     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4266         my $bridgeid = $2;
4267         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4268         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4269
4270         qemu_deviceadd($vmid, $devicefull);
4271         qemu_deviceaddverify($vmid, $deviceid);
4272     } else {
4273         die "can't hotplug device '$deviceid'\n";
4274     }
4275
4276     return 1;
4277 }
4278
4279 # fixme: this should raise exceptions on error!
4280 sub vm_deviceunplug {
4281     my ($vmid, $conf, $deviceid) = @_;
4282
4283     my $devices_list = vm_devices_list($vmid);
4284     return 1 if !defined($devices_list->{$deviceid});
4285
4286     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4287     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4288
4289     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4290         qemu_devicedel($vmid, $deviceid);
4291     } elsif ($deviceid =~ m/^usb\d+$/) {
4292         die "usb hotplug currently not reliable\n";
4293         # when unplugging usb devices this way, there may be remaining usb
4294         # controllers/hubs so we disable it for now
4295         #qemu_devicedel($vmid, $deviceid);
4296         #qemu_devicedelverify($vmid, $deviceid);
4297     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4298         my $device = parse_drive($deviceid, $conf->{$deviceid});
4299
4300         qemu_devicedel($vmid, $deviceid);
4301         qemu_devicedelverify($vmid, $deviceid);
4302         qemu_drivedel($vmid, $deviceid);
4303         qemu_iothread_del($vmid, $deviceid, $device);
4304     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4305         qemu_devicedel($vmid, $deviceid);
4306         qemu_devicedelverify($vmid, $deviceid);
4307     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4308         my $device = parse_drive($deviceid, $conf->{$deviceid});
4309
4310         qemu_devicedel($vmid, $deviceid);
4311         qemu_devicedelverify($vmid, $deviceid);
4312         qemu_drivedel($vmid, $deviceid);
4313         qemu_deletescsihw($conf, $vmid, $deviceid);
4314
4315         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4316             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4317     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4318         qemu_devicedel($vmid, $deviceid);
4319         qemu_devicedelverify($vmid, $deviceid);
4320         qemu_netdevdel($vmid, $deviceid);
4321     } else {
4322         die "can't unplug device '$deviceid'\n";
4323     }
4324
4325     return 1;
4326 }
4327
4328 sub qemu_deviceadd {
4329     my ($vmid, $devicefull) = @_;
4330
4331     $devicefull = "driver=".$devicefull;
4332     my %options =  split(/[=,]/, $devicefull);
4333
4334     mon_cmd($vmid, "device_add" , %options);
4335 }
4336
4337 sub qemu_devicedel {
4338     my ($vmid, $deviceid) = @_;
4339
4340     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4341 }
4342
4343 sub qemu_iothread_add {
4344     my ($vmid, $deviceid, $device) = @_;
4345
4346     if ($device->{iothread}) {
4347         my $iothreads = vm_iothreads_list($vmid);
4348         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4349     }
4350 }
4351
4352 sub qemu_iothread_del {
4353     my ($vmid, $deviceid, $device) = @_;
4354
4355     if ($device->{iothread}) {
4356         my $iothreads = vm_iothreads_list($vmid);
4357         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4358     }
4359 }
4360
4361 sub qemu_objectadd {
4362     my ($vmid, $objectid, $qomtype) = @_;
4363
4364     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4365
4366     return 1;
4367 }
4368
4369 sub qemu_objectdel {
4370     my ($vmid, $objectid) = @_;
4371
4372     mon_cmd($vmid, "object-del", id => $objectid);
4373
4374     return 1;
4375 }
4376
4377 sub qemu_driveadd {
4378     my ($storecfg, $vmid, $device) = @_;
4379
4380     my $kvmver = get_running_qemu_version($vmid);
4381     my $io_uring = min_version($kvmver, 6, 0);
4382     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4383     $drive =~ s/\\/\\\\/g;
4384     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4385
4386     # If the command succeeds qemu prints: "OK"
4387     return 1 if $ret =~ m/OK/s;
4388
4389     die "adding drive failed: $ret\n";
4390 }
4391
4392 sub qemu_drivedel {
4393     my ($vmid, $deviceid) = @_;
4394
4395     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4396     $ret =~ s/^\s+//;
4397
4398     return 1 if $ret eq "";
4399
4400     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4401     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4402
4403     die "deleting drive $deviceid failed : $ret\n";
4404 }
4405
4406 sub qemu_deviceaddverify {
4407     my ($vmid, $deviceid) = @_;
4408
4409     for (my $i = 0; $i <= 5; $i++) {
4410          my $devices_list = vm_devices_list($vmid);
4411          return 1 if defined($devices_list->{$deviceid});
4412          sleep 1;
4413     }
4414
4415     die "error on hotplug device '$deviceid'\n";
4416 }
4417
4418
4419 sub qemu_devicedelverify {
4420     my ($vmid, $deviceid) = @_;
4421
4422     # need to verify that the device is correctly removed as device_del
4423     # is async and empty return is not reliable
4424
4425     for (my $i = 0; $i <= 5; $i++) {
4426          my $devices_list = vm_devices_list($vmid);
4427          return 1 if !defined($devices_list->{$deviceid});
4428          sleep 1;
4429     }
4430
4431     die "error on hot-unplugging device '$deviceid'\n";
4432 }
4433
4434 sub qemu_findorcreatescsihw {
4435     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4436
4437     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4438
4439     my $scsihwid="$controller_prefix$controller";
4440     my $devices_list = vm_devices_list($vmid);
4441
4442     if (!defined($devices_list->{$scsihwid})) {
4443         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4444     }
4445
4446     return 1;
4447 }
4448
4449 sub qemu_deletescsihw {
4450     my ($conf, $vmid, $opt) = @_;
4451
4452     my $device = parse_drive($opt, $conf->{$opt});
4453
4454     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4455         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4456         return 1;
4457     }
4458
4459     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4460
4461     my $devices_list = vm_devices_list($vmid);
4462     foreach my $opt (keys %{$devices_list}) {
4463         if (is_valid_drivename($opt)) {
4464             my $drive = parse_drive($opt, $conf->{$opt});
4465             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4466                 return 1;
4467             }
4468         }
4469     }
4470
4471     my $scsihwid="scsihw$controller";
4472
4473     vm_deviceunplug($vmid, $conf, $scsihwid);
4474
4475     return 1;
4476 }
4477
4478 sub qemu_add_pci_bridge {
4479     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4480
4481     my $bridges = {};
4482
4483     my $bridgeid;
4484
4485     print_pci_addr($device, $bridges, $arch, $machine_type);
4486
4487     while (my ($k, $v) = each %$bridges) {
4488         $bridgeid = $k;
4489     }
4490     return 1 if !defined($bridgeid) || $bridgeid < 1;
4491
4492     my $bridge = "pci.$bridgeid";
4493     my $devices_list = vm_devices_list($vmid);
4494
4495     if (!defined($devices_list->{$bridge})) {
4496         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4497     }
4498
4499     return 1;
4500 }
4501
4502 sub qemu_set_link_status {
4503     my ($vmid, $device, $up) = @_;
4504
4505     mon_cmd($vmid, "set_link", name => $device,
4506                up => $up ? JSON::true : JSON::false);
4507 }
4508
4509 sub qemu_netdevadd {
4510     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4511
4512     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4513     my %options =  split(/[=,]/, $netdev);
4514
4515     if (defined(my $vhost = $options{vhost})) {
4516         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4517     }
4518
4519     if (defined(my $queues = $options{queues})) {
4520         $options{queues} = $queues + 0;
4521     }
4522
4523     mon_cmd($vmid, "netdev_add",  %options);
4524     return 1;
4525 }
4526
4527 sub qemu_netdevdel {
4528     my ($vmid, $deviceid) = @_;
4529
4530     mon_cmd($vmid, "netdev_del", id => $deviceid);
4531 }
4532
4533 sub qemu_usb_hotplug {
4534     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4535
4536     return if !$device;
4537
4538     # remove the old one first
4539     vm_deviceunplug($vmid, $conf, $deviceid);
4540
4541     # check if xhci controller is necessary and available
4542     if ($device->{usb3}) {
4543
4544         my $devicelist = vm_devices_list($vmid);
4545
4546         if (!$devicelist->{xhci}) {
4547             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4548             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4549         }
4550     }
4551     my $d = parse_usb_device($device->{host});
4552     $d->{usb3} = $device->{usb3};
4553
4554     # add the new one
4555     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4556 }
4557
4558 sub qemu_cpu_hotplug {
4559     my ($vmid, $conf, $vcpus) = @_;
4560
4561     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4562
4563     my $sockets = 1;
4564     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4565     $sockets = $conf->{sockets} if  $conf->{sockets};
4566     my $cores = $conf->{cores} || 1;
4567     my $maxcpus = $sockets * $cores;
4568
4569     $vcpus = $maxcpus if !$vcpus;
4570
4571     die "you can't add more vcpus than maxcpus\n"
4572         if $vcpus > $maxcpus;
4573
4574     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4575
4576     if ($vcpus < $currentvcpus) {
4577
4578         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4579
4580             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4581                 qemu_devicedel($vmid, "cpu$i");
4582                 my $retry = 0;
4583                 my $currentrunningvcpus = undef;
4584                 while (1) {
4585                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4586                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4587                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4588                     $retry++;
4589                     sleep 1;
4590                 }
4591                 #update conf after each succesfull cpu unplug
4592                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4593                 PVE::QemuConfig->write_config($vmid, $conf);
4594             }
4595         } else {
4596             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4597         }
4598
4599         return;
4600     }
4601
4602     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4603     die "vcpus in running vm does not match its configuration\n"
4604         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4605
4606     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4607
4608         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4609             my $cpustr = print_cpu_device($conf, $i);
4610             qemu_deviceadd($vmid, $cpustr);
4611
4612             my $retry = 0;
4613             my $currentrunningvcpus = undef;
4614             while (1) {
4615                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4616                 last if scalar(@{$currentrunningvcpus}) == $i;
4617                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4618                 sleep 1;
4619                 $retry++;
4620             }
4621             #update conf after each succesfull cpu hotplug
4622             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4623             PVE::QemuConfig->write_config($vmid, $conf);
4624         }
4625     } else {
4626
4627         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4628             mon_cmd($vmid, "cpu-add", id => int($i));
4629         }
4630     }
4631 }
4632
4633 sub qemu_block_set_io_throttle {
4634     my ($vmid, $deviceid,
4635         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4636         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4637         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4638         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4639
4640     return if !check_running($vmid) ;
4641
4642     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4643         bps => int($bps),
4644         bps_rd => int($bps_rd),
4645         bps_wr => int($bps_wr),
4646         iops => int($iops),
4647         iops_rd => int($iops_rd),
4648         iops_wr => int($iops_wr),
4649         bps_max => int($bps_max),
4650         bps_rd_max => int($bps_rd_max),
4651         bps_wr_max => int($bps_wr_max),
4652         iops_max => int($iops_max),
4653         iops_rd_max => int($iops_rd_max),
4654         iops_wr_max => int($iops_wr_max),
4655         bps_max_length => int($bps_max_length),
4656         bps_rd_max_length => int($bps_rd_max_length),
4657         bps_wr_max_length => int($bps_wr_max_length),
4658         iops_max_length => int($iops_max_length),
4659         iops_rd_max_length => int($iops_rd_max_length),
4660         iops_wr_max_length => int($iops_wr_max_length),
4661     );
4662
4663 }
4664
4665 sub qemu_block_resize {
4666     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4667
4668     my $running = check_running($vmid);
4669
4670     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4671
4672     return if !$running;
4673
4674     my $padding = (1024 - $size % 1024) % 1024;
4675     $size = $size + $padding;
4676
4677     mon_cmd(
4678         $vmid,
4679         "block_resize",
4680         device => $deviceid,
4681         size => int($size),
4682         timeout => 60,
4683     );
4684 }
4685
4686 sub qemu_volume_snapshot {
4687     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4688
4689     my $running = check_running($vmid);
4690
4691     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4692         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4693     } else {
4694         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4695     }
4696 }
4697
4698 sub qemu_volume_snapshot_delete {
4699     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4700
4701     my $running = check_running($vmid);
4702
4703     if($running) {
4704
4705         $running = undef;
4706         my $conf = PVE::QemuConfig->load_config($vmid);
4707         PVE::QemuConfig->foreach_volume($conf, sub {
4708             my ($ds, $drive) = @_;
4709             $running = 1 if $drive->{file} eq $volid;
4710         });
4711     }
4712
4713     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4714         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4715     } else {
4716         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4717     }
4718 }
4719
4720 sub set_migration_caps {
4721     my ($vmid, $savevm) = @_;
4722
4723     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4724
4725     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4726     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4727
4728     my $cap_ref = [];
4729
4730     my $enabled_cap = {
4731         "auto-converge" => 1,
4732         "xbzrle" => 1,
4733         "x-rdma-pin-all" => 0,
4734         "zero-blocks" => 0,
4735         "compress" => 0,
4736         "dirty-bitmaps" => $dirty_bitmaps,
4737     };
4738
4739     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4740
4741     for my $supported_capability (@$supported_capabilities) {
4742         push @$cap_ref, {
4743             capability => $supported_capability->{capability},
4744             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4745         };
4746     }
4747
4748     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4749 }
4750
4751 sub foreach_volid {
4752     my ($conf, $func, @param) = @_;
4753
4754     my $volhash = {};
4755
4756     my $test_volid = sub {
4757         my ($key, $drive, $snapname) = @_;
4758
4759         my $volid = $drive->{file};
4760         return if !$volid;
4761
4762         $volhash->{$volid}->{cdrom} //= 1;
4763         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4764
4765         my $replicate = $drive->{replicate} // 1;
4766         $volhash->{$volid}->{replicate} //= 0;
4767         $volhash->{$volid}->{replicate} = 1 if $replicate;
4768
4769         $volhash->{$volid}->{shared} //= 0;
4770         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4771
4772         $volhash->{$volid}->{referenced_in_config} //= 0;
4773         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4774
4775         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4776             if defined($snapname);
4777
4778         my $size = $drive->{size};
4779         $volhash->{$volid}->{size} //= $size if $size;
4780
4781         $volhash->{$volid}->{is_vmstate} //= 0;
4782         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4783
4784         $volhash->{$volid}->{is_tpmstate} //= 0;
4785         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4786
4787         $volhash->{$volid}->{is_unused} //= 0;
4788         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4789
4790         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4791     };
4792
4793     my $include_opts = {
4794         extra_keys => ['vmstate'],
4795         include_unused => 1,
4796     };
4797
4798     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4799     foreach my $snapname (keys %{$conf->{snapshots}}) {
4800         my $snap = $conf->{snapshots}->{$snapname};
4801         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4802     }
4803
4804     foreach my $volid (keys %$volhash) {
4805         &$func($volid, $volhash->{$volid}, @param);
4806     }
4807 }
4808
4809 my $fast_plug_option = {
4810     'lock' => 1,
4811     'name' => 1,
4812     'onboot' => 1,
4813     'shares' => 1,
4814     'startup' => 1,
4815     'description' => 1,
4816     'protection' => 1,
4817     'vmstatestorage' => 1,
4818     'hookscript' => 1,
4819     'tags' => 1,
4820 };
4821
4822 # hotplug changes in [PENDING]
4823 # $selection hash can be used to only apply specified options, for
4824 # example: { cores => 1 } (only apply changed 'cores')
4825 # $errors ref is used to return error messages
4826 sub vmconfig_hotplug_pending {
4827     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4828
4829     my $defaults = load_defaults();
4830     my $arch = get_vm_arch($conf);
4831     my $machine_type = get_vm_machine($conf, undef, $arch);
4832
4833     # commit values which do not have any impact on running VM first
4834     # Note: those option cannot raise errors, we we do not care about
4835     # $selection and always apply them.
4836
4837     my $add_error = sub {
4838         my ($opt, $msg) = @_;
4839         $errors->{$opt} = "hotplug problem - $msg";
4840     };
4841
4842     my $changes = 0;
4843     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4844         if ($fast_plug_option->{$opt}) {
4845             $conf->{$opt} = $conf->{pending}->{$opt};
4846             delete $conf->{pending}->{$opt};
4847             $changes = 1;
4848         }
4849     }
4850
4851     if ($changes) {
4852         PVE::QemuConfig->write_config($vmid, $conf);
4853     }
4854
4855     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4856
4857     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4858     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4859     foreach my $opt (sort keys %$pending_delete_hash) {
4860         next if $selection && !$selection->{$opt};
4861         my $force = $pending_delete_hash->{$opt}->{force};
4862         eval {
4863             if ($opt eq 'hotplug') {
4864                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4865             } elsif ($opt eq 'tablet') {
4866                 die "skip\n" if !$hotplug_features->{usb};
4867                 if ($defaults->{tablet}) {
4868                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4869                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4870                         if $arch eq 'aarch64';
4871                 } else {
4872                     vm_deviceunplug($vmid, $conf, 'tablet');
4873                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4874                 }
4875             } elsif ($opt =~ m/^usb\d+/) {
4876                 die "skip\n";
4877                 # since we cannot reliably hot unplug usb devices we are disabling it
4878                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4879                 #vm_deviceunplug($vmid, $conf, $opt);
4880             } elsif ($opt eq 'vcpus') {
4881                 die "skip\n" if !$hotplug_features->{cpu};
4882                 qemu_cpu_hotplug($vmid, $conf, undef);
4883             } elsif ($opt eq 'balloon') {
4884                 # enable balloon device is not hotpluggable
4885                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4886                 # here we reset the ballooning value to memory
4887                 my $balloon = $conf->{memory} || $defaults->{memory};
4888                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4889             } elsif ($fast_plug_option->{$opt}) {
4890                 # do nothing
4891             } elsif ($opt =~ m/^net(\d+)$/) {
4892                 die "skip\n" if !$hotplug_features->{network};
4893                 vm_deviceunplug($vmid, $conf, $opt);
4894             } elsif (is_valid_drivename($opt)) {
4895                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4896                 vm_deviceunplug($vmid, $conf, $opt);
4897                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4898             } elsif ($opt =~ m/^memory$/) {
4899                 die "skip\n" if !$hotplug_features->{memory};
4900                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4901             } elsif ($opt eq 'cpuunits') {
4902                 $cgroup->change_cpu_shares(undef, 1024);
4903             } elsif ($opt eq 'cpulimit') {
4904                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4905             } else {
4906                 die "skip\n";
4907             }
4908         };
4909         if (my $err = $@) {
4910             &$add_error($opt, $err) if $err ne "skip\n";
4911         } else {
4912             delete $conf->{$opt};
4913             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4914         }
4915     }
4916
4917     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4918     $apply_pending_cloudinit = sub {
4919         return if $apply_pending_cloudinit_done; # once is enough
4920         $apply_pending_cloudinit_done = 1; # once is enough
4921
4922         my ($key, $value) = @_;
4923
4924         my @cloudinit_opts = keys %$confdesc_cloudinit;
4925         foreach my $opt (keys %{$conf->{pending}}) {
4926             next if !grep { $_ eq $opt } @cloudinit_opts;
4927             $conf->{$opt} = delete $conf->{pending}->{$opt};
4928         }
4929
4930         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4931         foreach my $opt (sort keys %$pending_delete_hash) {
4932             next if !grep { $_ eq $opt } @cloudinit_opts;
4933             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4934             delete $conf->{$opt};
4935         }
4936
4937         my $new_conf = { %$conf };
4938         $new_conf->{$key} = $value;
4939         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4940     };
4941
4942     foreach my $opt (keys %{$conf->{pending}}) {
4943         next if $selection && !$selection->{$opt};
4944         my $value = $conf->{pending}->{$opt};
4945         eval {
4946             if ($opt eq 'hotplug') {
4947                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4948             } elsif ($opt eq 'tablet') {
4949                 die "skip\n" if !$hotplug_features->{usb};
4950                 if ($value == 1) {
4951                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4952                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4953                         if $arch eq 'aarch64';
4954                 } elsif ($value == 0) {
4955                     vm_deviceunplug($vmid, $conf, 'tablet');
4956                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4957                 }
4958             } elsif ($opt =~ m/^usb\d+$/) {
4959                 die "skip\n";
4960                 # since we cannot reliably hot unplug usb devices we disable it for now
4961                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4962                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4963                 #die "skip\n" if !$d;
4964                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4965             } elsif ($opt eq 'vcpus') {
4966                 die "skip\n" if !$hotplug_features->{cpu};
4967                 qemu_cpu_hotplug($vmid, $conf, $value);
4968             } elsif ($opt eq 'balloon') {
4969                 # enable/disable balloning device is not hotpluggable
4970                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4971                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4972                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4973
4974                 # allow manual ballooning if shares is set to zero
4975                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4976                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4977                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4978                 }
4979             } elsif ($opt =~ m/^net(\d+)$/) {
4980                 # some changes can be done without hotplug
4981                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4982                                     $vmid, $opt, $value, $arch, $machine_type);
4983             } elsif (is_valid_drivename($opt)) {
4984                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4985                 # some changes can be done without hotplug
4986                 my $drive = parse_drive($opt, $value);
4987                 if (drive_is_cloudinit($drive)) {
4988                     &$apply_pending_cloudinit($opt, $value);
4989                 }
4990                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4991                                      $vmid, $opt, $value, $arch, $machine_type);
4992             } elsif ($opt =~ m/^memory$/) { #dimms
4993                 die "skip\n" if !$hotplug_features->{memory};
4994                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4995             } elsif ($opt eq 'cpuunits') {
4996                 my $new_cpuunits = get_cpuunits({ $opt => $conf->{pending}->{$opt} }); # to clamp
4997                 $cgroup->change_cpu_shares($new_cpuunits, 1024);
4998             } elsif ($opt eq 'cpulimit') {
4999                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5000                 $cgroup->change_cpu_quota($cpulimit, 100000);
5001             } elsif ($opt eq 'agent') {
5002                 vmconfig_update_agent($conf, $opt, $value);
5003             } else {
5004                 die "skip\n";  # skip non-hot-pluggable options
5005             }
5006         };
5007         if (my $err = $@) {
5008             &$add_error($opt, $err) if $err ne "skip\n";
5009         } else {
5010             $conf->{$opt} = $value;
5011             delete $conf->{pending}->{$opt};
5012         }
5013     }
5014
5015     PVE::QemuConfig->write_config($vmid, $conf);
5016 }
5017
5018 sub try_deallocate_drive {
5019     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5020
5021     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5022         my $volid = $drive->{file};
5023         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5024             my $sid = PVE::Storage::parse_volume_id($volid);
5025             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5026
5027             # check if the disk is really unused
5028             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5029                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
5030             PVE::Storage::vdisk_free($storecfg, $volid);
5031             return 1;
5032         } else {
5033             # If vm is not owner of this disk remove from config
5034             return 1;
5035         }
5036     }
5037
5038     return;
5039 }
5040
5041 sub vmconfig_delete_or_detach_drive {
5042     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5043
5044     my $drive = parse_drive($opt, $conf->{$opt});
5045
5046     my $rpcenv = PVE::RPCEnvironment::get();
5047     my $authuser = $rpcenv->get_user();
5048
5049     if ($force) {
5050         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5051         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5052     } else {
5053         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5054     }
5055 }
5056
5057
5058
5059 sub vmconfig_apply_pending {
5060     my ($vmid, $conf, $storecfg, $errors) = @_;
5061
5062     return if !scalar(keys %{$conf->{pending}});
5063
5064     my $add_apply_error = sub {
5065         my ($opt, $msg) = @_;
5066         my $err_msg = "unable to apply pending change $opt : $msg";
5067         $errors->{$opt} = $err_msg;
5068         warn $err_msg;
5069     };
5070
5071     # cold plug
5072
5073     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5074     foreach my $opt (sort keys %$pending_delete_hash) {
5075         my $force = $pending_delete_hash->{$opt}->{force};
5076         eval {
5077             if ($opt =~ m/^unused/) {
5078                 die "internal error";
5079             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5080                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5081             }
5082         };
5083         if (my $err = $@) {
5084             $add_apply_error->($opt, $err);
5085         } else {
5086             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5087             delete $conf->{$opt};
5088         }
5089     }
5090
5091     PVE::QemuConfig->cleanup_pending($conf);
5092
5093     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5094         next if $opt eq 'delete'; # just to be sure
5095         eval {
5096             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5097                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5098             }
5099         };
5100         if (my $err = $@) {
5101             $add_apply_error->($opt, $err);
5102         } else {
5103             $conf->{$opt} = delete $conf->{pending}->{$opt};
5104         }
5105     }
5106
5107     # write all changes at once to avoid unnecessary i/o
5108     PVE::QemuConfig->write_config($vmid, $conf);
5109 }
5110
5111 sub vmconfig_update_net {
5112     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5113
5114     my $newnet = parse_net($value);
5115
5116     if ($conf->{$opt}) {
5117         my $oldnet = parse_net($conf->{$opt});
5118
5119         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5120             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5121             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5122             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5123
5124             # for non online change, we try to hot-unplug
5125             die "skip\n" if !$hotplug;
5126             vm_deviceunplug($vmid, $conf, $opt);
5127         } else {
5128
5129             die "internal error" if $opt !~ m/net(\d+)/;
5130             my $iface = "tap${vmid}i$1";
5131
5132             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5133                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5134                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5135                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5136                 PVE::Network::tap_unplug($iface);
5137
5138                 if ($have_sdn) {
5139                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5140                 } else {
5141                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5142                 }
5143             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5144                 # Rate can be applied on its own but any change above needs to
5145                 # include the rate in tap_plug since OVS resets everything.
5146                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5147             }
5148
5149             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5150                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5151             }
5152
5153             return 1;
5154         }
5155     }
5156
5157     if ($hotplug) {
5158         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5159     } else {
5160         die "skip\n";
5161     }
5162 }
5163
5164 sub vmconfig_update_agent {
5165     my ($conf, $opt, $value) = @_;
5166
5167     die "skip\n" if !$conf->{$opt};
5168
5169     my $hotplug_options = { fstrim_cloned_disks => 1 };
5170
5171     my $old_agent = parse_guest_agent($conf);
5172     my $agent = parse_guest_agent({$opt => $value});
5173
5174     for my $option (keys %$agent) { # added/changed options
5175         next if defined($hotplug_options->{$option});
5176         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5177     }
5178
5179     for my $option (keys %$old_agent) { # removed options
5180         next if defined($hotplug_options->{$option});
5181         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5182     }
5183
5184     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5185 }
5186
5187 sub vmconfig_update_disk {
5188     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5189
5190     my $drive = parse_drive($opt, $value);
5191
5192     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5193         my $media = $drive->{media} || 'disk';
5194         my $oldmedia = $old_drive->{media} || 'disk';
5195         die "unable to change media type\n" if $media ne $oldmedia;
5196
5197         if (!drive_is_cdrom($old_drive)) {
5198
5199             if ($drive->{file} ne $old_drive->{file}) {
5200
5201                 die "skip\n" if !$hotplug;
5202
5203                 # unplug and register as unused
5204                 vm_deviceunplug($vmid, $conf, $opt);
5205                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5206
5207             } else {
5208                 # update existing disk
5209
5210                 # skip non hotpluggable value
5211                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5212                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5213                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5214                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5215                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5216                     die "skip\n";
5217                 }
5218
5219                 # apply throttle
5220                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5221                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5222                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5223                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5224                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5225                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5226                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5227                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5228                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5229                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5230                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5231                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5232                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5233                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5234                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5235                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5236                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5237                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5238
5239                     qemu_block_set_io_throttle(
5240                         $vmid,"drive-$opt",
5241                         ($drive->{mbps} || 0)*1024*1024,
5242                         ($drive->{mbps_rd} || 0)*1024*1024,
5243                         ($drive->{mbps_wr} || 0)*1024*1024,
5244                          $drive->{iops} || 0,
5245                          $drive->{iops_rd} || 0,
5246                          $drive->{iops_wr} || 0,
5247                         ($drive->{mbps_max} || 0)*1024*1024,
5248                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5249                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5250                          $drive->{iops_max} || 0,
5251                          $drive->{iops_rd_max} || 0,
5252                          $drive->{iops_wr_max} || 0,
5253                          $drive->{bps_max_length} || 1,
5254                          $drive->{bps_rd_max_length} || 1,
5255                          $drive->{bps_wr_max_length} || 1,
5256                          $drive->{iops_max_length} || 1,
5257                          $drive->{iops_rd_max_length} || 1,
5258                          $drive->{iops_wr_max_length} || 1,
5259                     );
5260
5261                 }
5262
5263                 return 1;
5264             }
5265
5266         } else { # cdrom
5267
5268             if ($drive->{file} eq 'none') {
5269                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5270                 if (drive_is_cloudinit($old_drive)) {
5271                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5272                 }
5273             } else {
5274                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5275
5276                 # force eject if locked
5277                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5278
5279                 if ($path) {
5280                     mon_cmd($vmid, "blockdev-change-medium",
5281                         id => "$opt", filename => "$path");
5282                 }
5283             }
5284
5285             return 1;
5286         }
5287     }
5288
5289     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5290     # hotplug new disks
5291     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5292     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5293 }
5294
5295 # called in locked context by incoming migration
5296 sub vm_migrate_get_nbd_disks {
5297     my ($storecfg, $conf, $replicated_volumes) = @_;
5298
5299     my $local_volumes = {};
5300     PVE::QemuConfig->foreach_volume($conf, sub {
5301         my ($ds, $drive) = @_;
5302
5303         return if drive_is_cdrom($drive);
5304         return if $ds eq 'tpmstate0';
5305
5306         my $volid = $drive->{file};
5307
5308         return if !$volid;
5309
5310         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5311
5312         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5313         return if $scfg->{shared};
5314
5315         # replicated disks re-use existing state via bitmap
5316         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5317         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5318     });
5319     return $local_volumes;
5320 }
5321
5322 # called in locked context by incoming migration
5323 sub vm_migrate_alloc_nbd_disks {
5324     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5325
5326     my $nbd = {};
5327     foreach my $opt (sort keys %$source_volumes) {
5328         my ($volid, $storeid, $volname, $drive, $use_existing, $format) = @{$source_volumes->{$opt}};
5329
5330         if ($use_existing) {
5331             $nbd->{$opt}->{drivestr} = print_drive($drive);
5332             $nbd->{$opt}->{volid} = $volid;
5333             $nbd->{$opt}->{replicated} = 1;
5334             next;
5335         }
5336
5337         # storage mapping + volname = regular migration
5338         # storage mapping + format = remote migration
5339         # order of precedence, filtered by whether storage supports it:
5340         # 1. explicit requested format
5341         # 2. format of current volume
5342         # 3. default format of storage
5343         if (!$storagemap->{identity}) {
5344             $storeid = PVE::JSONSchema::map_id($storagemap, $storeid);
5345             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5346             if (!$format || !grep { $format eq $_ } @$validFormats) {
5347                 if ($volname) {
5348                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5349                     my $fileFormat = qemu_img_format($scfg, $volname);
5350                     $format = $fileFormat
5351                         if grep { $fileFormat eq $_ } @$validFormats;
5352                 }
5353                 $format //= $defFormat;
5354             }
5355         } else {
5356             # can't happen for remote migration, so $volname is always defined
5357             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5358             $format = qemu_img_format($scfg, $volname);
5359         }
5360
5361         my $size = $drive->{size} / 1024;
5362         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5363         my $newdrive = $drive;
5364         $newdrive->{format} = $format;
5365         $newdrive->{file} = $newvolid;
5366         my $drivestr = print_drive($newdrive);
5367         $nbd->{$opt}->{drivestr} = $drivestr;
5368         $nbd->{$opt}->{volid} = $newvolid;
5369     }
5370
5371     return $nbd;
5372 }
5373
5374 # see vm_start_nolock for parameters, additionally:
5375 # migrate_opts:
5376 #   storagemap = parsed storage map for allocating NBD disks
5377 sub vm_start {
5378     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5379
5380     return PVE::QemuConfig->lock_config($vmid, sub {
5381         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5382
5383         die "you can't start a vm if it's a template\n"
5384             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5385
5386         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5387         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5388
5389         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5390
5391         if ($has_backup_lock && $running) {
5392             # a backup is currently running, attempt to start the guest in the
5393             # existing QEMU instance
5394             return vm_resume($vmid);
5395         }
5396
5397         PVE::QemuConfig->check_lock($conf)
5398             if !($params->{skiplock} || $has_suspended_lock);
5399
5400         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5401
5402         die "VM $vmid already running\n" if $running;
5403
5404         if (my $storagemap = $migrate_opts->{storagemap}) {
5405             my $replicated = $migrate_opts->{replicated_volumes};
5406             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5407             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5408
5409             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5410                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5411             }
5412         }
5413
5414         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5415     });
5416 }
5417
5418
5419 # params:
5420 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5421 #   skiplock => 0/1, skip checking for config lock
5422 #   skiptemplate => 0/1, skip checking whether VM is template
5423 #   forcemachine => to force Qemu machine (rollback/migration)
5424 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5425 #   timeout => in seconds
5426 #   paused => start VM in paused state (backup)
5427 #   resume => resume from hibernation
5428 #   pbs-backing => {
5429 #      sata0 => {
5430 #         repository
5431 #         snapshot
5432 #         keyfile
5433 #         archive
5434 #      },
5435 #      virtio2 => ...
5436 #   }
5437 # migrate_opts:
5438 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5439 #   migratedfrom => source node
5440 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5441 #   network => CIDR of migration network
5442 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5443 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5444 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5445 #   offline_volumes => new volids of offline migrated disks like tpmstate and cloudinit, not yet
5446 #       contained in config
5447 sub vm_start_nolock {
5448     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5449
5450     my $statefile = $params->{statefile};
5451     my $resume = $params->{resume};
5452
5453     my $migratedfrom = $migrate_opts->{migratedfrom};
5454     my $migration_type = $migrate_opts->{type};
5455
5456     my $res = {};
5457
5458     # clean up leftover reboot request files
5459     eval { clear_reboot_request($vmid); };
5460     warn $@ if $@;
5461
5462     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5463         vmconfig_apply_pending($vmid, $conf, $storecfg);
5464         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5465     }
5466
5467     # don't regenerate the ISO if the VM is started as part of a live migration
5468     # this way we can reuse the old ISO with the correct config
5469     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5470
5471     # override offline migrated volumes, conf is out of date still
5472     if (my $offline_volumes = $migrate_opts->{offline_volumes}) {
5473         for my $key (sort keys $offline_volumes->%*) {
5474             my $parsed = parse_drive($key, $conf->{$key});
5475             $parsed->{file} = $offline_volumes->{$key};
5476             $conf->{$key} = print_drive($parsed);
5477         }
5478     }
5479
5480     my $defaults = load_defaults();
5481
5482     # set environment variable useful inside network script
5483     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5484
5485     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5486
5487     my $forcemachine = $params->{forcemachine};
5488     my $forcecpu = $params->{forcecpu};
5489     if ($resume) {
5490         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5491         $forcemachine = $conf->{runningmachine};
5492         $forcecpu = $conf->{runningcpu};
5493         print "Resuming suspended VM\n";
5494     }
5495
5496     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5497         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5498
5499     my $migration_ip;
5500     my $get_migration_ip = sub {
5501         my ($nodename) = @_;
5502
5503         return $migration_ip if defined($migration_ip);
5504
5505         my $cidr = $migrate_opts->{network};
5506
5507         if (!defined($cidr)) {
5508             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5509             $cidr = $dc_conf->{migration}->{network};
5510         }
5511
5512         if (defined($cidr)) {
5513             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5514
5515             die "could not get IP: no address configured on local " .
5516                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5517
5518             die "could not get IP: multiple addresses configured on local " .
5519                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5520
5521             $migration_ip = @$ips[0];
5522         }
5523
5524         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5525             if !defined($migration_ip);
5526
5527         return $migration_ip;
5528     };
5529
5530     my $migrate_uri;
5531     if ($statefile) {
5532         if ($statefile eq 'tcp') {
5533             my $localip = "localhost";
5534             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5535             my $nodename = nodename();
5536
5537             if (!defined($migration_type)) {
5538                 if (defined($datacenterconf->{migration}->{type})) {
5539                     $migration_type = $datacenterconf->{migration}->{type};
5540                 } else {
5541                     $migration_type = 'secure';
5542                 }
5543             }
5544
5545             if ($migration_type eq 'insecure') {
5546                 $localip = $get_migration_ip->($nodename);
5547                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5548             }
5549
5550             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5551             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5552             $migrate_uri = "tcp:${localip}:${migrate_port}";
5553             push @$cmd, '-incoming', $migrate_uri;
5554             push @$cmd, '-S';
5555
5556         } elsif ($statefile eq 'unix') {
5557             # should be default for secure migrations as a ssh TCP forward
5558             # tunnel is not deterministic reliable ready and fails regurarly
5559             # to set up in time, so use UNIX socket forwards
5560             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5561             unlink $socket_addr;
5562
5563             $migrate_uri = "unix:$socket_addr";
5564
5565             push @$cmd, '-incoming', $migrate_uri;
5566             push @$cmd, '-S';
5567
5568         } elsif (-e $statefile) {
5569             push @$cmd, '-loadstate', $statefile;
5570         } else {
5571             my $statepath = PVE::Storage::path($storecfg, $statefile);
5572             push @$vollist, $statefile;
5573             push @$cmd, '-loadstate', $statepath;
5574         }
5575     } elsif ($params->{paused}) {
5576         push @$cmd, '-S';
5577     }
5578
5579     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5580
5581     my $pci_devices = {}; # host pci devices
5582     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5583         my $dev = $conf->{"hostpci$i"} or next;
5584         $pci_devices->{$i} = parse_hostpci($dev);
5585     }
5586
5587     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5588     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5589
5590     # map to a flat list of pci ids
5591     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5592
5593     # reserve all PCI IDs before actually doing anything with them
5594     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5595
5596     eval {
5597         my $uuid;
5598         for my $id (sort keys %$pci_devices) {
5599             my $d = $pci_devices->{$id};
5600             for my $dev ($d->{pciid}->@*) {
5601                 my $info = PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5602
5603                 # nvidia grid needs the uuid of the mdev as qemu parameter
5604                 if ($d->{mdev} && !defined($uuid) && $info->{vendor} eq '10de') {
5605                     $uuid = PVE::QemuServer::PCI::generate_mdev_uuid($vmid, $id);
5606                 }
5607             }
5608         }
5609         push @$cmd, '-uuid', $uuid if defined($uuid);
5610     };
5611     if (my $err = $@) {
5612         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5613         warn $@ if $@;
5614         die $err;
5615     }
5616
5617     PVE::Storage::activate_volumes($storecfg, $vollist);
5618
5619     eval {
5620         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5621     };
5622     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5623     # timeout should be more than enough here...
5624     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 20);
5625
5626     my $cpuunits = get_cpuunits($conf);
5627
5628     my %run_params = (
5629         timeout => $statefile ? undef : $start_timeout,
5630         umask => 0077,
5631         noerr => 1,
5632     );
5633
5634     # when migrating, prefix QEMU output so other side can pick up any
5635     # errors that might occur and show the user
5636     if ($migratedfrom) {
5637         $run_params{quiet} = 1;
5638         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5639     }
5640
5641     my %systemd_properties = (
5642         Slice => 'qemu.slice',
5643         KillMode => 'process',
5644         SendSIGKILL => 0,
5645         TimeoutStopUSec => ULONG_MAX, # infinity
5646     );
5647
5648     if (PVE::CGroup::cgroup_mode() == 2) {
5649         $systemd_properties{CPUWeight} = $cpuunits;
5650     } else {
5651         $systemd_properties{CPUShares} = $cpuunits;
5652     }
5653
5654     if (my $cpulimit = $conf->{cpulimit}) {
5655         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5656     }
5657     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5658
5659     my $run_qemu = sub {
5660         PVE::Tools::run_fork sub {
5661             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5662
5663             my $tpmpid;
5664             if (my $tpm = $conf->{tpmstate0}) {
5665                 # start the TPM emulator so QEMU can connect on start
5666                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5667             }
5668
5669             my $exitcode = run_command($cmd, %run_params);
5670             if ($exitcode) {
5671                 if ($tpmpid) {
5672                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5673                     kill 'TERM', $tpmpid;
5674                 }
5675                 die "QEMU exited with code $exitcode\n";
5676             }
5677         };
5678     };
5679
5680     if ($conf->{hugepages}) {
5681
5682         my $code = sub {
5683             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5684             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5685
5686             PVE::QemuServer::Memory::hugepages_mount();
5687             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5688
5689             eval { $run_qemu->() };
5690             if (my $err = $@) {
5691                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5692                     if !$conf->{keephugepages};
5693                 die $err;
5694             }
5695
5696             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5697                 if !$conf->{keephugepages};
5698         };
5699         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5700
5701     } else {
5702         eval { $run_qemu->() };
5703     }
5704
5705     if (my $err = $@) {
5706         # deactivate volumes if start fails
5707         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5708         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5709
5710         die "start failed: $err";
5711     }
5712
5713     # re-reserve all PCI IDs now that we can know the actual VM PID
5714     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5715     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5716     warn $@ if $@;
5717
5718     print "migration listens on $migrate_uri\n" if $migrate_uri;
5719     $res->{migrate_uri} = $migrate_uri;
5720
5721     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5722         eval { mon_cmd($vmid, "cont"); };
5723         warn $@ if $@;
5724     }
5725
5726     #start nbd server for storage migration
5727     if (my $nbd = $migrate_opts->{nbd}) {
5728         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5729
5730         my $migrate_storage_uri;
5731         # nbd_protocol_version > 0 for unix socket support
5732         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5733             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5734             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5735             $migrate_storage_uri = "nbd:unix:$socket_path";
5736         } else {
5737             my $nodename = nodename();
5738             my $localip = $get_migration_ip->($nodename);
5739             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5740             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5741
5742             mon_cmd($vmid, "nbd-server-start", addr => {
5743                 type => 'inet',
5744                 data => {
5745                     host => "${localip}",
5746                     port => "${storage_migrate_port}",
5747                 },
5748             });
5749             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5750             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5751         }
5752
5753         $res->{migrate_storage_uri} = $migrate_storage_uri;
5754
5755         foreach my $opt (sort keys %$nbd) {
5756             my $drivestr = $nbd->{$opt}->{drivestr};
5757             my $volid = $nbd->{$opt}->{volid};
5758             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5759             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5760             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5761             print "re-using replicated volume: $opt - $volid\n"
5762                 if $nbd->{$opt}->{replicated};
5763
5764             $res->{drives}->{$opt} = $nbd->{$opt};
5765             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5766         }
5767     }
5768
5769     if ($migratedfrom) {
5770         eval {
5771             set_migration_caps($vmid);
5772         };
5773         warn $@ if $@;
5774
5775         if ($spice_port) {
5776             print "spice listens on port $spice_port\n";
5777             $res->{spice_port} = $spice_port;
5778             if ($migrate_opts->{spice_ticket}) {
5779                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5780                     $migrate_opts->{spice_ticket});
5781                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5782             }
5783         }
5784
5785     } else {
5786         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5787             if !$statefile && $conf->{balloon};
5788
5789         foreach my $opt (keys %$conf) {
5790             next if $opt !~  m/^net\d+$/;
5791             my $nicconf = parse_net($conf->{$opt});
5792             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5793         }
5794     }
5795
5796     mon_cmd($vmid, 'qom-set',
5797                 path => "machine/peripheral/balloon0",
5798                 property => "guest-stats-polling-interval",
5799                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5800
5801     if ($resume) {
5802         print "Resumed VM, removing state\n";
5803         if (my $vmstate = $conf->{vmstate}) {
5804             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5805             PVE::Storage::vdisk_free($storecfg, $vmstate);
5806         }
5807         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5808         PVE::QemuConfig->write_config($vmid, $conf);
5809     }
5810
5811     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5812
5813     return $res;
5814 }
5815
5816 sub vm_commandline {
5817     my ($storecfg, $vmid, $snapname) = @_;
5818
5819     my $conf = PVE::QemuConfig->load_config($vmid);
5820
5821     my ($forcemachine, $forcecpu);
5822     if ($snapname) {
5823         my $snapshot = $conf->{snapshots}->{$snapname};
5824         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5825
5826         # check for machine or CPU overrides in snapshot
5827         $forcemachine = $snapshot->{runningmachine};
5828         $forcecpu = $snapshot->{runningcpu};
5829
5830         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5831
5832         $conf = $snapshot;
5833     }
5834
5835     my $defaults = load_defaults();
5836
5837     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5838
5839     return PVE::Tools::cmd2string($cmd);
5840 }
5841
5842 sub vm_reset {
5843     my ($vmid, $skiplock) = @_;
5844
5845     PVE::QemuConfig->lock_config($vmid, sub {
5846
5847         my $conf = PVE::QemuConfig->load_config($vmid);
5848
5849         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5850
5851         mon_cmd($vmid, "system_reset");
5852     });
5853 }
5854
5855 sub get_vm_volumes {
5856     my ($conf) = @_;
5857
5858     my $vollist = [];
5859     foreach_volid($conf, sub {
5860         my ($volid, $attr) = @_;
5861
5862         return if $volid =~ m|^/|;
5863
5864         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5865         return if !$sid;
5866
5867         push @$vollist, $volid;
5868     });
5869
5870     return $vollist;
5871 }
5872
5873 sub vm_stop_cleanup {
5874     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5875
5876     eval {
5877
5878         if (!$keepActive) {
5879             my $vollist = get_vm_volumes($conf);
5880             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5881
5882             if (my $tpmdrive = $conf->{tpmstate0}) {
5883                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5884                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5885                 if ($storeid) {
5886                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5887                 }
5888             }
5889         }
5890
5891         foreach my $ext (qw(mon qmp pid vnc qga)) {
5892             unlink "/var/run/qemu-server/${vmid}.$ext";
5893         }
5894
5895         if ($conf->{ivshmem}) {
5896             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5897             # just delete it for now, VMs which have this already open do not
5898             # are affected, but new VMs will get a separated one. If this
5899             # becomes an issue we either add some sort of ref-counting or just
5900             # add a "don't delete on stop" flag to the ivshmem format.
5901             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5902         }
5903
5904         my $ids = [];
5905         foreach my $key (keys %$conf) {
5906             next if $key !~ m/^hostpci(\d+)$/;
5907             my $hostpciindex = $1;
5908             my $d = parse_hostpci($conf->{$key});
5909             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5910
5911             foreach my $pci (@{$d->{pciid}}) {
5912                 my $pciid = $pci->{id};
5913                 push @$ids, $pci->{id};
5914                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5915             }
5916         }
5917         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5918
5919         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5920     };
5921     warn $@ if $@; # avoid errors - just warn
5922 }
5923
5924 # call only in locked context
5925 sub _do_vm_stop {
5926     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5927
5928     my $pid = check_running($vmid, $nocheck);
5929     return if !$pid;
5930
5931     my $conf;
5932     if (!$nocheck) {
5933         $conf = PVE::QemuConfig->load_config($vmid);
5934         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5935         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5936             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5937             $timeout = $opts->{down} if $opts->{down};
5938         }
5939         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5940     }
5941
5942     eval {
5943         if ($shutdown) {
5944             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5945                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5946             } else {
5947                 mon_cmd($vmid, "system_powerdown");
5948             }
5949         } else {
5950             mon_cmd($vmid, "quit");
5951         }
5952     };
5953     my $err = $@;
5954
5955     if (!$err) {
5956         $timeout = 60 if !defined($timeout);
5957
5958         my $count = 0;
5959         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5960             $count++;
5961             sleep 1;
5962         }
5963
5964         if ($count >= $timeout) {
5965             if ($force) {
5966                 warn "VM still running - terminating now with SIGTERM\n";
5967                 kill 15, $pid;
5968             } else {
5969                 die "VM quit/powerdown failed - got timeout\n";
5970             }
5971         } else {
5972             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5973             return;
5974         }
5975     } else {
5976        if (!check_running($vmid, $nocheck)) {
5977             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5978             return;
5979        }
5980        if ($force) {
5981             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5982             kill 15, $pid;
5983         } else {
5984             die "VM quit/powerdown failed\n";
5985         }
5986     }
5987
5988     # wait again
5989     $timeout = 10;
5990
5991     my $count = 0;
5992     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5993         $count++;
5994         sleep 1;
5995     }
5996
5997     if ($count >= $timeout) {
5998         warn "VM still running - terminating now with SIGKILL\n";
5999         kill 9, $pid;
6000         sleep 1;
6001     }
6002
6003     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
6004 }
6005
6006 # Note: use $nocheck to skip tests if VM configuration file exists.
6007 # We need that when migration VMs to other nodes (files already moved)
6008 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
6009 sub vm_stop {
6010     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
6011
6012     $force = 1 if !defined($force) && !$shutdown;
6013
6014     if ($migratedfrom){
6015         my $pid = check_running($vmid, $nocheck, $migratedfrom);
6016         kill 15, $pid if $pid;
6017         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
6018         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
6019         return;
6020     }
6021
6022     PVE::QemuConfig->lock_config($vmid, sub {
6023         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
6024    });
6025 }
6026
6027 sub vm_reboot {
6028     my ($vmid, $timeout) = @_;
6029
6030     PVE::QemuConfig->lock_config($vmid, sub {
6031         eval {
6032
6033             # only reboot if running, as qmeventd starts it again on a stop event
6034             return if !check_running($vmid);
6035
6036             create_reboot_request($vmid);
6037
6038             my $storecfg = PVE::Storage::config();
6039             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
6040
6041         };
6042         if (my $err = $@) {
6043             # avoid that the next normal shutdown will be confused for a reboot
6044             clear_reboot_request($vmid);
6045             die $err;
6046         }
6047    });
6048 }
6049
6050 # note: if using the statestorage parameter, the caller has to check privileges
6051 sub vm_suspend {
6052     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
6053
6054     my $conf;
6055     my $path;
6056     my $storecfg;
6057     my $vmstate;
6058
6059     PVE::QemuConfig->lock_config($vmid, sub {
6060
6061         $conf = PVE::QemuConfig->load_config($vmid);
6062
6063         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
6064         PVE::QemuConfig->check_lock($conf)
6065             if !($skiplock || $is_backing_up);
6066
6067         die "cannot suspend to disk during backup\n"
6068             if $is_backing_up && $includestate;
6069
6070         if ($includestate) {
6071             $conf->{lock} = 'suspending';
6072             my $date = strftime("%Y-%m-%d", localtime(time()));
6073             $storecfg = PVE::Storage::config();
6074             if (!$statestorage) {
6075                 $statestorage = find_vmstate_storage($conf, $storecfg);
6076                 # check permissions for the storage
6077                 my $rpcenv = PVE::RPCEnvironment::get();
6078                 if ($rpcenv->{type} ne 'cli') {
6079                     my $authuser = $rpcenv->get_user();
6080                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6081                 }
6082             }
6083
6084
6085             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6086                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6087             $path = PVE::Storage::path($storecfg, $vmstate);
6088             PVE::QemuConfig->write_config($vmid, $conf);
6089         } else {
6090             mon_cmd($vmid, "stop");
6091         }
6092     });
6093
6094     if ($includestate) {
6095         # save vm state
6096         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6097
6098         eval {
6099             set_migration_caps($vmid, 1);
6100             mon_cmd($vmid, "savevm-start", statefile => $path);
6101             for(;;) {
6102                 my $state = mon_cmd($vmid, "query-savevm");
6103                 if (!$state->{status}) {
6104                     die "savevm not active\n";
6105                 } elsif ($state->{status} eq 'active') {
6106                     sleep(1);
6107                     next;
6108                 } elsif ($state->{status} eq 'completed') {
6109                     print "State saved, quitting\n";
6110                     last;
6111                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6112                     die "query-savevm failed with error '$state->{error}'\n"
6113                 } else {
6114                     die "query-savevm returned status '$state->{status}'\n";
6115                 }
6116             }
6117         };
6118         my $err = $@;
6119
6120         PVE::QemuConfig->lock_config($vmid, sub {
6121             $conf = PVE::QemuConfig->load_config($vmid);
6122             if ($err) {
6123                 # cleanup, but leave suspending lock, to indicate something went wrong
6124                 eval {
6125                     mon_cmd($vmid, "savevm-end");
6126                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6127                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6128                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6129                     PVE::QemuConfig->write_config($vmid, $conf);
6130                 };
6131                 warn $@ if $@;
6132                 die $err;
6133             }
6134
6135             die "lock changed unexpectedly\n"
6136                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6137
6138             mon_cmd($vmid, "quit");
6139             $conf->{lock} = 'suspended';
6140             PVE::QemuConfig->write_config($vmid, $conf);
6141         });
6142     }
6143 }
6144
6145 sub vm_resume {
6146     my ($vmid, $skiplock, $nocheck) = @_;
6147
6148     PVE::QemuConfig->lock_config($vmid, sub {
6149         my $res = mon_cmd($vmid, 'query-status');
6150         my $resume_cmd = 'cont';
6151         my $reset = 0;
6152
6153         if ($res->{status}) {
6154             return if $res->{status} eq 'running'; # job done, go home
6155             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6156             $reset = 1 if $res->{status} eq 'shutdown';
6157         }
6158
6159         if (!$nocheck) {
6160
6161             my $conf = PVE::QemuConfig->load_config($vmid);
6162
6163             PVE::QemuConfig->check_lock($conf)
6164                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6165         }
6166
6167         if ($reset) {
6168             # required if a VM shuts down during a backup and we get a resume
6169             # request before the backup finishes for example
6170             mon_cmd($vmid, "system_reset");
6171         }
6172         mon_cmd($vmid, $resume_cmd);
6173     });
6174 }
6175
6176 sub vm_sendkey {
6177     my ($vmid, $skiplock, $key) = @_;
6178
6179     PVE::QemuConfig->lock_config($vmid, sub {
6180
6181         my $conf = PVE::QemuConfig->load_config($vmid);
6182
6183         # there is no qmp command, so we use the human monitor command
6184         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6185         die $res if $res ne '';
6186     });
6187 }
6188
6189 # vzdump restore implementaion
6190
6191 sub tar_archive_read_firstfile {
6192     my $archive = shift;
6193
6194     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6195
6196     # try to detect archive type first
6197     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6198         die "unable to open file '$archive'\n";
6199     my $firstfile = <$fh>;
6200     kill 15, $pid;
6201     close $fh;
6202
6203     die "ERROR: archive contaions no data\n" if !$firstfile;
6204     chomp $firstfile;
6205
6206     return $firstfile;
6207 }
6208
6209 sub tar_restore_cleanup {
6210     my ($storecfg, $statfile) = @_;
6211
6212     print STDERR "starting cleanup\n";
6213
6214     if (my $fd = IO::File->new($statfile, "r")) {
6215         while (defined(my $line = <$fd>)) {
6216             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6217                 my $volid = $2;
6218                 eval {
6219                     if ($volid =~ m|^/|) {
6220                         unlink $volid || die 'unlink failed\n';
6221                     } else {
6222                         PVE::Storage::vdisk_free($storecfg, $volid);
6223                     }
6224                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6225                 };
6226                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6227             } else {
6228                 print STDERR "unable to parse line in statfile - $line";
6229             }
6230         }
6231         $fd->close();
6232     }
6233 }
6234
6235 sub restore_file_archive {
6236     my ($archive, $vmid, $user, $opts) = @_;
6237
6238     return restore_vma_archive($archive, $vmid, $user, $opts)
6239         if $archive eq '-';
6240
6241     my $info = PVE::Storage::archive_info($archive);
6242     my $format = $opts->{format} // $info->{format};
6243     my $comp = $info->{compression};
6244
6245     # try to detect archive format
6246     if ($format eq 'tar') {
6247         return restore_tar_archive($archive, $vmid, $user, $opts);
6248     } else {
6249         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6250     }
6251 }
6252
6253 # hepler to remove disks that will not be used after restore
6254 my $restore_cleanup_oldconf = sub {
6255     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6256
6257     my $kept_disks = {};
6258
6259     PVE::QemuConfig->foreach_volume($oldconf, sub {
6260         my ($ds, $drive) = @_;
6261
6262         return if drive_is_cdrom($drive, 1);
6263
6264         my $volid = $drive->{file};
6265         return if !$volid || $volid =~ m|^/|;
6266
6267         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6268         return if !$path || !$owner || ($owner != $vmid);
6269
6270         # Note: only delete disk we want to restore
6271         # other volumes will become unused
6272         if ($virtdev_hash->{$ds}) {
6273             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6274             if (my $err = $@) {
6275                 warn $err;
6276             }
6277         } else {
6278             $kept_disks->{$volid} = 1;
6279         }
6280     });
6281
6282     # after the restore we have no snapshots anymore
6283     for my $snapname (keys $oldconf->{snapshots}->%*) {
6284         my $snap = $oldconf->{snapshots}->{$snapname};
6285         if ($snap->{vmstate}) {
6286             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6287             if (my $err = $@) {
6288                 warn $err;
6289             }
6290         }
6291
6292         for my $volid (keys $kept_disks->%*) {
6293             eval { PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snapname); };
6294             warn $@ if $@;
6295         }
6296     }
6297 };
6298
6299 # Helper to parse vzdump backup device hints
6300 #
6301 # $rpcenv: Environment, used to ckeck storage permissions
6302 # $user: User ID, to check storage permissions
6303 # $storecfg: Storage configuration
6304 # $fh: the file handle for reading the configuration
6305 # $devinfo: should contain device sizes for all backu-up'ed devices
6306 # $options: backup options (pool, default storage)
6307 #
6308 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6309 my $parse_backup_hints = sub {
6310     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6311
6312     my $check_storage = sub { # assert if an image can be allocate
6313         my ($storeid, $scfg) = @_;
6314         die "Content type 'images' is not available on storage '$storeid'\n"
6315             if !$scfg->{content}->{images};
6316         $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace'])
6317             if $user ne 'root@pam';
6318     };
6319
6320     my $virtdev_hash = {};
6321     while (defined(my $line = <$fh>)) {
6322         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6323             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6324             die "archive does not contain data for drive '$virtdev'\n"
6325                 if !$devinfo->{$devname};
6326
6327             if (defined($options->{storage})) {
6328                 $storeid = $options->{storage} || 'local';
6329             } elsif (!$storeid) {
6330                 $storeid = 'local';
6331             }
6332             $format = 'raw' if !$format;
6333             $devinfo->{$devname}->{devname} = $devname;
6334             $devinfo->{$devname}->{virtdev} = $virtdev;
6335             $devinfo->{$devname}->{format} = $format;
6336             $devinfo->{$devname}->{storeid} = $storeid;
6337
6338             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6339             $check_storage->($storeid, $scfg); # permission and content type check
6340
6341             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6342         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6343             my $virtdev = $1;
6344             my $drive = parse_drive($virtdev, $2);
6345
6346             if (drive_is_cloudinit($drive)) {
6347                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6348                 $storeid = $options->{storage} if defined ($options->{storage});
6349                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6350                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6351
6352                 $check_storage->($storeid, $scfg); # permission and content type check
6353
6354                 $virtdev_hash->{$virtdev} = {
6355                     format => $format,
6356                     storeid => $storeid,
6357                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6358                     is_cloudinit => 1,
6359                 };
6360             }
6361         }
6362     }
6363
6364     return $virtdev_hash;
6365 };
6366
6367 # Helper to allocate and activate all volumes required for a restore
6368 #
6369 # $storecfg: Storage configuration
6370 # $virtdev_hash: as returned by parse_backup_hints()
6371 #
6372 # Returns: { $virtdev => $volid }
6373 my $restore_allocate_devices = sub {
6374     my ($storecfg, $virtdev_hash, $vmid) = @_;
6375
6376     my $map = {};
6377     foreach my $virtdev (sort keys %$virtdev_hash) {
6378         my $d = $virtdev_hash->{$virtdev};
6379         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6380         my $storeid = $d->{storeid};
6381         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6382
6383         # test if requested format is supported
6384         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6385         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6386         $d->{format} = $defFormat if !$supported;
6387
6388         my $name;
6389         if ($d->{is_cloudinit}) {
6390             $name = "vm-$vmid-cloudinit";
6391             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6392             if ($scfg->{path}) {
6393                 $name .= ".$d->{format}";
6394             }
6395         }
6396
6397         my $volid = PVE::Storage::vdisk_alloc(
6398             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6399
6400         print STDERR "new volume ID is '$volid'\n";
6401         $d->{volid} = $volid;
6402
6403         PVE::Storage::activate_volumes($storecfg, [$volid]);
6404
6405         $map->{$virtdev} = $volid;
6406     }
6407
6408     return $map;
6409 };
6410
6411 sub restore_update_config_line {
6412     my ($cookie, $map, $line, $unique) = @_;
6413
6414     return '' if $line =~ m/^\#qmdump\#/;
6415     return '' if $line =~ m/^\#vzdump\#/;
6416     return '' if $line =~ m/^lock:/;
6417     return '' if $line =~ m/^unused\d+:/;
6418     return '' if $line =~ m/^parent:/;
6419
6420     my $res = '';
6421
6422     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6423     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6424         # try to convert old 1.X settings
6425         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6426         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6427             my ($model, $macaddr) = split(/\=/, $devconfig);
6428             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6429             my $net = {
6430                 model => $model,
6431                 bridge => "vmbr$ind",
6432                 macaddr => $macaddr,
6433             };
6434             my $netstr = print_net($net);
6435
6436             $res .= "net$cookie->{netcount}: $netstr\n";
6437             $cookie->{netcount}++;
6438         }
6439     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6440         my ($id, $netstr) = ($1, $2);
6441         my $net = parse_net($netstr);
6442         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6443         $netstr = print_net($net);
6444         $res .= "$id: $netstr\n";
6445     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6446         my $virtdev = $1;
6447         my $value = $3;
6448         my $di = parse_drive($virtdev, $value);
6449         if (defined($di->{backup}) && !$di->{backup}) {
6450             $res .= "#$line";
6451         } elsif ($map->{$virtdev}) {
6452             delete $di->{format}; # format can change on restore
6453             $di->{file} = $map->{$virtdev};
6454             $value = print_drive($di);
6455             $res .= "$virtdev: $value\n";
6456         } else {
6457             $res .= $line;
6458         }
6459     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6460         my $vmgenid = $1;
6461         if ($vmgenid ne '0') {
6462             # always generate a new vmgenid if there was a valid one setup
6463             $vmgenid = generate_uuid();
6464         }
6465         $res .= "vmgenid: $vmgenid\n";
6466     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6467         my ($uuid, $uuid_str);
6468         UUID::generate($uuid);
6469         UUID::unparse($uuid, $uuid_str);
6470         my $smbios1 = parse_smbios1($2);
6471         $smbios1->{uuid} = $uuid_str;
6472         $res .= $1.print_smbios1($smbios1)."\n";
6473     } else {
6474         $res .= $line;
6475     }
6476
6477     return $res;
6478 }
6479
6480 my $restore_deactivate_volumes = sub {
6481     my ($storecfg, $virtdev_hash) = @_;
6482
6483     my $vollist = [];
6484     for my $dev (values $virtdev_hash->%*) {
6485         push $vollist->@*, $dev->{volid} if $dev->{volid};
6486     }
6487
6488     eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
6489     print STDERR $@ if $@;
6490 };
6491
6492 my $restore_destroy_volumes = sub {
6493     my ($storecfg, $virtdev_hash) = @_;
6494
6495     for my $dev (values $virtdev_hash->%*) {
6496         my $volid = $dev->{volid} or next;
6497         eval {
6498             PVE::Storage::vdisk_free($storecfg, $volid);
6499             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6500         };
6501         print STDERR "unable to cleanup '$volid' - $@" if $@;
6502     }
6503 };
6504
6505 my $restore_merge_config = sub {
6506     my ($filename, $backup_conf_raw, $override_conf) = @_;
6507
6508     my $backup_conf = parse_vm_config($filename, $backup_conf_raw);
6509     for my $key (keys $override_conf->%*) {
6510         $backup_conf->{$key} = $override_conf->{$key};
6511     }
6512
6513     return $backup_conf;
6514 };
6515
6516 sub scan_volids {
6517     my ($cfg, $vmid) = @_;
6518
6519     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6520
6521     my $volid_hash = {};
6522     foreach my $storeid (keys %$info) {
6523         foreach my $item (@{$info->{$storeid}}) {
6524             next if !($item->{volid} && $item->{size});
6525             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6526             $volid_hash->{$item->{volid}} = $item;
6527         }
6528     }
6529
6530     return $volid_hash;
6531 }
6532
6533 sub update_disk_config {
6534     my ($vmid, $conf, $volid_hash) = @_;
6535
6536     my $changes;
6537     my $prefix = "VM $vmid";
6538
6539     # used and unused disks
6540     my $referenced = {};
6541
6542     # Note: it is allowed to define multiple storages with same path (alias), so
6543     # we need to check both 'volid' and real 'path' (two different volid can point
6544     # to the same path).
6545
6546     my $referencedpath = {};
6547
6548     # update size info
6549     PVE::QemuConfig->foreach_volume($conf, sub {
6550         my ($opt, $drive) = @_;
6551
6552         my $volid = $drive->{file};
6553         return if !$volid;
6554         my $volume = $volid_hash->{$volid};
6555
6556         # mark volid as "in-use" for next step
6557         $referenced->{$volid} = 1;
6558         if ($volume && (my $path = $volume->{path})) {
6559             $referencedpath->{$path} = 1;
6560         }
6561
6562         return if drive_is_cdrom($drive);
6563         return if !$volume;
6564
6565         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6566         if (defined($updated)) {
6567             $changes = 1;
6568             $conf->{$opt} = print_drive($updated);
6569             print "$prefix ($opt): $msg\n";
6570         }
6571     });
6572
6573     # remove 'unusedX' entry if volume is used
6574     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6575         my ($opt, $drive) = @_;
6576
6577         my $volid = $drive->{file};
6578         return if !$volid;
6579
6580         my $path;
6581         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6582         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6583             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6584             $changes = 1;
6585             delete $conf->{$opt};
6586         }
6587
6588         $referenced->{$volid} = 1;
6589         $referencedpath->{$path} = 1 if $path;
6590     });
6591
6592     foreach my $volid (sort keys %$volid_hash) {
6593         next if $volid =~ m/vm-$vmid-state-/;
6594         next if $referenced->{$volid};
6595         my $path = $volid_hash->{$volid}->{path};
6596         next if !$path; # just to be sure
6597         next if $referencedpath->{$path};
6598         $changes = 1;
6599         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6600         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6601         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6602     }
6603
6604     return $changes;
6605 }
6606
6607 sub rescan {
6608     my ($vmid, $nolock, $dryrun) = @_;
6609
6610     my $cfg = PVE::Storage::config();
6611
6612     print "rescan volumes...\n";
6613     my $volid_hash = scan_volids($cfg, $vmid);
6614
6615     my $updatefn =  sub {
6616         my ($vmid) = @_;
6617
6618         my $conf = PVE::QemuConfig->load_config($vmid);
6619
6620         PVE::QemuConfig->check_lock($conf);
6621
6622         my $vm_volids = {};
6623         foreach my $volid (keys %$volid_hash) {
6624             my $info = $volid_hash->{$volid};
6625             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6626         }
6627
6628         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6629
6630         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6631     };
6632
6633     if (defined($vmid)) {
6634         if ($nolock) {
6635             &$updatefn($vmid);
6636         } else {
6637             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6638         }
6639     } else {
6640         my $vmlist = config_list();
6641         foreach my $vmid (keys %$vmlist) {
6642             if ($nolock) {
6643                 &$updatefn($vmid);
6644             } else {
6645                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6646             }
6647         }
6648     }
6649 }
6650
6651 sub restore_proxmox_backup_archive {
6652     my ($archive, $vmid, $user, $options) = @_;
6653
6654     my $storecfg = PVE::Storage::config();
6655
6656     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6657     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6658
6659     my $fingerprint = $scfg->{fingerprint};
6660     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6661
6662     my $repo = PVE::PBSClient::get_repository($scfg);
6663     my $namespace = $scfg->{namespace};
6664
6665     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6666     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6667     local $ENV{PBS_PASSWORD} = $password;
6668     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6669
6670     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6671         PVE::Storage::parse_volname($storecfg, $archive);
6672
6673     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6674
6675     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6676
6677     my $tmpdir = "/var/tmp/vzdumptmp$$";
6678     rmtree $tmpdir;
6679     mkpath $tmpdir;
6680
6681     my $conffile = PVE::QemuConfig->config_file($vmid);
6682      # disable interrupts (always do cleanups)
6683     local $SIG{INT} =
6684         local $SIG{TERM} =
6685         local $SIG{QUIT} =
6686         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6687
6688     # Note: $oldconf is undef if VM does not exists
6689     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6690     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6691     my $new_conf_raw = '';
6692
6693     my $rpcenv = PVE::RPCEnvironment::get();
6694     my $devinfo = {}; # info about drives included in backup
6695     my $virtdev_hash = {}; # info about allocated drives
6696
6697     eval {
6698         # enable interrupts
6699         local $SIG{INT} =
6700             local $SIG{TERM} =
6701             local $SIG{QUIT} =
6702             local $SIG{HUP} =
6703             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6704
6705         my $cfgfn = "$tmpdir/qemu-server.conf";
6706         my $firewall_config_fn = "$tmpdir/fw.conf";
6707         my $index_fn = "$tmpdir/index.json";
6708
6709         my $cmd = "restore";
6710
6711         my $param = [$pbs_backup_name, "index.json", $index_fn];
6712         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6713         my $index = PVE::Tools::file_get_contents($index_fn);
6714         $index = decode_json($index);
6715
6716         foreach my $info (@{$index->{files}}) {
6717             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6718                 my $devname = $1;
6719                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6720                     $devinfo->{$devname}->{size} = $1;
6721                 } else {
6722                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6723                 }
6724             }
6725         }
6726
6727         my $is_qemu_server_backup = scalar(
6728             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6729         );
6730         if (!$is_qemu_server_backup) {
6731             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6732         }
6733         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6734
6735         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6736         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6737
6738         if ($has_firewall_config) {
6739             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6740             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6741
6742             my $pve_firewall_dir = '/etc/pve/firewall';
6743             mkdir $pve_firewall_dir; # make sure the dir exists
6744             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6745         }
6746
6747         my $fh = IO::File->new($cfgfn, "r") ||
6748             die "unable to read qemu-server.conf - $!\n";
6749
6750         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6751
6752         # fixme: rate limit?
6753
6754         # create empty/temp config
6755         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6756
6757         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6758
6759         # allocate volumes
6760         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6761
6762         foreach my $virtdev (sort keys %$virtdev_hash) {
6763             my $d = $virtdev_hash->{$virtdev};
6764             next if $d->{is_cloudinit}; # no need to restore cloudinit
6765
6766             # this fails if storage is unavailable
6767             my $volid = $d->{volid};
6768             my $path = PVE::Storage::path($storecfg, $volid);
6769
6770             # for live-restore we only want to preload the efidisk and TPM state
6771             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6772
6773             my @ns_arg;
6774             if (defined(my $ns = $scfg->{namespace})) {
6775                 @ns_arg = ('--ns', $ns);
6776             }
6777
6778             my $pbs_restore_cmd = [
6779                 '/usr/bin/pbs-restore',
6780                 '--repository', $repo,
6781                 @ns_arg,
6782                 $pbs_backup_name,
6783                 "$d->{devname}.img.fidx",
6784                 $path,
6785                 '--verbose',
6786                 ];
6787
6788             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6789             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6790
6791             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6792                 push @$pbs_restore_cmd, '--skip-zero';
6793             }
6794
6795             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6796             print "restore proxmox backup image: $dbg_cmdstring\n";
6797             run_command($pbs_restore_cmd);
6798         }
6799
6800         $fh->seek(0, 0) || die "seek failed - $!\n";
6801
6802         my $cookie = { netcount => 0 };
6803         while (defined(my $line = <$fh>)) {
6804             $new_conf_raw .= restore_update_config_line(
6805                 $cookie,
6806                 $map,
6807                 $line,
6808                 $options->{unique},
6809             );
6810         }
6811
6812         $fh->close();
6813     };
6814     my $err = $@;
6815
6816     if ($err || !$options->{live}) {
6817         $restore_deactivate_volumes->($storecfg, $virtdev_hash);
6818     }
6819
6820     rmtree $tmpdir;
6821
6822     if ($err) {
6823         $restore_destroy_volumes->($storecfg, $virtdev_hash);
6824         die $err;
6825     }
6826
6827     if ($options->{live}) {
6828         # keep lock during live-restore
6829         $new_conf_raw .= "\nlock: create";
6830     }
6831
6832     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $options->{override_conf});
6833     PVE::QemuConfig->write_config($vmid, $new_conf);
6834
6835     eval { rescan($vmid, 1); };
6836     warn $@ if $@;
6837
6838     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6839
6840     if ($options->{live}) {
6841         # enable interrupts
6842         local $SIG{INT} =
6843             local $SIG{TERM} =
6844             local $SIG{QUIT} =
6845             local $SIG{HUP} =
6846             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6847
6848         my $conf = PVE::QemuConfig->load_config($vmid);
6849         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6850
6851         # these special drives are already restored before start
6852         delete $devinfo->{'drive-efidisk0'};
6853         delete $devinfo->{'drive-tpmstate0-backup'};
6854
6855         my $pbs_opts = {
6856             repo => $repo,
6857             keyfile => $keyfile,
6858             snapshot => $pbs_backup_name,
6859             namespace => $namespace,
6860         };
6861         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $pbs_opts);
6862
6863         PVE::QemuConfig->remove_lock($vmid, "create");
6864     }
6865 }
6866
6867 sub pbs_live_restore {
6868     my ($vmid, $conf, $storecfg, $restored_disks, $opts) = @_;
6869
6870     print "starting VM for live-restore\n";
6871     print "repository: '$opts->{repo}', snapshot: '$opts->{snapshot}'\n";
6872
6873     my $pbs_backing = {};
6874     for my $ds (keys %$restored_disks) {
6875         $ds =~ m/^drive-(.*)$/;
6876         my $confname = $1;
6877         $pbs_backing->{$confname} = {
6878             repository => $opts->{repo},
6879             snapshot => $opts->{snapshot},
6880             archive => "$ds.img.fidx",
6881         };
6882         $pbs_backing->{$confname}->{keyfile} = $opts->{keyfile} if -e $opts->{keyfile};
6883         $pbs_backing->{$confname}->{namespace} = $opts->{namespace} if defined($opts->{namespace});
6884
6885         my $drive = parse_drive($confname, $conf->{$confname});
6886         print "restoring '$ds' to '$drive->{file}'\n";
6887     }
6888
6889     my $drives_streamed = 0;
6890     eval {
6891         # make sure HA doesn't interrupt our restore by stopping the VM
6892         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6893             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6894         }
6895
6896         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6897         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6898         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6899
6900         my $qmeventd_fd = register_qmeventd_handle($vmid);
6901
6902         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6903         # this will effectively collapse the backing image chain consisting of
6904         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6905         # removes itself once all backing images vanish with 'auto-remove=on')
6906         my $jobs = {};
6907         for my $ds (sort keys %$restored_disks) {
6908             my $job_id = "restore-$ds";
6909             mon_cmd($vmid, 'block-stream',
6910                 'job-id' => $job_id,
6911                 device => "$ds",
6912             );
6913             $jobs->{$job_id} = {};
6914         }
6915
6916         mon_cmd($vmid, 'cont');
6917         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6918
6919         print "restore-drive jobs finished successfully, removing all tracking block devices"
6920             ." to disconnect from Proxmox Backup Server\n";
6921
6922         for my $ds (sort keys %$restored_disks) {
6923             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6924         }
6925
6926         close($qmeventd_fd);
6927     };
6928
6929     my $err = $@;
6930
6931     if ($err) {
6932         warn "An error occurred during live-restore: $err\n";
6933         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6934         die "live-restore failed\n";
6935     }
6936 }
6937
6938 sub restore_vma_archive {
6939     my ($archive, $vmid, $user, $opts, $comp) = @_;
6940
6941     my $readfrom = $archive;
6942
6943     my $cfg = PVE::Storage::config();
6944     my $commands = [];
6945     my $bwlimit = $opts->{bwlimit};
6946
6947     my $dbg_cmdstring = '';
6948     my $add_pipe = sub {
6949         my ($cmd) = @_;
6950         push @$commands, $cmd;
6951         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6952         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6953         $readfrom = '-';
6954     };
6955
6956     my $input = undef;
6957     if ($archive eq '-') {
6958         $input = '<&STDIN';
6959     } else {
6960         # If we use a backup from a PVE defined storage we also consider that
6961         # storage's rate limit:
6962         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6963         if (defined($volid)) {
6964             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6965             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6966             if ($readlimit) {
6967                 print STDERR "applying read rate limit: $readlimit\n";
6968                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6969                 $add_pipe->($cstream);
6970             }
6971         }
6972     }
6973
6974     if ($comp) {
6975         my $info = PVE::Storage::decompressor_info('vma', $comp);
6976         my $cmd = $info->{decompressor};
6977         push @$cmd, $readfrom;
6978         $add_pipe->($cmd);
6979     }
6980
6981     my $tmpdir = "/var/tmp/vzdumptmp$$";
6982     rmtree $tmpdir;
6983
6984     # disable interrupts (always do cleanups)
6985     local $SIG{INT} =
6986         local $SIG{TERM} =
6987         local $SIG{QUIT} =
6988         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6989
6990     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6991     POSIX::mkfifo($mapfifo, 0600);
6992     my $fifofh;
6993     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6994
6995     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6996
6997     my $oldtimeout;
6998     my $timeout = 5;
6999
7000     my $devinfo = {}; # info about drives included in backup
7001     my $virtdev_hash = {}; # info about allocated drives
7002
7003     my $rpcenv = PVE::RPCEnvironment::get();
7004
7005     my $conffile = PVE::QemuConfig->config_file($vmid);
7006
7007     # Note: $oldconf is undef if VM does not exist
7008     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
7009     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
7010     my $new_conf_raw = '';
7011
7012     my %storage_limits;
7013
7014     my $print_devmap = sub {
7015         my $cfgfn = "$tmpdir/qemu-server.conf";
7016
7017         # we can read the config - that is already extracted
7018         my $fh = IO::File->new($cfgfn, "r") ||
7019             die "unable to read qemu-server.conf - $!\n";
7020
7021         my $fwcfgfn = "$tmpdir/qemu-server.fw";
7022         if (-f $fwcfgfn) {
7023             my $pve_firewall_dir = '/etc/pve/firewall';
7024             mkdir $pve_firewall_dir; # make sure the dir exists
7025             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
7026         }
7027
7028         $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
7029
7030         foreach my $info (values %{$virtdev_hash}) {
7031             my $storeid = $info->{storeid};
7032             next if defined($storage_limits{$storeid});
7033
7034             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
7035             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
7036             $storage_limits{$storeid} = $limit * 1024;
7037         }
7038
7039         foreach my $devname (keys %$devinfo) {
7040             die "found no device mapping information for device '$devname'\n"
7041                 if !$devinfo->{$devname}->{virtdev};
7042         }
7043
7044         # create empty/temp config
7045         if ($oldconf) {
7046             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
7047             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
7048         }
7049
7050         # allocate volumes
7051         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
7052
7053         # print restore information to $fifofh
7054         foreach my $virtdev (sort keys %$virtdev_hash) {
7055             my $d = $virtdev_hash->{$virtdev};
7056             next if $d->{is_cloudinit}; # no need to restore cloudinit
7057
7058             my $storeid = $d->{storeid};
7059             my $volid = $d->{volid};
7060
7061             my $map_opts = '';
7062             if (my $limit = $storage_limits{$storeid}) {
7063                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
7064             }
7065
7066             my $write_zeros = 1;
7067             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
7068                 $write_zeros = 0;
7069             }
7070
7071             my $path = PVE::Storage::path($cfg, $volid);
7072
7073             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
7074
7075             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
7076         }
7077
7078         $fh->seek(0, 0) || die "seek failed - $!\n";
7079
7080         my $cookie = { netcount => 0 };
7081         while (defined(my $line = <$fh>)) {
7082             $new_conf_raw .= restore_update_config_line(
7083                 $cookie,
7084                 $map,
7085                 $line,
7086                 $opts->{unique},
7087             );
7088         }
7089
7090         $fh->close();
7091     };
7092
7093     eval {
7094         # enable interrupts
7095         local $SIG{INT} =
7096             local $SIG{TERM} =
7097             local $SIG{QUIT} =
7098             local $SIG{HUP} =
7099             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7100         local $SIG{ALRM} = sub { die "got timeout\n"; };
7101
7102         $oldtimeout = alarm($timeout);
7103
7104         my $parser = sub {
7105             my $line = shift;
7106
7107             print "$line\n";
7108
7109             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
7110                 my ($dev_id, $size, $devname) = ($1, $2, $3);
7111                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
7112             } elsif ($line =~ m/^CTIME: /) {
7113                 # we correctly received the vma config, so we can disable
7114                 # the timeout now for disk allocation (set to 10 minutes, so
7115                 # that we always timeout if something goes wrong)
7116                 alarm(600);
7117                 &$print_devmap();
7118                 print $fifofh "done\n";
7119                 my $tmp = $oldtimeout || 0;
7120                 $oldtimeout = undef;
7121                 alarm($tmp);
7122                 close($fifofh);
7123                 $fifofh = undef;
7124             }
7125         };
7126
7127         print "restore vma archive: $dbg_cmdstring\n";
7128         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7129     };
7130     my $err = $@;
7131
7132     alarm($oldtimeout) if $oldtimeout;
7133
7134     $restore_deactivate_volumes->($cfg, $virtdev_hash);
7135
7136     close($fifofh) if $fifofh;
7137     unlink $mapfifo;
7138     rmtree $tmpdir;
7139
7140     if ($err) {
7141         $restore_destroy_volumes->($cfg, $virtdev_hash);
7142         die $err;
7143     }
7144
7145     my $new_conf = $restore_merge_config->($conffile, $new_conf_raw, $opts->{override_conf});
7146     PVE::QemuConfig->write_config($vmid, $new_conf);
7147
7148     eval { rescan($vmid, 1); };
7149     warn $@ if $@;
7150
7151     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7152 }
7153
7154 sub restore_tar_archive {
7155     my ($archive, $vmid, $user, $opts) = @_;
7156
7157     if (scalar(keys $opts->{override_conf}->%*) > 0) {
7158         my $keystring = join(' ', keys $opts->{override_conf}->%*);
7159         die "cannot pass along options ($keystring) when restoring from tar archive\n";
7160     }
7161
7162     if ($archive ne '-') {
7163         my $firstfile = tar_archive_read_firstfile($archive);
7164         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7165             if $firstfile ne 'qemu-server.conf';
7166     }
7167
7168     my $storecfg = PVE::Storage::config();
7169
7170     # avoid zombie disks when restoring over an existing VM -> cleanup first
7171     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7172     # skiplock=1 because qmrestore has set the 'create' lock itself already
7173     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7174     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7175
7176     my $tocmd = "/usr/lib/qemu-server/qmextract";
7177
7178     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7179     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7180     $tocmd .= ' --prealloc' if $opts->{prealloc};
7181     $tocmd .= ' --info' if $opts->{info};
7182
7183     # tar option "xf" does not autodetect compression when read from STDIN,
7184     # so we pipe to zcat
7185     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7186         PVE::Tools::shellquote("--to-command=$tocmd");
7187
7188     my $tmpdir = "/var/tmp/vzdumptmp$$";
7189     mkpath $tmpdir;
7190
7191     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7192     local $ENV{VZDUMP_VMID} = $vmid;
7193     local $ENV{VZDUMP_USER} = $user;
7194
7195     my $conffile = PVE::QemuConfig->config_file($vmid);
7196     my $new_conf_raw = '';
7197
7198     # disable interrupts (always do cleanups)
7199     local $SIG{INT} =
7200         local $SIG{TERM} =
7201         local $SIG{QUIT} =
7202         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7203
7204     eval {
7205         # enable interrupts
7206         local $SIG{INT} =
7207             local $SIG{TERM} =
7208             local $SIG{QUIT} =
7209             local $SIG{HUP} =
7210             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7211
7212         if ($archive eq '-') {
7213             print "extracting archive from STDIN\n";
7214             run_command($cmd, input => "<&STDIN");
7215         } else {
7216             print "extracting archive '$archive'\n";
7217             run_command($cmd);
7218         }
7219
7220         return if $opts->{info};
7221
7222         # read new mapping
7223         my $map = {};
7224         my $statfile = "$tmpdir/qmrestore.stat";
7225         if (my $fd = IO::File->new($statfile, "r")) {
7226             while (defined (my $line = <$fd>)) {
7227                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7228                     $map->{$1} = $2 if $1;
7229                 } else {
7230                     print STDERR "unable to parse line in statfile - $line\n";
7231                 }
7232             }
7233             $fd->close();
7234         }
7235
7236         my $confsrc = "$tmpdir/qemu-server.conf";
7237
7238         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7239
7240         my $cookie = { netcount => 0 };
7241         while (defined (my $line = <$srcfd>)) {
7242             $new_conf_raw .= restore_update_config_line(
7243                 $cookie,
7244                 $map,
7245                 $line,
7246                 $opts->{unique},
7247             );
7248         }
7249
7250         $srcfd->close();
7251     };
7252     if (my $err = $@) {
7253         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7254         die $err;
7255     }
7256
7257     rmtree $tmpdir;
7258
7259     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7260
7261     PVE::Cluster::cfs_update(); # make sure we read new file
7262
7263     eval { rescan($vmid, 1); };
7264     warn $@ if $@;
7265 };
7266
7267 sub foreach_storage_used_by_vm {
7268     my ($conf, $func) = @_;
7269
7270     my $sidhash = {};
7271
7272     PVE::QemuConfig->foreach_volume($conf, sub {
7273         my ($ds, $drive) = @_;
7274         return if drive_is_cdrom($drive);
7275
7276         my $volid = $drive->{file};
7277
7278         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7279         $sidhash->{$sid} = $sid if $sid;
7280     });
7281
7282     foreach my $sid (sort keys %$sidhash) {
7283         &$func($sid);
7284     }
7285 }
7286
7287 my $qemu_snap_storage = {
7288     rbd => 1,
7289 };
7290 sub do_snapshots_with_qemu {
7291     my ($storecfg, $volid, $deviceid) = @_;
7292
7293     return if $deviceid =~ m/tpmstate0/;
7294
7295     my $storage_name = PVE::Storage::parse_volume_id($volid);
7296     my $scfg = $storecfg->{ids}->{$storage_name};
7297     die "could not find storage '$storage_name'\n" if !defined($scfg);
7298
7299     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7300         return 1;
7301     }
7302
7303     if ($volid =~ m/\.(qcow2|qed)$/){
7304         return 1;
7305     }
7306
7307     return;
7308 }
7309
7310 sub qga_check_running {
7311     my ($vmid, $nowarn) = @_;
7312
7313     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7314     if ($@) {
7315         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7316         return 0;
7317     }
7318     return 1;
7319 }
7320
7321 sub template_create {
7322     my ($vmid, $conf, $disk) = @_;
7323
7324     my $storecfg = PVE::Storage::config();
7325
7326     PVE::QemuConfig->foreach_volume($conf, sub {
7327         my ($ds, $drive) = @_;
7328
7329         return if drive_is_cdrom($drive);
7330         return if $disk && $ds ne $disk;
7331
7332         my $volid = $drive->{file};
7333         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7334
7335         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7336         $drive->{file} = $voliddst;
7337         $conf->{$ds} = print_drive($drive);
7338         PVE::QemuConfig->write_config($vmid, $conf);
7339     });
7340 }
7341
7342 sub convert_iscsi_path {
7343     my ($path) = @_;
7344
7345     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7346         my $portal = $1;
7347         my $target = $2;
7348         my $lun = $3;
7349
7350         my $initiator_name = get_initiator_name();
7351
7352         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7353                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7354     }
7355
7356     die "cannot convert iscsi path '$path', unkown format\n";
7357 }
7358
7359 sub qemu_img_convert {
7360     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7361
7362     my $storecfg = PVE::Storage::config();
7363     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7364     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7365
7366     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7367
7368     my $cachemode;
7369     my $src_path;
7370     my $src_is_iscsi = 0;
7371     my $src_format;
7372
7373     if ($src_storeid) {
7374         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7375         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7376         $src_format = qemu_img_format($src_scfg, $src_volname);
7377         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7378         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7379         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7380     } elsif (-f $src_volid || -b $src_volid) {
7381         $src_path = $src_volid;
7382         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7383             $src_format = $1;
7384         }
7385     }
7386
7387     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7388
7389     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7390     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7391     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7392     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7393
7394     my $cmd = [];
7395     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7396     push @$cmd, '-l', "snapshot.name=$snapname"
7397         if $snapname && $src_format && $src_format eq "qcow2";
7398     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7399     push @$cmd, '-T', $cachemode if defined($cachemode);
7400
7401     if ($src_is_iscsi) {
7402         push @$cmd, '--image-opts';
7403         $src_path = convert_iscsi_path($src_path);
7404     } elsif ($src_format) {
7405         push @$cmd, '-f', $src_format;
7406     }
7407
7408     if ($dst_is_iscsi) {
7409         push @$cmd, '--target-image-opts';
7410         $dst_path = convert_iscsi_path($dst_path);
7411     } else {
7412         push @$cmd, '-O', $dst_format;
7413     }
7414
7415     push @$cmd, $src_path;
7416
7417     if (!$dst_is_iscsi && $is_zero_initialized) {
7418         push @$cmd, "zeroinit:$dst_path";
7419     } else {
7420         push @$cmd, $dst_path;
7421     }
7422
7423     my $parser = sub {
7424         my $line = shift;
7425         if($line =~ m/\((\S+)\/100\%\)/){
7426             my $percent = $1;
7427             my $transferred = int($size * $percent / 100);
7428             my $total_h = render_bytes($size, 1);
7429             my $transferred_h = render_bytes($transferred, 1);
7430
7431             print "transferred $transferred_h of $total_h ($percent%)\n";
7432         }
7433
7434     };
7435
7436     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7437     my $err = $@;
7438     die "copy failed: $err" if $err;
7439 }
7440
7441 sub qemu_img_format {
7442     my ($scfg, $volname) = @_;
7443
7444     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7445         return $1;
7446     } else {
7447         return "raw";
7448     }
7449 }
7450
7451 sub qemu_drive_mirror {
7452     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7453
7454     $jobs = {} if !$jobs;
7455
7456     my $qemu_target;
7457     my $format;
7458     $jobs->{"drive-$drive"} = {};
7459
7460     if ($dst_volid =~ /^nbd:/) {
7461         $qemu_target = $dst_volid;
7462         $format = "nbd";
7463     } else {
7464         my $storecfg = PVE::Storage::config();
7465         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7466
7467         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7468
7469         $format = qemu_img_format($dst_scfg, $dst_volname);
7470
7471         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7472
7473         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7474     }
7475
7476     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7477     $opts->{format} = $format if $format;
7478
7479     if (defined($src_bitmap)) {
7480         $opts->{sync} = 'incremental';
7481         $opts->{bitmap} = $src_bitmap;
7482         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7483     }
7484
7485     if (defined($bwlimit)) {
7486         $opts->{speed} = $bwlimit * 1024;
7487         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7488     } else {
7489         print "drive mirror is starting for drive-$drive\n";
7490     }
7491
7492     # if a job already runs for this device we get an error, catch it for cleanup
7493     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7494     if (my $err = $@) {
7495         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7496         warn "$@\n" if $@;
7497         die "mirroring error: $err\n";
7498     }
7499
7500     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7501 }
7502
7503 # $completion can be either
7504 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7505 # 'cancel': wait until all jobs are ready, block-job-cancel them
7506 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7507 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7508 sub qemu_drive_mirror_monitor {
7509     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7510
7511     $completion //= 'complete';
7512     $op //= "mirror";
7513
7514     eval {
7515         my $err_complete = 0;
7516
7517         my $starttime = time ();
7518         while (1) {
7519             die "block job ('$op') timed out\n" if $err_complete > 300;
7520
7521             my $stats = mon_cmd($vmid, "query-block-jobs");
7522             my $ctime = time();
7523
7524             my $running_jobs = {};
7525             for my $stat (@$stats) {
7526                 next if $stat->{type} ne $op;
7527                 $running_jobs->{$stat->{device}} = $stat;
7528             }
7529
7530             my $readycounter = 0;
7531
7532             for my $job_id (sort keys %$jobs) {
7533                 my $job = $running_jobs->{$job_id};
7534
7535                 my $vanished = !defined($job);
7536                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7537                 if($complete || ($vanished && $completion eq 'auto')) {
7538                     print "$job_id: $op-job finished\n";
7539                     delete $jobs->{$job_id};
7540                     next;
7541                 }
7542
7543                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7544
7545                 my $busy = $job->{busy};
7546                 my $ready = $job->{ready};
7547                 if (my $total = $job->{len}) {
7548                     my $transferred = $job->{offset} || 0;
7549                     my $remaining = $total - $transferred;
7550                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7551
7552                     my $duration = $ctime - $starttime;
7553                     my $total_h = render_bytes($total, 1);
7554                     my $transferred_h = render_bytes($transferred, 1);
7555
7556                     my $status = sprintf(
7557                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7558                         render_duration($duration),
7559                     );
7560
7561                     if ($ready) {
7562                         if ($busy) {
7563                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7564                         } else {
7565                             $status .= ", ready";
7566                         }
7567                     }
7568                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7569                     $jobs->{$job_id}->{ready} = $ready;
7570                 }
7571
7572                 $readycounter++ if $job->{ready};
7573             }
7574
7575             last if scalar(keys %$jobs) == 0;
7576
7577             if ($readycounter == scalar(keys %$jobs)) {
7578                 print "all '$op' jobs are ready\n";
7579
7580                 # do the complete later (or has already been done)
7581                 last if $completion eq 'skip' || $completion eq 'auto';
7582
7583                 if ($vmiddst && $vmiddst != $vmid) {
7584                     my $agent_running = $qga && qga_check_running($vmid);
7585                     if ($agent_running) {
7586                         print "freeze filesystem\n";
7587                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7588                         warn $@ if $@;
7589                     } else {
7590                         print "suspend vm\n";
7591                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7592                         warn $@ if $@;
7593                     }
7594
7595                     # if we clone a disk for a new target vm, we don't switch the disk
7596                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7597
7598                     if ($agent_running) {
7599                         print "unfreeze filesystem\n";
7600                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7601                         warn $@ if $@;
7602                     } else {
7603                         print "resume vm\n";
7604                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7605                         warn $@ if $@;
7606                     }
7607
7608                     last;
7609                 } else {
7610
7611                     for my $job_id (sort keys %$jobs) {
7612                         # try to switch the disk if source and destination are on the same guest
7613                         print "$job_id: Completing block job_id...\n";
7614
7615                         my $op;
7616                         if ($completion eq 'complete') {
7617                             $op = 'block-job-complete';
7618                         } elsif ($completion eq 'cancel') {
7619                             $op = 'block-job-cancel';
7620                         } else {
7621                             die "invalid completion value: $completion\n";
7622                         }
7623                         eval { mon_cmd($vmid, $op, device => $job_id) };
7624                         if ($@ =~ m/cannot be completed/) {
7625                             print "$job_id: block job cannot be completed, trying again.\n";
7626                             $err_complete++;
7627                         }else {
7628                             print "$job_id: Completed successfully.\n";
7629                             $jobs->{$job_id}->{complete} = 1;
7630                         }
7631                     }
7632                 }
7633             }
7634             sleep 1;
7635         }
7636     };
7637     my $err = $@;
7638
7639     if ($err) {
7640         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7641         die "block job ($op) error: $err";
7642     }
7643 }
7644
7645 sub qemu_blockjobs_cancel {
7646     my ($vmid, $jobs) = @_;
7647
7648     foreach my $job (keys %$jobs) {
7649         print "$job: Cancelling block job\n";
7650         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7651         $jobs->{$job}->{cancel} = 1;
7652     }
7653
7654     while (1) {
7655         my $stats = mon_cmd($vmid, "query-block-jobs");
7656
7657         my $running_jobs = {};
7658         foreach my $stat (@$stats) {
7659             $running_jobs->{$stat->{device}} = $stat;
7660         }
7661
7662         foreach my $job (keys %$jobs) {
7663
7664             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7665                 print "$job: Done.\n";
7666                 delete $jobs->{$job};
7667             }
7668         }
7669
7670         last if scalar(keys %$jobs) == 0;
7671
7672         sleep 1;
7673     }
7674 }
7675
7676 sub clone_disk {
7677     my ($storecfg, $source, $dest, $full, $newvollist, $jobs, $completion, $qga, $bwlimit) = @_;
7678
7679     my ($vmid, $running) = $source->@{qw(vmid running)};
7680     my ($src_drivename, $drive, $snapname) = $source->@{qw(drivename drive snapname)};
7681
7682     my ($newvmid, $dst_drivename, $efisize) = $dest->@{qw(vmid drivename efisize)};
7683     my ($storage, $format) = $dest->@{qw(storage format)};
7684
7685     my $use_drive_mirror = $full && $running && $src_drivename && !$snapname;
7686
7687     if ($src_drivename && $dst_drivename && $src_drivename ne $dst_drivename) {
7688         die "cloning from/to EFI disk requires EFI disk\n"
7689             if $src_drivename eq 'efidisk0' || $dst_drivename eq 'efidisk0';
7690         die "cloning from/to TPM state requires TPM state\n"
7691             if $src_drivename eq 'tpmstate0' || $dst_drivename eq 'tpmstate0';
7692
7693         # This would lead to two device nodes in QEMU pointing to the same backing image!
7694         die "cannot change drive name when cloning disk from/to the same VM\n"
7695             if $use_drive_mirror && $vmid == $newvmid;
7696     }
7697
7698     die "cannot move TPM state while VM is running\n"
7699         if $use_drive_mirror && $src_drivename eq 'tpmstate0';
7700
7701     my $newvolid;
7702
7703     print "create " . ($full ? 'full' : 'linked') . " clone of drive ";
7704     print "$src_drivename " if $src_drivename;
7705     print "($drive->{file})\n";
7706
7707     if (!$full) {
7708         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7709         push @$newvollist, $newvolid;
7710     } else {
7711
7712         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7713         $storeid = $storage if $storage;
7714
7715         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7716
7717         my $name = undef;
7718         my $size = undef;
7719         if (drive_is_cloudinit($drive)) {
7720             $name = "vm-$newvmid-cloudinit";
7721             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7722             if ($scfg->{path}) {
7723                 $name .= ".$dst_format";
7724             }
7725             $snapname = undef;
7726             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7727         } elsif ($dst_drivename eq 'efidisk0') {
7728             $size = $efisize or die "internal error - need to specify EFI disk size\n";
7729         } elsif ($dst_drivename eq 'tpmstate0') {
7730             $dst_format = 'raw';
7731             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7732         } else {
7733             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7734         }
7735         $newvolid = PVE::Storage::vdisk_alloc(
7736             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7737         );
7738         push @$newvollist, $newvolid;
7739
7740         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7741
7742         if (drive_is_cloudinit($drive)) {
7743             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7744             # if this is the case, we have to complete any block-jobs still there from
7745             # previous drive-mirrors
7746             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7747                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7748             }
7749             goto no_data_clone;
7750         }
7751
7752         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7753         if ($use_drive_mirror) {
7754             qemu_drive_mirror($vmid, $src_drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7755                 $completion, $qga, $bwlimit);
7756         } else {
7757             # TODO: handle bwlimits
7758             if ($dst_drivename eq 'efidisk0') {
7759                 # the relevant data on the efidisk may be smaller than the source
7760                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7761                 # that is given by the OVMF_VARS.fd
7762                 my $src_path = PVE::Storage::path($storecfg, $drive->{file}, $snapname);
7763                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7764
7765                 my $src_format = (PVE::Storage::parse_volname($storecfg, $drive->{file}))[6];
7766
7767                 # better for Ceph if block size is not too small, see bug #3324
7768                 my $bs = 1024*1024;
7769
7770                 my $cmd = ['qemu-img', 'dd', '-n', '-O', $dst_format];
7771
7772                 if ($src_format eq 'qcow2' && $snapname) {
7773                     die "cannot clone qcow2 EFI disk snapshot - requires QEMU >= 6.2\n"
7774                         if !min_version(kvm_user_version(), 6, 2);
7775                     push $cmd->@*, '-l', $snapname;
7776                 }
7777                 push $cmd->@*, "bs=$bs", "osize=$size", "if=$src_path", "of=$dst_path";
7778                 run_command($cmd);
7779             } else {
7780                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7781             }
7782         }
7783     }
7784
7785 no_data_clone:
7786     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7787
7788     my $disk = dclone($drive);
7789     delete $disk->{format};
7790     $disk->{file} = $newvolid;
7791     $disk->{size} = $size if defined($size);
7792
7793     return $disk;
7794 }
7795
7796 sub get_running_qemu_version {
7797     my ($vmid) = @_;
7798     my $res = mon_cmd($vmid, "query-version");
7799     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7800 }
7801
7802 sub qemu_use_old_bios_files {
7803     my ($machine_type) = @_;
7804
7805     return if !$machine_type;
7806
7807     my $use_old_bios_files = undef;
7808
7809     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7810         $machine_type = $1;
7811         $use_old_bios_files = 1;
7812     } else {
7813         my $version = extract_version($machine_type, kvm_user_version());
7814         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7815         # load new efi bios files on migration. So this hack is required to allow
7816         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7817         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7818         $use_old_bios_files = !min_version($version, 2, 4);
7819     }
7820
7821     return ($use_old_bios_files, $machine_type);
7822 }
7823
7824 sub get_efivars_size {
7825     my ($conf, $efidisk) = @_;
7826
7827     my $arch = get_vm_arch($conf);
7828     $efidisk //= $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7829     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7830     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7831     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7832     return -s $ovmf_vars;
7833 }
7834
7835 sub update_efidisk_size {
7836     my ($conf) = @_;
7837
7838     return if !defined($conf->{efidisk0});
7839
7840     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7841     $disk->{size} = get_efivars_size($conf);
7842     $conf->{efidisk0} = print_drive($disk);
7843
7844     return;
7845 }
7846
7847 sub update_tpmstate_size {
7848     my ($conf) = @_;
7849
7850     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7851     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7852     $conf->{tpmstate0} = print_drive($disk);
7853 }
7854
7855 sub create_efidisk($$$$$$$) {
7856     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7857
7858     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7859     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7860
7861     my $vars_size_b = -s $ovmf_vars;
7862     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7863     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7864     PVE::Storage::activate_volumes($storecfg, [$volid]);
7865
7866     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7867     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7868
7869     return ($volid, $size/1024);
7870 }
7871
7872 sub vm_iothreads_list {
7873     my ($vmid) = @_;
7874
7875     my $res = mon_cmd($vmid, 'query-iothreads');
7876
7877     my $iothreads = {};
7878     foreach my $iothread (@$res) {
7879         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7880     }
7881
7882     return $iothreads;
7883 }
7884
7885 sub scsihw_infos {
7886     my ($conf, $drive) = @_;
7887
7888     my $maxdev = 0;
7889
7890     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7891         $maxdev = 7;
7892     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7893         $maxdev = 1;
7894     } else {
7895         $maxdev = 256;
7896     }
7897
7898     my $controller = int($drive->{index} / $maxdev);
7899     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7900         ? "virtioscsi"
7901         : "scsihw";
7902
7903     return ($maxdev, $controller, $controller_prefix);
7904 }
7905
7906 sub windows_version {
7907     my ($ostype) = @_;
7908
7909     return 0 if !$ostype;
7910
7911     my $winversion = 0;
7912
7913     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7914         $winversion = 5;
7915     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7916         $winversion = 6;
7917     } elsif ($ostype =~ m/^win(\d+)$/) {
7918         $winversion = $1;
7919     }
7920
7921     return $winversion;
7922 }
7923
7924 sub resolve_dst_disk_format {
7925         my ($storecfg, $storeid, $src_volname, $format) = @_;
7926         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7927
7928         if (!$format) {
7929             # if no target format is specified, use the source disk format as hint
7930             if ($src_volname) {
7931                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7932                 $format = qemu_img_format($scfg, $src_volname);
7933             } else {
7934                 return $defFormat;
7935             }
7936         }
7937
7938         # test if requested format is supported - else use default
7939         my $supported = grep { $_ eq $format } @$validFormats;
7940         $format = $defFormat if !$supported;
7941         return $format;
7942 }
7943
7944 # NOTE: if this logic changes, please update docs & possibly gui logic
7945 sub find_vmstate_storage {
7946     my ($conf, $storecfg) = @_;
7947
7948     # first, return storage from conf if set
7949     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7950
7951     my ($target, $shared, $local);
7952
7953     foreach_storage_used_by_vm($conf, sub {
7954         my ($sid) = @_;
7955         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7956         my $dst = $scfg->{shared} ? \$shared : \$local;
7957         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7958     });
7959
7960     # second, use shared storage where VM has at least one disk
7961     # third, use local storage where VM has at least one disk
7962     # fall back to local storage
7963     $target = $shared // $local // 'local';
7964
7965     return $target;
7966 }
7967
7968 sub generate_uuid {
7969     my ($uuid, $uuid_str);
7970     UUID::generate($uuid);
7971     UUID::unparse($uuid, $uuid_str);
7972     return $uuid_str;
7973 }
7974
7975 sub generate_smbios1_uuid {
7976     return "uuid=".generate_uuid();
7977 }
7978
7979 sub nbd_stop {
7980     my ($vmid) = @_;
7981
7982     mon_cmd($vmid, 'nbd-server-stop');
7983 }
7984
7985 sub create_reboot_request {
7986     my ($vmid) = @_;
7987     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7988         or die "failed to create reboot trigger file: $!\n";
7989     close($fh);
7990 }
7991
7992 sub clear_reboot_request {
7993     my ($vmid) = @_;
7994     my $path = "/run/qemu-server/$vmid.reboot";
7995     my $res = 0;
7996
7997     $res = unlink($path);
7998     die "could not remove reboot request for $vmid: $!"
7999         if !$res && $! != POSIX::ENOENT;
8000
8001     return $res;
8002 }
8003
8004 sub bootorder_from_legacy {
8005     my ($conf, $bootcfg) = @_;
8006
8007     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
8008     my $bootindex_hash = {};
8009     my $i = 1;
8010     foreach my $o (split(//, $boot)) {
8011         $bootindex_hash->{$o} = $i*100;
8012         $i++;
8013     }
8014
8015     my $bootorder = {};
8016
8017     PVE::QemuConfig->foreach_volume($conf, sub {
8018         my ($ds, $drive) = @_;
8019
8020         if (drive_is_cdrom ($drive, 1)) {
8021             if ($bootindex_hash->{d}) {
8022                 $bootorder->{$ds} = $bootindex_hash->{d};
8023                 $bootindex_hash->{d} += 1;
8024             }
8025         } elsif ($bootindex_hash->{c}) {
8026             $bootorder->{$ds} = $bootindex_hash->{c}
8027                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
8028             $bootindex_hash->{c} += 1;
8029         }
8030     });
8031
8032     if ($bootindex_hash->{n}) {
8033         for (my $i = 0; $i < $MAX_NETS; $i++) {
8034             my $netname = "net$i";
8035             next if !$conf->{$netname};
8036             $bootorder->{$netname} = $bootindex_hash->{n};
8037             $bootindex_hash->{n} += 1;
8038         }
8039     }
8040
8041     return $bootorder;
8042 }
8043
8044 # Generate default device list for 'boot: order=' property. Matches legacy
8045 # default boot order, but with explicit device names. This is important, since
8046 # the fallback for when neither 'order' nor the old format is specified relies
8047 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
8048 sub get_default_bootdevices {
8049     my ($conf) = @_;
8050
8051     my @ret = ();
8052
8053     # harddisk
8054     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
8055     push @ret, $first if $first;
8056
8057     # cdrom
8058     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
8059     push @ret, $first if $first;
8060
8061     # network
8062     for (my $i = 0; $i < $MAX_NETS; $i++) {
8063         my $netname = "net$i";
8064         next if !$conf->{$netname};
8065         push @ret, $netname;
8066         last;
8067     }
8068
8069     return \@ret;
8070 }
8071
8072 sub device_bootorder {
8073     my ($conf) = @_;
8074
8075     return bootorder_from_legacy($conf) if !defined($conf->{boot});
8076
8077     my $boot = parse_property_string($boot_fmt, $conf->{boot});
8078
8079     my $bootorder = {};
8080     if (!defined($boot) || $boot->{legacy}) {
8081         $bootorder = bootorder_from_legacy($conf, $boot);
8082     } elsif ($boot->{order}) {
8083         my $i = 100; # start at 100 to allow user to insert devices before us with -args
8084         for my $dev (PVE::Tools::split_list($boot->{order})) {
8085             $bootorder->{$dev} = $i++;
8086         }
8087     }
8088
8089     return $bootorder;
8090 }
8091
8092 sub register_qmeventd_handle {
8093     my ($vmid) = @_;
8094
8095     my $fh;
8096     my $peer = "/var/run/qmeventd.sock";
8097     my $count = 0;
8098
8099     for (;;) {
8100         $count++;
8101         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
8102         last if $fh;
8103         if ($! != EINTR && $! != EAGAIN) {
8104             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
8105         }
8106         if ($count > 4) {
8107             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
8108               . "after $count retries\n";
8109         }
8110         usleep(25000);
8111     }
8112
8113     # send handshake to mark VM as backing up
8114     print $fh to_json({vzdump => {vmid => "$vmid"}});
8115
8116     # return handle to be closed later when inhibit is no longer required
8117     return $fh;
8118 }
8119
8120 # bash completion helper
8121
8122 sub complete_backup_archives {
8123     my ($cmdname, $pname, $cvalue) = @_;
8124
8125     my $cfg = PVE::Storage::config();
8126
8127     my $storeid;
8128
8129     if ($cvalue =~ m/^([^:]+):/) {
8130         $storeid = $1;
8131     }
8132
8133     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
8134
8135     my $res = [];
8136     foreach my $id (keys %$data) {
8137         foreach my $item (@{$data->{$id}}) {
8138             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
8139             push @$res, $item->{volid} if defined($item->{volid});
8140         }
8141     }
8142
8143     return $res;
8144 }
8145
8146 my $complete_vmid_full = sub {
8147     my ($running) = @_;
8148
8149     my $idlist = vmstatus();
8150
8151     my $res = [];
8152
8153     foreach my $id (keys %$idlist) {
8154         my $d = $idlist->{$id};
8155         if (defined($running)) {
8156             next if $d->{template};
8157             next if $running && $d->{status} ne 'running';
8158             next if !$running && $d->{status} eq 'running';
8159         }
8160         push @$res, $id;
8161
8162     }
8163     return $res;
8164 };
8165
8166 sub complete_vmid {
8167     return &$complete_vmid_full();
8168 }
8169
8170 sub complete_vmid_stopped {
8171     return &$complete_vmid_full(0);
8172 }
8173
8174 sub complete_vmid_running {
8175     return &$complete_vmid_full(1);
8176 }
8177
8178 sub complete_storage {
8179
8180     my $cfg = PVE::Storage::config();
8181     my $ids = $cfg->{ids};
8182
8183     my $res = [];
8184     foreach my $sid (keys %$ids) {
8185         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8186         next if !$ids->{$sid}->{content}->{images};
8187         push @$res, $sid;
8188     }
8189
8190     return $res;
8191 }
8192
8193 sub complete_migration_storage {
8194     my ($cmd, $param, $current_value, $all_args) = @_;
8195
8196     my $targetnode = @$all_args[1];
8197
8198     my $cfg = PVE::Storage::config();
8199     my $ids = $cfg->{ids};
8200
8201     my $res = [];
8202     foreach my $sid (keys %$ids) {
8203         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8204         next if !$ids->{$sid}->{content}->{images};
8205         push @$res, $sid;
8206     }
8207
8208     return $res;
8209 }
8210
8211 sub vm_is_paused {
8212     my ($vmid) = @_;
8213     my $qmpstatus = eval {
8214         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8215         mon_cmd($vmid, "query-status");
8216     };
8217     warn "$@\n" if $@;
8218     return $qmpstatus && $qmpstatus->{status} eq "paused";
8219 }
8220
8221 sub check_volume_storage_type {
8222     my ($storecfg, $vol) = @_;
8223
8224     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8225     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8226     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8227
8228     die "storage '$storeid' does not support content-type '$vtype'\n"
8229         if !$scfg->{content}->{$vtype};
8230
8231     return 1;
8232 }
8233
8234 1;