PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m-no-smm' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-no-smm-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         '4m' => [
  76             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  78         ],
  79         '4m-ms' => [
  80             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  81             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  82         ],
  83         default => [
  84             "$EDK2_FW_BASE/OVMF_CODE.fd",
  85             "$EDK2_FW_BASE/OVMF_VARS.fd",
  86         ],
  87     },
  88     aarch64 => {
  89         default => [
  90             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  91             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  92         ],
  93     },
  94 };
  95
  96 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  97
  98 # Note about locking: we use flock on the config file protect against concurent actions.
  99 # Aditionaly, we have a 'lock' setting in the config file. This  can be set to 'migrate',
 100 # 'backup', 'snapshot' or 'rollback'. Most actions are not allowed when such lock is set.
 101 # But you can ignore this kind of lock with the --skiplock flag.
 102
 103 cfs_register_file('/qemu-server/',
 104                   \&parse_vm_config,
 105                   \&write_vm_config);
 106
 107 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 108     description => "Some command save/restore state from this location.",
 109     type => 'string',
 110     maxLength => 128,
 111     optional => 1,
 112 });
 113
 114 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 115         description => "Specifies the Qemu machine type.",
 116         type => 'string',
 117         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 118         maxLength => 40,
 119         optional => 1,
 120 });
 121
 122
 123 sub map_storage {
 124     my ($map, $source) = @_;
 125
 126     return $source if !defined($map);
 127
 128     return $map->{entries}->{$source}
 129         if $map->{entries} && defined($map->{entries}->{$source});
 130
 131     return $map->{default} if $map->{default};
 132
 133     # identity (fallback)
 134     return $source;
 135 }
 136
 137 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 138     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 139     type => 'string',
 140     format => 'storage-pair-list',
 141     optional => 1,
 142 });
 143
 144 #no warnings 'redefine';
 145
 146 my $nodename_cache;
 147 sub nodename {
 148     $nodename_cache //= PVE::INotify::nodename();
 149     return $nodename_cache;
 150 }
 151
 152 my $watchdog_fmt = {
 153     model => {
 154         default_key => 1,
 155         type => 'string',
 156         enum => [qw(i6300esb ib700)],
 157         description => "Watchdog type to emulate.",
 158         default => 'i6300esb',
 159         optional => 1,
 160     },
 161     action => {
 162         type => 'string',
 163         enum => [qw(reset shutdown poweroff pause debug none)],
 164         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 165         optional => 1,
 166     },
 167 };
 168 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 169
 170 my $agent_fmt = {
 171     enabled => {
 172         description => "Enable/disable communication with a Qemu Guest Agent (QGA) running in the VM.",
 173         type => 'boolean',
 174         default => 0,
 175         default_key => 1,
 176     },
 177     fstrim_cloned_disks => {
 178         description => "Run fstrim after moving a disk or migrating the VM.",
 179         type => 'boolean',
 180         optional => 1,
 181         default => 0
 182     },
 183     type => {
 184         description => "Select the agent type",
 185         type => 'string',
 186         default => 'virtio',
 187         optional => 1,
 188         enum => [qw(virtio isa)],
 189     },
 190 };
 191
 192 my $vga_fmt = {
 193     type => {
 194         description => "Select the VGA type.",
 195         type => 'string',
 196         default => 'std',
 197         optional => 1,
 198         default_key => 1,
 199         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 200     },
 201     memory => {
 202         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 203         type => 'integer',
 204         optional => 1,
 205         minimum => 4,
 206         maximum => 512,
 207     },
 208 };
 209
 210 my $ivshmem_fmt = {
 211     size => {
 212         type => 'integer',
 213         minimum => 1,
 214         description => "The size of the file in MB.",
 215     },
 216     name => {
 217         type => 'string',
 218         pattern => '[a-zA-Z0-9\-]+',
 219         optional => 1,
 220         format_description => 'string',
 221         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 222     },
 223 };
 224
 225 my $audio_fmt = {
 226     device => {
 227         type => 'string',
 228         enum => [qw(ich9-intel-hda intel-hda AC97)],
 229         description =>  "Configure an audio device."
 230     },
 231     driver =>  {
 232         type => 'string',
 233         enum => ['spice', 'none'],
 234         default => 'spice',
 235         optional => 1,
 236         description => "Driver backend for the audio device."
 237     },
 238 };
 239
 240 my $spice_enhancements_fmt = {
 241     foldersharing => {
 242         type => 'boolean',
 243         optional => 1,
 244         default => '0',
 245         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 246     },
 247     videostreaming =>  {
 248         type => 'string',
 249         enum => ['off', 'all', 'filter'],
 250         default => 'off',
 251         optional => 1,
 252         description => "Enable video streaming. Uses compression for detected video streams."
 253     },
 254 };
 255
 256 my $rng_fmt = {
 257     source => {
 258         type => 'string',
 259         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 260         default_key => 1,
 261         description => "The file on the host to gather entropy from. In most cases '/dev/urandom'"
 262             ." should be preferred over '/dev/random' to avoid entropy-starvation issues on the"
 263             ." host. Using urandom does *not* decrease security in any meaningful way, as it's"
 264             ." still seeded from real entropy, and the bytes provided will most likely be mixed"
 265             ." with real entropy on the guest as well. '/dev/hwrng' can be used to pass through"
 266             ." a hardware RNG from the host.",
 267     },
 268     max_bytes => {
 269         type => 'integer',
 270         description => "Maximum bytes of entropy allowed to get injected into the guest every"
 271             ." 'period' milliseconds. Prefer a lower value when using '/dev/random' as source. Use"
 272             ." `0` to disable limiting (potentially dangerous!).",
 273         optional => 1,
 274
 275         # default is 1 KiB/s, provides enough entropy to the guest to avoid boot-starvation issues
 276         # (e.g. systemd etc...) while allowing no chance of overwhelming the host, provided we're
 277         # reading from /dev/urandom
 278         default => 1024,
 279     },
 280     period => {
 281         type => 'integer',
 282         description => "Every 'period' milliseconds the entropy-injection quota is reset, allowing"
 283             ." the guest to retrieve another 'max_bytes' of entropy.",
 284         optional => 1,
 285         default => 1000,
 286     },
 287 };
 288
 289 my $meta_info_fmt = {
 290     'ctime' => {
 291         type => 'integer',
 292         description => "The guest creation timestamp as UNIX epoch time",
 293         minimum => 0,
 294         optional => 1,
 295     },
 296     'creation-qemu' => {
 297         type => 'string',
 298         description => "The QEMU (machine) version from the time this VM was created.",
 299         pattern => '\d+(\.\d+)+',
 300         optional => 1,
 301     },
 302 };
 303
 304 my $confdesc = {
 305     onboot => {
 306         optional => 1,
 307         type => 'boolean',
 308         description => "Specifies whether a VM will be started during system bootup.",
 309         default => 0,
 310     },
 311     autostart => {
 312         optional => 1,
 313         type => 'boolean',
 314         description => "Automatic restart after crash (currently ignored).",
 315         default => 0,
 316     },
 317     hotplug => {
 318         optional => 1,
 319         type => 'string', format => 'pve-hotplug-features',
 320         description => "Selectively enable hotplug features. This is a comma separated list of"
 321             ." hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable"
 322             ." hotplug completely. Using '1' as value is an alias for the default `network,disk,usb`.",
 323         default => 'network,disk,usb',
 324     },
 325     reboot => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 329         default => 1,
 330     },
 331     lock => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Lock/unlock the VM.",
 335         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 336     },
 337     cpulimit => {
 338         optional => 1,
 339         type => 'number',
 340         description => "Limit of CPU usage.",
 341         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has"
 342             ." total of '2' CPU time. Value '0' indicates no CPU limit.",
 343         minimum => 0,
 344         maximum => 128,
 345         default => 0,
 346     },
 347     cpuunits => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 351         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 352             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 353             ." weights of all the other running VMs.",
 354         minimum => 2,
 355         maximum => 262144,
 356         default => 'cgroup v1: 1024, cgroup v2: 100',
 357     },
 358     memory => {
 359         optional => 1,
 360         type => 'integer',
 361         description => "Amount of RAM for the VM in MB. This is the maximum available memory when"
 362             ." you use the balloon device.",
 363         minimum => 16,
 364         default => 512,
 365     },
 366     balloon => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 370         minimum => 0,
 371     },
 372     shares => {
 373         optional => 1,
 374         type => 'integer',
 375         description => "Amount of memory shares for auto-ballooning. The larger the number is, the"
 376             ." more memory this VM gets. Number is relative to weights of all other running VMs."
 377             ." Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 378         minimum => 0,
 379         maximum => 50000,
 380         default => 1000,
 381     },
 382     keyboard => {
 383         optional => 1,
 384         type => 'string',
 385         description => "Keyboard layout for VNC server. The default is read from the"
 386             ."'/etc/pve/datacenter.cfg' configuration file. It should not be necessary to set it.",
 387         enum => PVE::Tools::kvmkeymaplist(),
 388         default => undef,
 389     },
 390     name => {
 391         optional => 1,
 392         type => 'string', format => 'dns-name',
 393         description => "Set a name for the VM. Only used on the configuration web interface.",
 394     },
 395     scsihw => {
 396         optional => 1,
 397         type => 'string',
 398         description => "SCSI controller model",
 399         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 400         default => 'lsi',
 401     },
 402     description => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Description for the VM. Shown in the web-interface VM's summary."
 406             ." This is saved as comment inside the configuration file.",
 407         maxLength => 1024 * 8,
 408     },
 409     ostype => {
 410         optional => 1,
 411         type => 'string',
 412         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 win11 l24 l26 solaris)],
 413         description => "Specify guest operating system.",
 414         verbose_description => <<EODESC,
 415 Specify guest operating system. This is used to enable special
 416 optimization/features for specific operating systems:
 417
 418 [horizontal]
 419 other;; unspecified OS
 420 wxp;; Microsoft Windows XP
 421 w2k;; Microsoft Windows 2000
 422 w2k3;; Microsoft Windows 2003
 423 w2k8;; Microsoft Windows 2008
 424 wvista;; Microsoft Windows Vista
 425 win7;; Microsoft Windows 7
 426 win8;; Microsoft Windows 8/2012/2012r2
 427 win10;; Microsoft Windows 10/2016/2019
 428 win11;; Microsoft Windows 11/2022
 429 l24;; Linux 2.4 Kernel
 430 l26;; Linux 2.6 - 5.X Kernel
 431 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 432 EODESC
 433     },
 434     boot => {
 435         optional => 1,
 436         type => 'string', format => 'pve-qm-boot',
 437         description => "Specify guest boot order. Use the 'order=' sub-property as usage with no"
 438             ." key or 'legacy=' is deprecated.",
 439     },
 440     bootdisk => {
 441         optional => 1,
 442         type => 'string', format => 'pve-qm-bootdisk',
 443         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 444         pattern => '(ide|sata|scsi|virtio)\d+',
 445     },
 446     smp => {
 447         optional => 1,
 448         type => 'integer',
 449         description => "The number of CPUs. Please use option -sockets instead.",
 450         minimum => 1,
 451         default => 1,
 452     },
 453     sockets => {
 454         optional => 1,
 455         type => 'integer',
 456         description => "The number of CPU sockets.",
 457         minimum => 1,
 458         default => 1,
 459     },
 460     cores => {
 461         optional => 1,
 462         type => 'integer',
 463         description => "The number of cores per socket.",
 464         minimum => 1,
 465         default => 1,
 466     },
 467     numa => {
 468         optional => 1,
 469         type => 'boolean',
 470         description => "Enable/disable NUMA.",
 471         default => 0,
 472     },
 473     hugepages => {
 474         optional => 1,
 475         type => 'string',
 476         description => "Enable/disable hugepages memory.",
 477         enum => [qw(any 2 1024)],
 478     },
 479     keephugepages => {
 480         optional => 1,
 481         type => 'boolean',
 482         default => 0,
 483         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 484             ." after VM shutdown and can be used for subsequent starts.",
 485     },
 486     vcpus => {
 487         optional => 1,
 488         type => 'integer',
 489         description => "Number of hotplugged vcpus.",
 490         minimum => 1,
 491         default => 0,
 492     },
 493     acpi => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Enable/disable ACPI.",
 497         default => 1,
 498     },
 499     agent => {
 500         optional => 1,
 501         description => "Enable/disable communication with the Qemu Guest Agent and its properties.",
 502         type => 'string',
 503         format => $agent_fmt,
 504     },
 505     kvm => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable KVM hardware virtualization.",
 509         default => 1,
 510     },
 511     tdf => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable time drift fix.",
 515         default => 0,
 516     },
 517     localtime => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Set the real time clock (RTC) to local time. This is enabled by default if"
 521             ." the `ostype` indicates a Microsoft Windows OS.",
 522     },
 523     freeze => {
 524         optional => 1,
 525         type => 'boolean',
 526         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 527     },
 528     vga => {
 529         optional => 1,
 530         type => 'string', format => $vga_fmt,
 531         description => "Configure the VGA hardware.",
 532         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 533             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 534             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 535             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 536             ." display server. For win* OS you can select how many independent displays you want,"
 537             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 538             ." using a serial device as terminal.",
 539     },
 540     watchdog => {
 541         optional => 1,
 542         type => 'string', format => 'pve-qm-watchdog',
 543         description => "Create a virtual hardware watchdog device.",
 544         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 545             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 546             ." else the watchdog will reset the guest (or execute the respective action specified)",
 547     },
 548     startdate => {
 549         optional => 1,
 550         type => 'string',
 551         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 552         description => "Set the initial date of the real time clock. Valid format for date are:"
 553             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 554         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 555         default => 'now',
 556     },
 557     startup =>  get_standard_option('pve-startup-order'),
 558     template => {
 559         optional => 1,
 560         type => 'boolean',
 561         description => "Enable/disable Template.",
 562         default => 0,
 563     },
 564     args => {
 565         optional => 1,
 566         type => 'string',
 567         description => "Arbitrary arguments passed to kvm.",
 568         verbose_description => <<EODESCR,
 569 Arbitrary arguments passed to kvm, for example:
 570
 571 args: -no-reboot -no-hpet
 572
 573 NOTE: this option is for experts only.
 574 EODESCR
 575     },
 576     tablet => {
 577         optional => 1,
 578         type => 'boolean',
 579         default => 1,
 580         description => "Enable/disable the USB tablet device.",
 581         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 582             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 583             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 584             ." may consider disabling this to save some context switches. This is turned off by"
 585             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 586     },
 587     migrate_speed => {
 588         optional => 1,
 589         type => 'integer',
 590         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 591         minimum => 0,
 592         default => 0,
 593     },
 594     migrate_downtime => {
 595         optional => 1,
 596         type => 'number',
 597         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 598         minimum => 0,
 599         default => 0.1,
 600     },
 601     cdrom => {
 602         optional => 1,
 603         type => 'string', format => 'pve-qm-ide',
 604         typetext => '<volume>',
 605         description => "This is an alias for option -ide2",
 606     },
 607     cpu => {
 608         optional => 1,
 609         description => "Emulated CPU type.",
 610         type => 'string',
 611         format => 'pve-vm-cpu-conf',
 612     },
 613     parent => get_standard_option('pve-snapshot-name', {
 614         optional => 1,
 615         description => "Parent snapshot name. This is used internally, and should not be modified.",
 616     }),
 617     snaptime => {
 618         optional => 1,
 619         description => "Timestamp for snapshots.",
 620         type => 'integer',
 621         minimum => 0,
 622     },
 623     vmstate => {
 624         optional => 1,
 625         type => 'string', format => 'pve-volume-id',
 626         description => "Reference to a volume which stores the VM state. This is used internally"
 627             ." for snapshots.",
 628     },
 629     vmstatestorage => get_standard_option('pve-storage-id', {
 630         description => "Default storage for VM state volumes/files.",
 631         optional => 1,
 632     }),
 633     runningmachine => get_standard_option('pve-qemu-machine', {
 634         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 635             ." for snapshots.",
 636     }),
 637     runningcpu => {
 638         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 639             ." internally for snapshots.",
 640         optional => 1,
 641         type => 'string',
 642         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 643         format_description => 'QEMU -cpu parameter'
 644     },
 645     machine => get_standard_option('pve-qemu-machine'),
 646     arch => {
 647         description => "Virtual processor architecture. Defaults to the host.",
 648         optional => 1,
 649         type => 'string',
 650         enum => [qw(x86_64 aarch64)],
 651     },
 652     smbios1 => {
 653         description => "Specify SMBIOS type 1 fields.",
 654         type => 'string', format => 'pve-qm-smbios1',
 655         maxLength => 512,
 656         optional => 1,
 657     },
 658     protection => {
 659         optional => 1,
 660         type => 'boolean',
 661         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 662             ." remove disk operations.",
 663         default => 0,
 664     },
 665     bios => {
 666         optional => 1,
 667         type => 'string',
 668         enum => [ qw(seabios ovmf) ],
 669         description => "Select BIOS implementation.",
 670         default => 'seabios',
 671     },
 672     vmgenid => {
 673         type => 'string',
 674         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 675         format_description => 'UUID',
 676         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 677             ." to disable explicitly.",
 678         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 679             ." value identifier to the guest OS. This allows to notify the guest operating system"
 680             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 681             ." execution or creation from a template). The guest operating system notices the"
 682             ." change, and is then able to react as appropriate by marking its copies of"
 683             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 684             ."Note that auto-creation only works when done through API/CLI create or update methods"
 685             .", but not when manually editing the config file.",
 686         default => "1 (autogenerated)",
 687         optional => 1,
 688     },
 689     hookscript => {
 690         type => 'string',
 691         format => 'pve-volume-id',
 692         optional => 1,
 693         description => "Script that will be executed during various steps in the vms lifetime.",
 694     },
 695     ivshmem => {
 696         type => 'string',
 697         format => $ivshmem_fmt,
 698         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 699             ." the host.",
 700         optional => 1,
 701     },
 702     audio0 => {
 703         type => 'string',
 704         format => $audio_fmt,
 705         description => "Configure a audio device, useful in combination with QXL/Spice.",
 706         optional => 1
 707     },
 708     spice_enhancements => {
 709         type => 'string',
 710         format => $spice_enhancements_fmt,
 711         description => "Configure additional enhancements for SPICE.",
 712         optional => 1
 713     },
 714     tags => {
 715         type => 'string', format => 'pve-tag-list',
 716         description => 'Tags of the VM. This is only meta information.',
 717         optional => 1,
 718     },
 719     rng0 => {
 720         type => 'string',
 721         format => $rng_fmt,
 722         description => "Configure a VirtIO-based Random Number Generator.",
 723         optional => 1,
 724     },
 725     meta => {
 726         type => 'string',
 727         format => $meta_info_fmt,
 728         description => "Some (read-only) meta-information about this guest.",
 729         optional => 1,
 730     },
 731 };
 732
 733 my $cicustom_fmt = {
 734     meta => {
 735         type => 'string',
 736         optional => 1,
 737         description => 'Specify a custom file containing all meta data passed to the VM via"
 738             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 739         format => 'pve-volume-id',
 740         format_description => 'volume',
 741     },
 742     network => {
 743         type => 'string',
 744         optional => 1,
 745         description => 'Specify a custom file containing all network data passed to the VM via'
 746             .' cloud-init.',
 747         format => 'pve-volume-id',
 748         format_description => 'volume',
 749     },
 750     user => {
 751         type => 'string',
 752         optional => 1,
 753         description => 'Specify a custom file containing all user data passed to the VM via'
 754             .' cloud-init.',
 755         format => 'pve-volume-id',
 756         format_description => 'volume',
 757     },
 758     vendor => {
 759     type => 'string',
 760     optional => 1,
 761     description => 'Specify a custom file containing all vendor data passed to the VM via'
 762      .' cloud-init.',
 763     format => 'pve-volume-id',
 764     format_description => 'volume',
 765     },
 766 };
 767 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 768
 769 my $confdesc_cloudinit = {
 770     citype => {
 771         optional => 1,
 772         type => 'string',
 773         description => 'Specifies the cloud-init configuration format. The default depends on the'
 774             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 775             .' and `configdrive2` for windows.',
 776         enum => ['configdrive2', 'nocloud', 'opennebula'],
 777     },
 778     ciuser => {
 779         optional => 1,
 780         type => 'string',
 781         description => "cloud-init: User name to change ssh keys and password for instead of the"
 782             ." image's configured default user.",
 783     },
 784     cipassword => {
 785         optional => 1,
 786         type => 'string',
 787         description => 'cloud-init: Password to assign the user. Using this is generally not'
 788             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 789             .' support hashed passwords.',
 790     },
 791     cicustom => {
 792         optional => 1,
 793         type => 'string',
 794         description => 'cloud-init: Specify custom files to replace the automatically generated'
 795             .' ones at start.',
 796         format => 'pve-qm-cicustom',
 797     },
 798     searchdomain => {
 799         optional => 1,
 800         type => 'string',
 801         description => "cloud-init: Sets DNS search domains for a container. Create will'
 802             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 803             .' are set.",
 804     },
 805     nameserver => {
 806         optional => 1,
 807         type => 'string', format => 'address-list',
 808         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 809             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 810             .' are set.",
 811     },
 812     sshkeys => {
 813         optional => 1,
 814         type => 'string',
 815         format => 'urlencoded',
 816         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 817     },
 818 };
 819
 820 # what about other qemu settings ?
 821 #cpu => 'string',
 822 #machine => 'string',
 823 #fda => 'file',
 824 #fdb => 'file',
 825 #mtdblock => 'file',
 826 #sd => 'file',
 827 #pflash => 'file',
 828 #snapshot => 'bool',
 829 #bootp => 'file',
 830 ##tftp => 'dir',
 831 ##smb => 'dir',
 832 #kernel => 'file',
 833 #append => 'string',
 834 #initrd => 'file',
 835 ##soundhw => 'string',
 836
 837 while (my ($k, $v) = each %$confdesc) {
 838     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 839 }
 840
 841 my $MAX_USB_DEVICES = 5;
 842 my $MAX_NETS = 32;
 843 my $MAX_SERIAL_PORTS = 4;
 844 my $MAX_PARALLEL_PORTS = 3;
 845 my $MAX_NUMA = 8;
 846
 847 my $numa_fmt = {
 848     cpus => {
 849         type => "string",
 850         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 851         description => "CPUs accessing this NUMA node.",
 852         format_description => "id[-id];...",
 853     },
 854     memory => {
 855         type => "number",
 856         description => "Amount of memory this NUMA node provides.",
 857         optional => 1,
 858     },
 859     hostnodes => {
 860         type => "string",
 861         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 862         description => "Host NUMA nodes to use.",
 863         format_description => "id[-id];...",
 864         optional => 1,
 865     },
 866     policy => {
 867         type => 'string',
 868         enum => [qw(preferred bind interleave)],
 869         description => "NUMA allocation policy.",
 870         optional => 1,
 871     },
 872 };
 873 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 874 my $numadesc = {
 875     optional => 1,
 876     type => 'string', format => $numa_fmt,
 877     description => "NUMA topology.",
 878 };
 879 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 880
 881 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 882     $confdesc->{"numa$i"} = $numadesc;
 883 }
 884
 885 my $nic_model_list = [
 886     'e1000',
 887     'e1000-82540em',
 888     'e1000-82544gc',
 889     'e1000-82545em',
 890     'e1000e',
 891     'i82551',
 892     'i82557b',
 893     'i82559er',
 894     'ne2k_isa',
 895     'ne2k_pci',
 896     'pcnet',
 897     'rtl8139',
 898     'virtio',
 899     'vmxnet3',
 900 ];
 901 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 902
 903 my $net_fmt_bridge_descr = <<__EOD__;
 904 Bridge to attach the network device to. The Proxmox VE standard bridge
 905 is called 'vmbr0'.
 906
 907 If you do not specify a bridge, we create a kvm user (NATed) network
 908 device, which provides DHCP and DNS services. The following addresses
 909 are used:
 910
 911  10.0.2.2   Gateway
 912  10.0.2.3   DNS Server
 913  10.0.2.4   SMB Server
 914
 915 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 916 __EOD__
 917
 918 my $net_fmt = {
 919     macaddr  => get_standard_option('mac-addr', {
 920         description => "MAC address. That address must be unique withing your network. This is"
 921             ." automatically generated if not specified.",
 922     }),
 923     model => {
 924         type => 'string',
 925         description => "Network Card Model. The 'virtio' model provides the best performance with"
 926             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 927             ." best to use 'e1000'.",
 928         enum => $nic_model_list,
 929         default_key => 1,
 930     },
 931     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 932     bridge => {
 933         type => 'string',
 934         description => $net_fmt_bridge_descr,
 935         format_description => 'bridge',
 936         pattern => '[-_.\w\d]+',
 937         optional => 1,
 938     },
 939     queues => {
 940         type => 'integer',
 941         minimum => 0, maximum => 16,
 942         description => 'Number of packet queues to be used on the device.',
 943         optional => 1,
 944     },
 945     rate => {
 946         type => 'number',
 947         minimum => 0,
 948         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 949         optional => 1,
 950     },
 951     tag => {
 952         type => 'integer',
 953         minimum => 1, maximum => 4094,
 954         description => 'VLAN tag to apply to packets on this interface.',
 955         optional => 1,
 956     },
 957     trunks => {
 958         type => 'string',
 959         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 960         description => 'VLAN trunks to pass through this interface.',
 961         format_description => 'vlanid[;vlanid...]',
 962         optional => 1,
 963     },
 964     firewall => {
 965         type => 'boolean',
 966         description => 'Whether this interface should be protected by the firewall.',
 967         optional => 1,
 968     },
 969     link_down => {
 970         type => 'boolean',
 971         description => 'Whether this interface should be disconnected (like pulling the plug).',
 972         optional => 1,
 973     },
 974     mtu => {
 975         type => 'integer',
 976         minimum => 1, maximum => 65520,
 977         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 978         optional => 1,
 979     },
 980 };
 981
 982 my $netdesc = {
 983     optional => 1,
 984     type => 'string', format => $net_fmt,
 985     description => "Specify network devices.",
 986 };
 987
 988 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 989
 990 my $ipconfig_fmt = {
 991     ip => {
 992         type => 'string',
 993         format => 'pve-ipv4-config',
 994         format_description => 'IPv4Format/CIDR',
 995         description => 'IPv4 address in CIDR format.',
 996         optional => 1,
 997         default => 'dhcp',
 998     },
 999     gw => {
1000         type => 'string',
1001         format => 'ipv4',
1002         format_description => 'GatewayIPv4',
1003         description => 'Default gateway for IPv4 traffic.',
1004         optional => 1,
1005         requires => 'ip',
1006     },
1007     ip6 => {
1008         type => 'string',
1009         format => 'pve-ipv6-config',
1010         format_description => 'IPv6Format/CIDR',
1011         description => 'IPv6 address in CIDR format.',
1012         optional => 1,
1013         default => 'dhcp',
1014     },
1015     gw6 => {
1016         type => 'string',
1017         format => 'ipv6',
1018         format_description => 'GatewayIPv6',
1019         description => 'Default gateway for IPv6 traffic.',
1020         optional => 1,
1021         requires => 'ip6',
1022     },
1023 };
1024 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
1025 my $ipconfigdesc = {
1026     optional => 1,
1027     type => 'string', format => 'pve-qm-ipconfig',
1028     description => <<'EODESCR',
1029 cloud-init: Specify IP addresses and gateways for the corresponding interface.
1030
1031 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
1032
1033 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
1034 gateway should be provided.
1035 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
1036 cloud-init 19.4 or newer.
1037
1038 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1039 dhcp on IPv4.
1040 EODESCR
1041 };
1042 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1043
1044 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1045     $confdesc->{"net$i"} = $netdesc;
1046     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1047 }
1048
1049 foreach my $key (keys %$confdesc_cloudinit) {
1050     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1051 }
1052
1053 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1054 sub verify_volume_id_or_qm_path {
1055     my ($volid, $noerr) = @_;
1056
1057     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1058         return $volid;
1059     }
1060
1061     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1062     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1063     if ($@) {
1064         return if $noerr;
1065         die $@;
1066     }
1067     return $volid;
1068 }
1069
1070 my $usb_fmt = {
1071     host => {
1072         default_key => 1,
1073         type => 'string', format => 'pve-qm-usb-device',
1074         format_description => 'HOSTUSBDEVICE|spice',
1075         description => <<EODESCR,
1076 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1077
1078  'bus-port(.port)*' (decimal numbers) or
1079  'vendor_id:product_id' (hexadeciaml numbers) or
1080  'spice'
1081
1082 You can use the 'lsusb -t' command to list existing usb devices.
1083
1084 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1085 machines - use with special care.
1086
1087 The value 'spice' can be used to add a usb redirection devices for spice.
1088 EODESCR
1089     },
1090     usb3 => {
1091         optional => 1,
1092         type => 'boolean',
1093         description => "Specifies whether if given host option is a USB3 device or port.",
1094         default => 0,
1095     },
1096 };
1097
1098 my $usbdesc = {
1099     optional => 1,
1100     type => 'string', format => $usb_fmt,
1101     description => "Configure an USB device (n is 0 to 4).",
1102 };
1103 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1104
1105 my $serialdesc = {
1106         optional => 1,
1107         type => 'string',
1108         pattern => '(/dev/.+|socket)',
1109         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1110         verbose_description =>  <<EODESCR,
1111 Create a serial device inside the VM (n is 0 to 3), and pass through a
1112 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1113 host side (use 'qm terminal' to open a terminal connection).
1114
1115 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1116 use with special care.
1117
1118 CAUTION: Experimental! User reported problems with this option.
1119 EODESCR
1120 };
1121
1122 my $paralleldesc= {
1123         optional => 1,
1124         type => 'string',
1125         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1126         description =>  "Map host parallel devices (n is 0 to 2).",
1127         verbose_description =>  <<EODESCR,
1128 Map host parallel devices (n is 0 to 2).
1129
1130 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1131 machines - use with special care.
1132
1133 CAUTION: Experimental! User reported problems with this option.
1134 EODESCR
1135 };
1136
1137 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1138     $confdesc->{"parallel$i"} = $paralleldesc;
1139 }
1140
1141 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1142     $confdesc->{"serial$i"} = $serialdesc;
1143 }
1144
1145 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1146     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1147 }
1148
1149 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1150     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1151 }
1152
1153 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1154     $confdesc->{"usb$i"} = $usbdesc;
1155 }
1156
1157 my $boot_fmt = {
1158     legacy => {
1159         optional => 1,
1160         default_key => 1,
1161         type => 'string',
1162         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1163                      . " Deprecated, use 'order=' instead.",
1164         pattern => '[acdn]{1,4}',
1165         format_description => "[acdn]{1,4}",
1166
1167         # note: this is also the fallback if boot: is not given at all
1168         default => 'cdn',
1169     },
1170     order => {
1171         optional => 1,
1172         type => 'string',
1173         format => 'pve-qm-bootdev-list',
1174         format_description => "device[;device...]",
1175         description => <<EODESC,
1176 The guest will attempt to boot from devices in the order they appear here.
1177
1178 Disks, optical drives and passed-through storage USB devices will be directly
1179 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1180 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1181
1182 Note that only devices in this list will be marked as bootable and thus loaded
1183 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1184 (e.g. software-raid), you need to specify all of them here.
1185
1186 Overrides the deprecated 'legacy=[acdn]*' value when given.
1187 EODESC
1188     },
1189 };
1190 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1191
1192 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1193 sub verify_bootdev {
1194     my ($dev, $noerr) = @_;
1195
1196     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1197     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1198
1199     my $check = sub {
1200         my ($base) = @_;
1201         return 0 if $dev !~ m/^$base\d+$/;
1202         return 0 if !$confdesc->{$dev};
1203         return 1;
1204     };
1205
1206     return $dev if $check->("net");
1207     return $dev if $check->("usb");
1208     return $dev if $check->("hostpci");
1209
1210     return if $noerr;
1211     die "invalid boot device '$dev'\n";
1212 }
1213
1214 sub print_bootorder {
1215     my ($devs) = @_;
1216     return "" if !@$devs;
1217     my $data = { order => join(';', @$devs) };
1218     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1219 }
1220
1221 my $kvm_api_version = 0;
1222
1223 sub kvm_version {
1224     return $kvm_api_version if $kvm_api_version;
1225
1226     open my $fh, '<', '/dev/kvm' or return;
1227
1228     # 0xae00 => KVM_GET_API_VERSION
1229     $kvm_api_version = ioctl($fh, 0xae00, 0);
1230     close($fh);
1231
1232     return $kvm_api_version;
1233 }
1234
1235 my $kvm_user_version = {};
1236 my $kvm_mtime = {};
1237
1238 sub kvm_user_version {
1239     my ($binary) = @_;
1240
1241     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1242     my $st = stat($binary);
1243
1244     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1245     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1246         $cachedmtime == $st->mtime;
1247
1248     $kvm_user_version->{$binary} = 'unknown';
1249     $kvm_mtime->{$binary} = $st->mtime;
1250
1251     my $code = sub {
1252         my $line = shift;
1253         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1254             $kvm_user_version->{$binary} = $2;
1255         }
1256     };
1257
1258     eval { run_command([$binary, '--version'], outfunc => $code); };
1259     warn $@ if $@;
1260
1261     return $kvm_user_version->{$binary};
1262
1263 }
1264 my sub extract_version {
1265     my ($machine_type, $version) = @_;
1266     $version = kvm_user_version() if !defined($version);
1267     return PVE::QemuServer::Machine::extract_version($machine_type, $version)
1268 }
1269
1270 sub kernel_has_vhost_net {
1271     return -c '/dev/vhost-net';
1272 }
1273
1274 sub option_exists {
1275     my $key = shift;
1276     return defined($confdesc->{$key});
1277 }
1278
1279 my $cdrom_path;
1280 sub get_cdrom_path {
1281
1282     return  $cdrom_path if $cdrom_path;
1283
1284     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1285     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1286     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1287 }
1288
1289 sub get_iso_path {
1290     my ($storecfg, $vmid, $cdrom) = @_;
1291
1292     if ($cdrom eq 'cdrom') {
1293         return get_cdrom_path();
1294     } elsif ($cdrom eq 'none') {
1295         return '';
1296     } elsif ($cdrom =~ m|^/|) {
1297         return $cdrom;
1298     } else {
1299         return PVE::Storage::path($storecfg, $cdrom);
1300     }
1301 }
1302
1303 # try to convert old style file names to volume IDs
1304 sub filename_to_volume_id {
1305     my ($vmid, $file, $media) = @_;
1306
1307      if (!($file eq 'none' || $file eq 'cdrom' ||
1308           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1309
1310         return if $file =~ m|/|;
1311
1312         if ($media && $media eq 'cdrom') {
1313             $file = "local:iso/$file";
1314         } else {
1315             $file = "local:$vmid/$file";
1316         }
1317     }
1318
1319     return $file;
1320 }
1321
1322 sub verify_media_type {
1323     my ($opt, $vtype, $media) = @_;
1324
1325     return if !$media;
1326
1327     my $etype;
1328     if ($media eq 'disk') {
1329         $etype = 'images';
1330     } elsif ($media eq 'cdrom') {
1331         $etype = 'iso';
1332     } else {
1333         die "internal error";
1334     }
1335
1336     return if ($vtype eq $etype);
1337
1338     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1339 }
1340
1341 sub cleanup_drive_path {
1342     my ($opt, $storecfg, $drive) = @_;
1343
1344     # try to convert filesystem paths to volume IDs
1345
1346     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1347         ($drive->{file} !~ m|^/dev/.+|) &&
1348         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1349         ($drive->{file} !~ m/^\d+$/)) {
1350         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1351         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1352             if !$vtype;
1353         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1354         verify_media_type($opt, $vtype, $drive->{media});
1355         $drive->{file} = $volid;
1356     }
1357
1358     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1359 }
1360
1361 sub parse_hotplug_features {
1362     my ($data) = @_;
1363
1364     my $res = {};
1365
1366     return $res if $data eq '0';
1367
1368     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1369
1370     foreach my $feature (PVE::Tools::split_list($data)) {
1371         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1372             $res->{$1} = 1;
1373         } else {
1374             die "invalid hotplug feature '$feature'\n";
1375         }
1376     }
1377     return $res;
1378 }
1379
1380 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1381 sub pve_verify_hotplug_features {
1382     my ($value, $noerr) = @_;
1383
1384     return $value if parse_hotplug_features($value);
1385
1386     return if $noerr;
1387
1388     die "unable to parse hotplug option\n";
1389 }
1390
1391 sub scsi_inquiry {
1392     my($fh, $noerr) = @_;
1393
1394     my $SG_IO = 0x2285;
1395     my $SG_GET_VERSION_NUM = 0x2282;
1396
1397     my $versionbuf = "\x00" x 8;
1398     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1399     if (!$ret) {
1400         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1401         return;
1402     }
1403     my $version = unpack("I", $versionbuf);
1404     if ($version < 30000) {
1405         die "scsi generic interface too old\n"  if !$noerr;
1406         return;
1407     }
1408
1409     my $buf = "\x00" x 36;
1410     my $sensebuf = "\x00" x 8;
1411     my $cmd = pack("C x3 C x1", 0x12, 36);
1412
1413     # see /usr/include/scsi/sg.h
1414     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1415
1416     my $packet = pack(
1417         $sg_io_hdr_t, ord('S'), -3, length($cmd), length($sensebuf), 0, length($buf), $buf, $cmd, $sensebuf, 6000
1418     );
1419
1420     $ret = ioctl($fh, $SG_IO, $packet);
1421     if (!$ret) {
1422         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1423         return;
1424     }
1425
1426     my @res = unpack($sg_io_hdr_t, $packet);
1427     if ($res[17] || $res[18]) {
1428         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1429         return;
1430     }
1431
1432     my $res = {};
1433     $res->@{qw(type removable vendor product revision)} = unpack("C C x6 A8 A16 A4", $buf);
1434
1435     $res->{removable} = $res->{removable} & 128 ? 1 : 0;
1436     $res->{type} &= 0x1F;
1437
1438     return $res;
1439 }
1440
1441 sub path_is_scsi {
1442     my ($path) = @_;
1443
1444     my $fh = IO::File->new("+<$path") || return;
1445     my $res = scsi_inquiry($fh, 1);
1446     close($fh);
1447
1448     return $res;
1449 }
1450
1451 sub print_tabletdevice_full {
1452     my ($conf, $arch) = @_;
1453
1454     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1455
1456     # we use uhci for old VMs because tablet driver was buggy in older qemu
1457     my $usbbus;
1458     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1459         $usbbus = 'ehci';
1460     } else {
1461         $usbbus = 'uhci';
1462     }
1463
1464     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1465 }
1466
1467 sub print_keyboarddevice_full {
1468     my ($conf, $arch) = @_;
1469
1470     return if $arch ne 'aarch64';
1471
1472     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1473 }
1474
1475 my sub get_drive_id {
1476     my ($drive) = @_;
1477     return "$drive->{interface}$drive->{index}";
1478 }
1479
1480 sub print_drivedevice_full {
1481     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1482
1483     my $device = '';
1484     my $maxdev = 0;
1485
1486     my $drive_id = get_drive_id($drive);
1487     if ($drive->{interface} eq 'virtio') {
1488         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1489         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1490         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1491     } elsif ($drive->{interface} eq 'scsi') {
1492
1493         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1494         my $unit = $drive->{index} % $maxdev;
1495         my $devicetype = 'hd';
1496         my $path = '';
1497         if (drive_is_cdrom($drive)) {
1498             $devicetype = 'cd';
1499         } else {
1500             if ($drive->{file} =~ m|^/|) {
1501                 $path = $drive->{file};
1502                 if (my $info = path_is_scsi($path)) {
1503                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1504                         $devicetype = 'block';
1505                     } elsif ($info->{type} == 1) { # tape
1506                         $devicetype = 'generic';
1507                     }
1508                 }
1509             } else {
1510                  $path = PVE::Storage::path($storecfg, $drive->{file});
1511             }
1512
1513             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1514             my $version = extract_version($machine_type, kvm_user_version());
1515             if ($path =~ m/^iscsi\:\/\// &&
1516                !min_version($version, 4, 1)) {
1517                 $devicetype = 'generic';
1518             }
1519         }
1520
1521         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1522             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1523         } else {
1524             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1525                 .",lun=$drive->{index}";
1526         }
1527         $device .= ",drive=drive-$drive_id,id=$drive_id";
1528
1529         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1530             $device .= ",rotation_rate=1";
1531         }
1532         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1533
1534     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1535         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1536         my $controller = int($drive->{index} / $maxdev);
1537         my $unit = $drive->{index} % $maxdev;
1538         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1539
1540         $device = "ide-$devicetype";
1541         if ($drive->{interface} eq 'ide') {
1542             $device .= ",bus=ide.$controller,unit=$unit";
1543         } else {
1544             $device .= ",bus=ahci$controller.$unit";
1545         }
1546         $device .= ",drive=drive-$drive_id,id=$drive_id";
1547
1548         if ($devicetype eq 'hd') {
1549             if (my $model = $drive->{model}) {
1550                 $model = URI::Escape::uri_unescape($model);
1551                 $device .= ",model=$model";
1552             }
1553             if ($drive->{ssd}) {
1554                 $device .= ",rotation_rate=1";
1555             }
1556         }
1557         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1558     } elsif ($drive->{interface} eq 'usb') {
1559         die "implement me";
1560         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1561     } else {
1562         die "unsupported interface type";
1563     }
1564
1565     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1566
1567     if (my $serial = $drive->{serial}) {
1568         $serial = URI::Escape::uri_unescape($serial);
1569         $device .= ",serial=$serial";
1570     }
1571
1572
1573     return $device;
1574 }
1575
1576 sub get_initiator_name {
1577     my $initiator;
1578
1579     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1580     while (defined(my $line = <$fh>)) {
1581         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1582         $initiator = $1;
1583         last;
1584     }
1585     $fh->close();
1586
1587     return $initiator;
1588 }
1589
1590 sub print_drive_commandline_full {
1591     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1592
1593     my $path;
1594     my $volid = $drive->{file};
1595     my $format = $drive->{format};
1596     my $drive_id = get_drive_id($drive);
1597
1598     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1599     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1600
1601     if (drive_is_cdrom($drive)) {
1602         $path = get_iso_path($storecfg, $vmid, $volid);
1603         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1604     } else {
1605         if ($storeid) {
1606             $path = PVE::Storage::path($storecfg, $volid);
1607             $format //= qemu_img_format($scfg, $volname);
1608         } else {
1609             $path = $volid;
1610             $format //= "raw";
1611         }
1612    }
1613
1614    my $is_rbd = $path =~ m/^rbd:/;
1615
1616     my $opts = '';
1617     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1618     foreach my $o (@qemu_drive_options) {
1619         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1620     }
1621
1622     # snapshot only accepts on|off
1623     if (defined($drive->{snapshot})) {
1624         my $v = $drive->{snapshot} ? 'on' : 'off';
1625         $opts .= ",snapshot=$v";
1626     }
1627
1628     if (defined($drive->{ro})) { # ro maps to QEMUs `readonly`, which accepts `on` or `off` only
1629         $opts .= ",readonly=" . ($drive->{ro} ? 'on' : 'off');
1630     }
1631
1632     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1633         my ($dir, $qmpname) = @$type;
1634         if (my $v = $drive->{"mbps$dir"}) {
1635             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1636         }
1637         if (my $v = $drive->{"mbps${dir}_max"}) {
1638             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1639         }
1640         if (my $v = $drive->{"bps${dir}_max_length"}) {
1641             $opts .= ",throttling.bps$qmpname-max-length=$v";
1642         }
1643         if (my $v = $drive->{"iops${dir}"}) {
1644             $opts .= ",throttling.iops$qmpname=$v";
1645         }
1646         if (my $v = $drive->{"iops${dir}_max"}) {
1647             $opts .= ",throttling.iops$qmpname-max=$v";
1648         }
1649         if (my $v = $drive->{"iops${dir}_max_length"}) {
1650             $opts .= ",throttling.iops$qmpname-max-length=$v";
1651         }
1652     }
1653
1654     if ($pbs_name) {
1655         $format = "rbd" if $is_rbd;
1656         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1657             if !$format;
1658         $opts .= ",format=alloc-track,file.driver=$format";
1659     } elsif ($format) {
1660         $opts .= ",format=$format";
1661     }
1662
1663     my $cache_direct = 0;
1664
1665     if (my $cache = $drive->{cache}) {
1666         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1667     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1668         $opts .= ",cache=none";
1669         $cache_direct = 1;
1670     }
1671
1672     # io_uring with cache mode writeback or writethrough on krbd will hang...
1673     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1674
1675     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1676     # sometimes, just plain disable...
1677     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1678
1679     if (!$drive->{aio}) {
1680         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1681             # io_uring supports all cache modes
1682             $opts .= ",aio=io_uring";
1683         } else {
1684             # aio native works only with O_DIRECT
1685             if($cache_direct) {
1686                 $opts .= ",aio=native";
1687             } else {
1688                 $opts .= ",aio=threads";
1689             }
1690         }
1691     }
1692
1693     if (!drive_is_cdrom($drive)) {
1694         my $detectzeroes;
1695         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1696             $detectzeroes = 'off';
1697         } elsif ($drive->{discard}) {
1698             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1699         } else {
1700             # This used to be our default with discard not being specified:
1701             $detectzeroes = 'on';
1702         }
1703
1704         # note: 'detect-zeroes' works per blockdev and we want it to persist
1705         # after the alloc-track is removed, so put it on 'file' directly
1706         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1707         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1708     }
1709
1710     if ($pbs_name) {
1711         $opts .= ",backing=$pbs_name";
1712         $opts .= ",auto-remove=on";
1713     }
1714
1715     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1716     my $file_param = "file";
1717     if ($pbs_name) {
1718         # non-rbd drivers require the underlying file to be a seperate block
1719         # node, so add a second .file indirection
1720         $file_param .= ".file" if !$is_rbd;
1721         $file_param .= ".filename";
1722     }
1723     my $pathinfo = $path ? "$file_param=$path," : '';
1724
1725     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1726 }
1727
1728 sub print_pbs_blockdev {
1729     my ($pbs_conf, $pbs_name) = @_;
1730     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1731     $blockdev .= ",repository=$pbs_conf->{repository}";
1732     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1733     $blockdev .= ",archive=$pbs_conf->{archive}";
1734     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1735     return $blockdev;
1736 }
1737
1738 sub print_netdevice_full {
1739     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1740
1741     my $device = $net->{model};
1742     if ($net->{model} eq 'virtio') {
1743          $device = 'virtio-net-pci';
1744      };
1745
1746     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1747     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1748     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1749         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1750         # and out of each queue plus one config interrupt and control vector queue
1751         my $vectors = $net->{queues} * 2 + 2;
1752         $tmpstr .= ",vectors=$vectors,mq=on";
1753     }
1754     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1755
1756     if (my $mtu = $net->{mtu}) {
1757         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1758             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1759             if ($mtu == 1) {
1760                  $mtu = $bridge_mtu;
1761             } elsif ($mtu < 576) {
1762                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1763             } elsif ($mtu > $bridge_mtu) {
1764                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1765             }
1766             $tmpstr .= ",host_mtu=$mtu";
1767         } else {
1768             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1769         }
1770     }
1771
1772     if ($use_old_bios_files) {
1773         my $romfile;
1774         if ($device eq 'virtio-net-pci') {
1775             $romfile = 'pxe-virtio.rom';
1776         } elsif ($device eq 'e1000') {
1777             $romfile = 'pxe-e1000.rom';
1778         } elsif ($device eq 'e1000e') {
1779             $romfile = 'pxe-e1000e.rom';
1780         } elsif ($device eq 'ne2k') {
1781             $romfile = 'pxe-ne2k_pci.rom';
1782         } elsif ($device eq 'pcnet') {
1783             $romfile = 'pxe-pcnet.rom';
1784         } elsif ($device eq 'rtl8139') {
1785             $romfile = 'pxe-rtl8139.rom';
1786         }
1787         $tmpstr .= ",romfile=$romfile" if $romfile;
1788     }
1789
1790     return $tmpstr;
1791 }
1792
1793 sub print_netdev_full {
1794     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1795
1796     my $i = '';
1797     if ($netid =~ m/^net(\d+)$/) {
1798         $i = int($1);
1799     }
1800
1801     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1802
1803     my $ifname = "tap${vmid}i$i";
1804
1805     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1806     die "interface name '$ifname' is too long (max 15 character)\n"
1807         if length($ifname) >= 16;
1808
1809     my $vhostparam = '';
1810     if (is_native($arch)) {
1811         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1812     }
1813
1814     my $vmname = $conf->{name} || "vm$vmid";
1815
1816     my $netdev = "";
1817     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1818
1819     if ($net->{bridge}) {
1820         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1821             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1822     } else {
1823         $netdev = "type=user,id=$netid,hostname=$vmname";
1824     }
1825
1826     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1827
1828     return $netdev;
1829 }
1830
1831 my $vga_map = {
1832     'cirrus' => 'cirrus-vga',
1833     'std' => 'VGA',
1834     'vmware' => 'vmware-svga',
1835     'virtio' => 'virtio-vga',
1836 };
1837
1838 sub print_vga_device {
1839     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1840
1841     my $type = $vga_map->{$vga->{type}};
1842     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1843         $type = 'virtio-gpu';
1844     }
1845     my $vgamem_mb = $vga->{memory};
1846
1847     my $max_outputs = '';
1848     if ($qxlnum) {
1849         $type = $id ? 'qxl' : 'qxl-vga';
1850
1851         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1852             # set max outputs so linux can have up to 4 qxl displays with one device
1853             if (min_version($machine_version, 4, 1)) {
1854                 $max_outputs = ",max_outputs=4";
1855             }
1856         }
1857     }
1858
1859     die "no devicetype for $vga->{type}\n" if !$type;
1860
1861     my $memory = "";
1862     if ($vgamem_mb) {
1863         if ($vga->{type} eq 'virtio') {
1864             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1865             $memory = ",max_hostmem=$bytes";
1866         } elsif ($qxlnum) {
1867             # from https://www.spice-space.org/multiple-monitors.html
1868             $memory = ",vgamem_mb=$vga->{memory}";
1869             my $ram = $vgamem_mb * 4;
1870             my $vram = $vgamem_mb * 2;
1871             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1872         } else {
1873             $memory = ",vgamem_mb=$vga->{memory}";
1874         }
1875     } elsif ($qxlnum && $id) {
1876         $memory = ",ram_size=67108864,vram_size=33554432";
1877     }
1878
1879     my $edidoff = "";
1880     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1881         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1882     }
1883
1884     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1885     my $vgaid = "vga" . ($id // '');
1886     my $pciaddr;
1887     if ($q35 && $vgaid eq 'vga') {
1888         # the first display uses pcie.0 bus on q35 machines
1889         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1890     } else {
1891         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1892     }
1893
1894     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1895 }
1896
1897 sub parse_number_sets {
1898     my ($set) = @_;
1899     my $res = [];
1900     foreach my $part (split(/;/, $set)) {
1901         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1902             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1903             push @$res, [ $1, $2 ];
1904         } else {
1905             die "invalid range: $part\n";
1906         }
1907     }
1908     return $res;
1909 }
1910
1911 sub parse_numa {
1912     my ($data) = @_;
1913
1914     my $res = parse_property_string($numa_fmt, $data);
1915     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1916     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1917     return $res;
1918 }
1919
1920 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1921 sub parse_net {
1922     my ($data) = @_;
1923
1924     my $res = eval { parse_property_string($net_fmt, $data) };
1925     if ($@) {
1926         warn $@;
1927         return;
1928     }
1929     if (!defined($res->{macaddr})) {
1930         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1931         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1932     }
1933     return $res;
1934 }
1935
1936 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1937 sub parse_ipconfig {
1938     my ($data) = @_;
1939
1940     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1941     if ($@) {
1942         warn $@;
1943         return;
1944     }
1945
1946     if ($res->{gw} && !$res->{ip}) {
1947         warn 'gateway specified without specifying an IP address';
1948         return;
1949     }
1950     if ($res->{gw6} && !$res->{ip6}) {
1951         warn 'IPv6 gateway specified without specifying an IPv6 address';
1952         return;
1953     }
1954     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1955         warn 'gateway specified together with DHCP';
1956         return;
1957     }
1958     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1959         # gw6 + auto/dhcp
1960         warn "IPv6 gateway specified together with $res->{ip6} address";
1961         return;
1962     }
1963
1964     if (!$res->{ip} && !$res->{ip6}) {
1965         return { ip => 'dhcp', ip6 => 'dhcp' };
1966     }
1967
1968     return $res;
1969 }
1970
1971 sub print_net {
1972     my $net = shift;
1973
1974     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1975 }
1976
1977 sub add_random_macs {
1978     my ($settings) = @_;
1979
1980     foreach my $opt (keys %$settings) {
1981         next if $opt !~ m/^net(\d+)$/;
1982         my $net = parse_net($settings->{$opt});
1983         next if !$net;
1984         $settings->{$opt} = print_net($net);
1985     }
1986 }
1987
1988 sub vm_is_volid_owner {
1989     my ($storecfg, $vmid, $volid) = @_;
1990
1991     if ($volid !~  m|^/|) {
1992         my ($path, $owner);
1993         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1994         if ($owner && ($owner == $vmid)) {
1995             return 1;
1996         }
1997     }
1998
1999     return;
2000 }
2001
2002 sub vmconfig_register_unused_drive {
2003     my ($storecfg, $vmid, $conf, $drive) = @_;
2004
2005     if (drive_is_cloudinit($drive)) {
2006         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2007         warn $@ if $@;
2008     } elsif (!drive_is_cdrom($drive)) {
2009         my $volid = $drive->{file};
2010         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2011             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2012         }
2013     }
2014 }
2015
2016 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2017 my $smbios1_fmt = {
2018     uuid => {
2019         type => 'string',
2020         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2021         format_description => 'UUID',
2022         description => "Set SMBIOS1 UUID.",
2023         optional => 1,
2024     },
2025     version => {
2026         type => 'string',
2027         pattern => '[A-Za-z0-9+\/]+={0,2}',
2028         format_description => 'Base64 encoded string',
2029         description => "Set SMBIOS1 version.",
2030         optional => 1,
2031     },
2032     serial => {
2033         type => 'string',
2034         pattern => '[A-Za-z0-9+\/]+={0,2}',
2035         format_description => 'Base64 encoded string',
2036         description => "Set SMBIOS1 serial number.",
2037         optional => 1,
2038     },
2039     manufacturer => {
2040         type => 'string',
2041         pattern => '[A-Za-z0-9+\/]+={0,2}',
2042         format_description => 'Base64 encoded string',
2043         description => "Set SMBIOS1 manufacturer.",
2044         optional => 1,
2045     },
2046     product => {
2047         type => 'string',
2048         pattern => '[A-Za-z0-9+\/]+={0,2}',
2049         format_description => 'Base64 encoded string',
2050         description => "Set SMBIOS1 product ID.",
2051         optional => 1,
2052     },
2053     sku => {
2054         type => 'string',
2055         pattern => '[A-Za-z0-9+\/]+={0,2}',
2056         format_description => 'Base64 encoded string',
2057         description => "Set SMBIOS1 SKU string.",
2058         optional => 1,
2059     },
2060     family => {
2061         type => 'string',
2062         pattern => '[A-Za-z0-9+\/]+={0,2}',
2063         format_description => 'Base64 encoded string',
2064         description => "Set SMBIOS1 family string.",
2065         optional => 1,
2066     },
2067     base64 => {
2068         type => 'boolean',
2069         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2070         optional => 1,
2071     },
2072 };
2073
2074 sub parse_smbios1 {
2075     my ($data) = @_;
2076
2077     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2078     warn $@ if $@;
2079     return $res;
2080 }
2081
2082 sub print_smbios1 {
2083     my ($smbios1) = @_;
2084     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2085 }
2086
2087 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2088
2089 sub parse_watchdog {
2090     my ($value) = @_;
2091
2092     return if !$value;
2093
2094     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2095     warn $@ if $@;
2096     return $res;
2097 }
2098
2099 sub parse_guest_agent {
2100     my ($conf) = @_;
2101
2102     return {} if !defined($conf->{agent});
2103
2104     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2105     warn $@ if $@;
2106
2107     # if the agent is disabled ignore the other potentially set properties
2108     return {} if !$res->{enabled};
2109     return $res;
2110 }
2111
2112 sub get_qga_key {
2113     my ($conf, $key) = @_;
2114     return undef if !defined($conf->{agent});
2115
2116     my $agent = parse_guest_agent($conf);
2117     return $agent->{$key};
2118 }
2119
2120 sub parse_vga {
2121     my ($value) = @_;
2122
2123     return {} if !$value;
2124     my $res = eval { parse_property_string($vga_fmt, $value) };
2125     warn $@ if $@;
2126     return $res;
2127 }
2128
2129 sub parse_rng {
2130     my ($value) = @_;
2131
2132     return if !$value;
2133
2134     my $res = eval { parse_property_string($rng_fmt, $value) };
2135     warn $@ if $@;
2136     return $res;
2137 }
2138
2139 sub parse_meta_info {
2140     my ($value) = @_;
2141
2142     return if !$value;
2143
2144     my $res = eval { parse_property_string($meta_info_fmt, $value) };
2145     warn $@ if $@;
2146     return $res;
2147 }
2148
2149 sub new_meta_info_string {
2150     my () = @_; # for now do not allow to override any value
2151
2152     return PVE::JSONSchema::print_property_string(
2153         {
2154             'creation-qemu' => kvm_user_version(),
2155             ctime => "". int(time()),
2156         },
2157         $meta_info_fmt
2158     );
2159 }
2160
2161 sub qemu_created_version_fixups {
2162     my ($conf, $forcemachine, $kvmver) = @_;
2163
2164     my $meta = parse_meta_info($conf->{meta}) // {};
2165     my $forced_vers = PVE::QemuServer::Machine::extract_version($forcemachine);
2166
2167     # check if we need to apply some handling for VMs that always use the latest machine version but
2168     # had a machine version transition happen that affected HW such that, e.g., an OS config change
2169     # would be required (we do not want to pin machine version for non-windows OS type)
2170     if (
2171         (!defined($conf->{machine}) || $conf->{machine} =~ m/^(?:pc|q35|virt)$/) # non-versioned machine
2172         && (!defined($meta->{'creation-qemu'}) || !min_version($meta->{'creation-qemu'}, 6, 1)) # created before 6.1
2173         && (!$forced_vers || min_version($forced_vers, 6, 1)) # handle snapshot-rollback/migrations
2174         && min_version($kvmver, 6, 1) # only need to apply the change since 6.1
2175     ) {
2176         my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
2177         if ($q35 && $conf->{ostype} && $conf->{ostype} eq 'l26') {
2178             # this changed to default-on in Q 6.1 for q35 machines, it will mess with PCI slot view
2179             # and thus with the predictable interface naming of systemd
2180             return ['-global', 'ICH9-LPC.acpi-pci-hotplug-with-bridge-support=off'];
2181         }
2182     }
2183     return;
2184 }
2185
2186 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2187 sub verify_usb_device {
2188     my ($value, $noerr) = @_;
2189
2190     return $value if parse_usb_device($value);
2191
2192     return if $noerr;
2193
2194     die "unable to parse usb device\n";
2195 }
2196
2197 # add JSON properties for create and set function
2198 sub json_config_properties {
2199     my $prop = shift;
2200
2201     my $skip_json_config_opts = {
2202         parent => 1,
2203         snaptime => 1,
2204         vmstate => 1,
2205         runningmachine => 1,
2206         runningcpu => 1,
2207         meta => 1,
2208     };
2209
2210     foreach my $opt (keys %$confdesc) {
2211         next if $skip_json_config_opts->{$opt};
2212         $prop->{$opt} = $confdesc->{$opt};
2213     }
2214
2215     return $prop;
2216 }
2217
2218 # return copy of $confdesc_cloudinit to generate documentation
2219 sub cloudinit_config_properties {
2220
2221     return dclone($confdesc_cloudinit);
2222 }
2223
2224 sub check_type {
2225     my ($key, $value) = @_;
2226
2227     die "unknown setting '$key'\n" if !$confdesc->{$key};
2228
2229     my $type = $confdesc->{$key}->{type};
2230
2231     if (!defined($value)) {
2232         die "got undefined value\n";
2233     }
2234
2235     if ($value =~ m/[\n\r]/) {
2236         die "property contains a line feed\n";
2237     }
2238
2239     if ($type eq 'boolean') {
2240         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2241         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2242         die "type check ('boolean') failed - got '$value'\n";
2243     } elsif ($type eq 'integer') {
2244         return int($1) if $value =~ m/^(\d+)$/;
2245         die "type check ('integer') failed - got '$value'\n";
2246     } elsif ($type eq 'number') {
2247         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2248         die "type check ('number') failed - got '$value'\n";
2249     } elsif ($type eq 'string') {
2250         if (my $fmt = $confdesc->{$key}->{format}) {
2251             PVE::JSONSchema::check_format($fmt, $value);
2252             return $value;
2253         }
2254         $value =~ s/^\"(.*)\"$/$1/;
2255         return $value;
2256     } else {
2257         die "internal error"
2258     }
2259 }
2260
2261 sub destroy_vm {
2262     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2263
2264     my $conf = PVE::QemuConfig->load_config($vmid);
2265
2266     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2267
2268     if ($conf->{template}) {
2269         # check if any base image is still used by a linked clone
2270         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2271                 my ($ds, $drive) = @_;
2272                 return if drive_is_cdrom($drive);
2273
2274                 my $volid = $drive->{file};
2275                 return if !$volid || $volid =~ m|^/|;
2276
2277                 die "base volume '$volid' is still in use by linked cloned\n"
2278                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2279
2280         });
2281     }
2282
2283     my $volids = {};
2284     my $remove_owned_drive = sub {
2285         my ($ds, $drive) = @_;
2286         return if drive_is_cdrom($drive, 1);
2287
2288         my $volid = $drive->{file};
2289         return if !$volid || $volid =~ m|^/|;
2290         return if $volids->{$volid};
2291
2292         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2293         return if !$path || !$owner || ($owner != $vmid);
2294
2295         $volids->{$volid} = 1;
2296         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2297         warn "Could not remove disk '$volid', check manually: $@" if $@;
2298     };
2299
2300     # only remove disks owned by this VM (referenced in the config)
2301     my $include_opts = {
2302         include_unused => 1,
2303         extra_keys => ['vmstate'],
2304     };
2305     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2306
2307     for my $snap (values %{$conf->{snapshots}}) {
2308         next if !defined($snap->{vmstate});
2309         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2310         next if !defined($drive);
2311         $remove_owned_drive->('vmstate', $drive);
2312     }
2313
2314     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2315
2316     if ($purge_unreferenced) { # also remove unreferenced disk
2317         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2318         PVE::Storage::foreach_volid($vmdisks, sub {
2319             my ($volid, $sid, $volname, $d) = @_;
2320             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2321             warn $@ if $@;
2322         });
2323     }
2324
2325     if (defined $replacement_conf) {
2326         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2327     } else {
2328         PVE::QemuConfig->destroy_config($vmid);
2329     }
2330 }
2331
2332 sub parse_vm_config {
2333     my ($filename, $raw) = @_;
2334
2335     return if !defined($raw);
2336
2337     my $res = {
2338         digest => Digest::SHA::sha1_hex($raw),
2339         snapshots => {},
2340         pending => {},
2341     };
2342
2343     $filename =~ m|/qemu-server/(\d+)\.conf$|
2344         || die "got strange filename '$filename'";
2345
2346     my $vmid = $1;
2347
2348     my $conf = $res;
2349     my $descr;
2350     my $section = '';
2351
2352     my @lines = split(/\n/, $raw);
2353     foreach my $line (@lines) {
2354         next if $line =~ m/^\s*$/;
2355
2356         if ($line =~ m/^\[PENDING\]\s*$/i) {
2357             $section = 'pending';
2358             if (defined($descr)) {
2359                 $descr =~ s/\s+$//;
2360                 $conf->{description} = $descr;
2361             }
2362             $descr = undef;
2363             $conf = $res->{$section} = {};
2364             next;
2365
2366         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2367             $section = $1;
2368             if (defined($descr)) {
2369                 $descr =~ s/\s+$//;
2370                 $conf->{description} = $descr;
2371             }
2372             $descr = undef;
2373             $conf = $res->{snapshots}->{$section} = {};
2374             next;
2375         }
2376
2377         if ($line =~ m/^\#(.*)\s*$/) {
2378             $descr = '' if !defined($descr);
2379             $descr .= PVE::Tools::decode_text($1) . "\n";
2380             next;
2381         }
2382
2383         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2384             $descr = '' if !defined($descr);
2385             $descr .= PVE::Tools::decode_text($2);
2386         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2387             $conf->{snapstate} = $1;
2388         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2389             my $key = $1;
2390             my $value = $2;
2391             $conf->{$key} = $value;
2392         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2393             my $value = $1;
2394             if ($section eq 'pending') {
2395                 $conf->{delete} = $value; # we parse this later
2396             } else {
2397                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2398             }
2399         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2400             my $key = $1;
2401             my $value = $2;
2402             eval { $value = check_type($key, $value); };
2403             if ($@) {
2404                 warn "vm $vmid - unable to parse value of '$key' - $@";
2405             } else {
2406                 $key = 'ide2' if $key eq 'cdrom';
2407                 my $fmt = $confdesc->{$key}->{format};
2408                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2409                     my $v = parse_drive($key, $value);
2410                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2411                         $v->{file} = $volid;
2412                         $value = print_drive($v);
2413                     } else {
2414                         warn "vm $vmid - unable to parse value of '$key'\n";
2415                         next;
2416                     }
2417                 }
2418
2419                 $conf->{$key} = $value;
2420             }
2421         } else {
2422             warn "vm $vmid - unable to parse config: $line\n";
2423         }
2424     }
2425
2426     if (defined($descr)) {
2427         $descr =~ s/\s+$//;
2428         $conf->{description} = $descr;
2429     }
2430     delete $res->{snapstate}; # just to be sure
2431
2432     return $res;
2433 }
2434
2435 sub write_vm_config {
2436     my ($filename, $conf) = @_;
2437
2438     delete $conf->{snapstate}; # just to be sure
2439
2440     if ($conf->{cdrom}) {
2441         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2442         $conf->{ide2} = $conf->{cdrom};
2443         delete $conf->{cdrom};
2444     }
2445
2446     # we do not use 'smp' any longer
2447     if ($conf->{sockets}) {
2448         delete $conf->{smp};
2449     } elsif ($conf->{smp}) {
2450         $conf->{sockets} = $conf->{smp};
2451         delete $conf->{cores};
2452         delete $conf->{smp};
2453     }
2454
2455     my $used_volids = {};
2456
2457     my $cleanup_config = sub {
2458         my ($cref, $pending, $snapname) = @_;
2459
2460         foreach my $key (keys %$cref) {
2461             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2462                 $key eq 'snapstate' || $key eq 'pending';
2463             my $value = $cref->{$key};
2464             if ($key eq 'delete') {
2465                 die "propertry 'delete' is only allowed in [PENDING]\n"
2466                     if !$pending;
2467                 # fixme: check syntax?
2468                 next;
2469             }
2470             eval { $value = check_type($key, $value); };
2471             die "unable to parse value of '$key' - $@" if $@;
2472
2473             $cref->{$key} = $value;
2474
2475             if (!$snapname && is_valid_drivename($key)) {
2476                 my $drive = parse_drive($key, $value);
2477                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2478             }
2479         }
2480     };
2481
2482     &$cleanup_config($conf);
2483
2484     &$cleanup_config($conf->{pending}, 1);
2485
2486     foreach my $snapname (keys %{$conf->{snapshots}}) {
2487         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2488         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2489     }
2490
2491     # remove 'unusedX' settings if we re-add a volume
2492     foreach my $key (keys %$conf) {
2493         my $value = $conf->{$key};
2494         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2495             delete $conf->{$key};
2496         }
2497     }
2498
2499     my $generate_raw_config = sub {
2500         my ($conf, $pending) = @_;
2501
2502         my $raw = '';
2503
2504         # add description as comment to top of file
2505         if (defined(my $descr = $conf->{description})) {
2506             if ($descr) {
2507                 foreach my $cl (split(/\n/, $descr)) {
2508                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2509                 }
2510             } else {
2511                 $raw .= "#\n" if $pending;
2512             }
2513         }
2514
2515         foreach my $key (sort keys %$conf) {
2516             next if $key =~ /^(digest|description|pending|snapshots)$/;
2517             $raw .= "$key: $conf->{$key}\n";
2518         }
2519         return $raw;
2520     };
2521
2522     my $raw = &$generate_raw_config($conf);
2523
2524     if (scalar(keys %{$conf->{pending}})){
2525         $raw .= "\n[PENDING]\n";
2526         $raw .= &$generate_raw_config($conf->{pending}, 1);
2527     }
2528
2529     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2530         $raw .= "\n[$snapname]\n";
2531         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2532     }
2533
2534     return $raw;
2535 }
2536
2537 sub load_defaults {
2538
2539     my $res = {};
2540
2541     # we use static defaults from our JSON schema configuration
2542     foreach my $key (keys %$confdesc) {
2543         if (defined(my $default = $confdesc->{$key}->{default})) {
2544             $res->{$key} = $default;
2545         }
2546     }
2547
2548     return $res;
2549 }
2550
2551 sub config_list {
2552     my $vmlist = PVE::Cluster::get_vmlist();
2553     my $res = {};
2554     return $res if !$vmlist || !$vmlist->{ids};
2555     my $ids = $vmlist->{ids};
2556     my $nodename = nodename();
2557
2558     foreach my $vmid (keys %$ids) {
2559         my $d = $ids->{$vmid};
2560         next if !$d->{node} || $d->{node} ne $nodename;
2561         next if !$d->{type} || $d->{type} ne 'qemu';
2562         $res->{$vmid}->{exists} = 1;
2563     }
2564     return $res;
2565 }
2566
2567 # test if VM uses local resources (to prevent migration)
2568 sub check_local_resources {
2569     my ($conf, $noerr) = @_;
2570
2571     my @loc_res = ();
2572
2573     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2574     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2575
2576     push @loc_res, "ivshmem" if $conf->{ivshmem};
2577
2578     foreach my $k (keys %$conf) {
2579         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2580         # sockets are safe: they will recreated be on the target side post-migrate
2581         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2582         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2583     }
2584
2585     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2586
2587     return \@loc_res;
2588 }
2589
2590 # check if used storages are available on all nodes (use by migrate)
2591 sub check_storage_availability {
2592     my ($storecfg, $conf, $node) = @_;
2593
2594     PVE::QemuConfig->foreach_volume($conf, sub {
2595         my ($ds, $drive) = @_;
2596
2597         my $volid = $drive->{file};
2598         return if !$volid;
2599
2600         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2601         return if !$sid;
2602
2603         # check if storage is available on both nodes
2604         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2605         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2606
2607         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2608
2609         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2610             if !$scfg->{content}->{$vtype};
2611    });
2612 }
2613
2614 # list nodes where all VM images are available (used by has_feature API)
2615 sub shared_nodes {
2616     my ($conf, $storecfg) = @_;
2617
2618     my $nodelist = PVE::Cluster::get_nodelist();
2619     my $nodehash = { map { $_ => 1 } @$nodelist };
2620     my $nodename = nodename();
2621
2622     PVE::QemuConfig->foreach_volume($conf, sub {
2623         my ($ds, $drive) = @_;
2624
2625         my $volid = $drive->{file};
2626         return if !$volid;
2627
2628         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2629         if ($storeid) {
2630             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2631             if ($scfg->{disable}) {
2632                 $nodehash = {};
2633             } elsif (my $avail = $scfg->{nodes}) {
2634                 foreach my $node (keys %$nodehash) {
2635                     delete $nodehash->{$node} if !$avail->{$node};
2636                 }
2637             } elsif (!$scfg->{shared}) {
2638                 foreach my $node (keys %$nodehash) {
2639                     delete $nodehash->{$node} if $node ne $nodename
2640                 }
2641             }
2642         }
2643     });
2644
2645     return $nodehash
2646 }
2647
2648 sub check_local_storage_availability {
2649     my ($conf, $storecfg) = @_;
2650
2651     my $nodelist = PVE::Cluster::get_nodelist();
2652     my $nodehash = { map { $_ => {} } @$nodelist };
2653
2654     PVE::QemuConfig->foreach_volume($conf, sub {
2655         my ($ds, $drive) = @_;
2656
2657         my $volid = $drive->{file};
2658         return if !$volid;
2659
2660         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2661         if ($storeid) {
2662             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2663
2664             if ($scfg->{disable}) {
2665                 foreach my $node (keys %$nodehash) {
2666                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2667                 }
2668             } elsif (my $avail = $scfg->{nodes}) {
2669                 foreach my $node (keys %$nodehash) {
2670                     if (!$avail->{$node}) {
2671                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2672                     }
2673                 }
2674             }
2675         }
2676     });
2677
2678     foreach my $node (values %$nodehash) {
2679         if (my $unavail = $node->{unavailable_storages}) {
2680             $node->{unavailable_storages} = [ sort keys %$unavail ];
2681         }
2682     }
2683
2684     return $nodehash
2685 }
2686
2687 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2688 sub check_running {
2689     my ($vmid, $nocheck, $node) = @_;
2690
2691     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2692     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2693 }
2694
2695 sub vzlist {
2696
2697     my $vzlist = config_list();
2698
2699     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2700
2701     while (defined(my $de = $fd->read)) {
2702         next if $de !~ m/^(\d+)\.pid$/;
2703         my $vmid = $1;
2704         next if !defined($vzlist->{$vmid});
2705         if (my $pid = check_running($vmid)) {
2706             $vzlist->{$vmid}->{pid} = $pid;
2707         }
2708     }
2709
2710     return $vzlist;
2711 }
2712
2713 our $vmstatus_return_properties = {
2714     vmid => get_standard_option('pve-vmid'),
2715     status => {
2716         description => "Qemu process status.",
2717         type => 'string',
2718         enum => ['stopped', 'running'],
2719     },
2720     maxmem => {
2721         description => "Maximum memory in bytes.",
2722         type => 'integer',
2723         optional => 1,
2724         renderer => 'bytes',
2725     },
2726     maxdisk => {
2727         description => "Root disk size in bytes.",
2728         type => 'integer',
2729         optional => 1,
2730         renderer => 'bytes',
2731     },
2732     name => {
2733         description => "VM name.",
2734         type => 'string',
2735         optional => 1,
2736     },
2737     qmpstatus => {
2738         description => "Qemu QMP agent status.",
2739         type => 'string',
2740         optional => 1,
2741     },
2742     pid => {
2743         description => "PID of running qemu process.",
2744         type => 'integer',
2745         optional => 1,
2746     },
2747     uptime => {
2748         description => "Uptime.",
2749         type => 'integer',
2750         optional => 1,
2751         renderer => 'duration',
2752     },
2753     cpus => {
2754         description => "Maximum usable CPUs.",
2755         type => 'number',
2756         optional => 1,
2757     },
2758     lock => {
2759         description => "The current config lock, if any.",
2760         type => 'string',
2761         optional => 1,
2762     },
2763     tags => {
2764         description  => "The current configured tags, if any",
2765         type => 'string',
2766         optional => 1,
2767     },
2768     'running-machine' => {
2769         description => "The currently running machine type (if running).",
2770         type => 'string',
2771         optional => 1,
2772     },
2773     'running-qemu' => {
2774         description => "The currently running QEMU version (if running).",
2775         type => 'string',
2776         optional => 1,
2777     },
2778 };
2779
2780 my $last_proc_pid_stat;
2781
2782 # get VM status information
2783 # This must be fast and should not block ($full == false)
2784 # We only query KVM using QMP if $full == true (this can be slow)
2785 sub vmstatus {
2786     my ($opt_vmid, $full) = @_;
2787
2788     my $res = {};
2789
2790     my $storecfg = PVE::Storage::config();
2791
2792     my $list = vzlist();
2793     my $defaults = load_defaults();
2794
2795     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2796
2797     my $cpucount = $cpuinfo->{cpus} || 1;
2798
2799     foreach my $vmid (keys %$list) {
2800         next if $opt_vmid && ($vmid ne $opt_vmid);
2801
2802         my $conf = PVE::QemuConfig->load_config($vmid);
2803
2804         my $d = { vmid => int($vmid) };
2805         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2806
2807         # fixme: better status?
2808         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2809
2810         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2811         if (defined($size)) {
2812             $d->{disk} = 0; # no info available
2813             $d->{maxdisk} = $size;
2814         } else {
2815             $d->{disk} = 0;
2816             $d->{maxdisk} = 0;
2817         }
2818
2819         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2820             * ($conf->{cores} || $defaults->{cores});
2821         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2822         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2823
2824         $d->{name} = $conf->{name} || "VM $vmid";
2825         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2826             : $defaults->{memory}*(1024*1024);
2827
2828         if ($conf->{balloon}) {
2829             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2830             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2831                 : $defaults->{shares};
2832         }
2833
2834         $d->{uptime} = 0;
2835         $d->{cpu} = 0;
2836         $d->{mem} = 0;
2837
2838         $d->{netout} = 0;
2839         $d->{netin} = 0;
2840
2841         $d->{diskread} = 0;
2842         $d->{diskwrite} = 0;
2843
2844         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2845
2846         $d->{serial} = 1 if conf_has_serial($conf);
2847         $d->{lock} = $conf->{lock} if $conf->{lock};
2848         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2849
2850         $res->{$vmid} = $d;
2851     }
2852
2853     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2854     foreach my $dev (keys %$netdev) {
2855         next if $dev !~ m/^tap([1-9]\d*)i/;
2856         my $vmid = $1;
2857         my $d = $res->{$vmid};
2858         next if !$d;
2859
2860         $d->{netout} += $netdev->{$dev}->{receive};
2861         $d->{netin} += $netdev->{$dev}->{transmit};
2862
2863         if ($full) {
2864             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2865             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2866         }
2867
2868     }
2869
2870     my $ctime = gettimeofday;
2871
2872     foreach my $vmid (keys %$list) {
2873
2874         my $d = $res->{$vmid};
2875         my $pid = $d->{pid};
2876         next if !$pid;
2877
2878         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2879         next if !$pstat; # not running
2880
2881         my $used = $pstat->{utime} + $pstat->{stime};
2882
2883         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2884
2885         if ($pstat->{vsize}) {
2886             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2887         }
2888
2889         my $old = $last_proc_pid_stat->{$pid};
2890         if (!$old) {
2891             $last_proc_pid_stat->{$pid} = {
2892                 time => $ctime,
2893                 used => $used,
2894                 cpu => 0,
2895             };
2896             next;
2897         }
2898
2899         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2900
2901         if ($dtime > 1000) {
2902             my $dutime = $used -  $old->{used};
2903
2904             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2905             $last_proc_pid_stat->{$pid} = {
2906                 time => $ctime,
2907                 used => $used,
2908                 cpu => $d->{cpu},
2909             };
2910         } else {
2911             $d->{cpu} = $old->{cpu};
2912         }
2913     }
2914
2915     return $res if !$full;
2916
2917     my $qmpclient = PVE::QMPClient->new();
2918
2919     my $ballooncb = sub {
2920         my ($vmid, $resp) = @_;
2921
2922         my $info = $resp->{'return'};
2923         return if !$info->{max_mem};
2924
2925         my $d = $res->{$vmid};
2926
2927         # use memory assigned to VM
2928         $d->{maxmem} = $info->{max_mem};
2929         $d->{balloon} = $info->{actual};
2930
2931         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2932             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2933             $d->{freemem} = $info->{free_mem};
2934         }
2935
2936         $d->{ballooninfo} = $info;
2937     };
2938
2939     my $blockstatscb = sub {
2940         my ($vmid, $resp) = @_;
2941         my $data = $resp->{'return'} || [];
2942         my $totalrdbytes = 0;
2943         my $totalwrbytes = 0;
2944
2945         for my $blockstat (@$data) {
2946             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2947             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2948
2949             $blockstat->{device} =~ s/drive-//;
2950             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2951         }
2952         $res->{$vmid}->{diskread} = $totalrdbytes;
2953         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2954     };
2955
2956     my $machinecb = sub {
2957         my ($vmid, $resp) = @_;
2958         my $data = $resp->{'return'} || [];
2959
2960         $res->{$vmid}->{'running-machine'} =
2961             PVE::QemuServer::Machine::current_from_query_machines($data);
2962     };
2963
2964     my $versioncb = sub {
2965         my ($vmid, $resp) = @_;
2966         my $data = $resp->{'return'} // {};
2967         my $version = 'unknown';
2968
2969         if (my $v = $data->{qemu}) {
2970             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2971         }
2972
2973         $res->{$vmid}->{'running-qemu'} = $version;
2974     };
2975
2976     my $statuscb = sub {
2977         my ($vmid, $resp) = @_;
2978
2979         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2980         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2981         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2982         # this fails if ballon driver is not loaded, so this must be
2983         # the last commnand (following command are aborted if this fails).
2984         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2985
2986         my $status = 'unknown';
2987         if (!defined($status = $resp->{'return'}->{status})) {
2988             warn "unable to get VM status\n";
2989             return;
2990         }
2991
2992         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2993     };
2994
2995     foreach my $vmid (keys %$list) {
2996         next if $opt_vmid && ($vmid ne $opt_vmid);
2997         next if !$res->{$vmid}->{pid}; # not running
2998         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2999     }
3000
3001     $qmpclient->queue_execute(undef, 2);
3002
3003     foreach my $vmid (keys %$list) {
3004         next if $opt_vmid && ($vmid ne $opt_vmid);
3005         next if !$res->{$vmid}->{pid}; #not running
3006
3007         # we can't use the $qmpclient since it might have already aborted on
3008         # 'query-balloon', but this might also fail for older versions...
3009         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
3010         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
3011     }
3012
3013     foreach my $vmid (keys %$list) {
3014         next if $opt_vmid && ($vmid ne $opt_vmid);
3015         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3016     }
3017
3018     return $res;
3019 }
3020
3021 sub conf_has_serial {
3022     my ($conf) = @_;
3023
3024     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3025         if ($conf->{"serial$i"}) {
3026             return 1;
3027         }
3028     }
3029
3030     return 0;
3031 }
3032
3033 sub conf_has_audio {
3034     my ($conf, $id) = @_;
3035
3036     $id //= 0;
3037     my $audio = $conf->{"audio$id"};
3038     return if !defined($audio);
3039
3040     my $audioproperties = parse_property_string($audio_fmt, $audio);
3041     my $audiodriver = $audioproperties->{driver} // 'spice';
3042
3043     return {
3044         dev => $audioproperties->{device},
3045         dev_id => "audiodev$id",
3046         backend => $audiodriver,
3047         backend_id => "$audiodriver-backend${id}",
3048     };
3049 }
3050
3051 sub audio_devs {
3052     my ($audio, $audiopciaddr, $machine_version) = @_;
3053
3054     my $devs = [];
3055
3056     my $id = $audio->{dev_id};
3057     my $audiodev = "";
3058     if (min_version($machine_version, 4, 2)) {
3059         $audiodev = ",audiodev=$audio->{backend_id}";
3060     }
3061
3062     if ($audio->{dev} eq 'AC97') {
3063         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
3064     } elsif ($audio->{dev} =~ /intel\-hda$/) {
3065         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3066         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
3067         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
3068     } else {
3069         die "unkown audio device '$audio->{dev}', implement me!";
3070     }
3071
3072     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3073
3074     return $devs;
3075 }
3076
3077 sub get_tpm_paths {
3078     my ($vmid) = @_;
3079     return {
3080         socket => "/var/run/qemu-server/$vmid.swtpm",
3081         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
3082     };
3083 }
3084
3085 sub add_tpm_device {
3086     my ($vmid, $devices, $conf) = @_;
3087
3088     return if !$conf->{tpmstate0};
3089
3090     my $paths = get_tpm_paths($vmid);
3091
3092     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
3093     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
3094     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3095 }
3096
3097 sub start_swtpm {
3098     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3099
3100     return if !$tpmdrive;
3101
3102     my $state;
3103     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3104     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3105     if ($storeid) {
3106         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3107     } else {
3108         $state = $tpm->{file};
3109     }
3110
3111     my $paths = get_tpm_paths($vmid);
3112
3113     # during migration, we will get state from remote
3114     #
3115     if (!$migration) {
3116         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3117         my $setup_cmd = [
3118             "swtpm_setup",
3119             "--tpmstate",
3120             "file://$state",
3121             "--createek",
3122             "--create-ek-cert",
3123             "--create-platform-cert",
3124             "--lock-nvram",
3125             "--config",
3126             "/etc/swtpm_setup.conf", # do not use XDG configs
3127             "--runas",
3128             "0", # force creation as root, error if not possible
3129             "--not-overwrite", # ignore existing state, do not modify
3130         ];
3131
3132         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3133         # TPM 2.0 supports ECC crypto, use if possible
3134         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3135
3136         run_command($setup_cmd, outfunc => sub {
3137             print "swtpm_setup: $1\n";
3138         });
3139     }
3140
3141     my $emulator_cmd = [
3142         "swtpm",
3143         "socket",
3144         "--tpmstate",
3145         "backend-uri=file://$state,mode=0600",
3146         "--ctrl",
3147         "type=unixio,path=$paths->{socket},mode=0600",
3148         "--pid",
3149         "file=$paths->{pid}",
3150         "--terminate", # terminate on QEMU disconnect
3151         "--daemon",
3152     ];
3153     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3154     run_command($emulator_cmd, outfunc => sub { print $1; });
3155
3156     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3157     while (! -e $paths->{pid}) {
3158         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n" if --$tries == 0;
3159         usleep(50_000);
3160     }
3161
3162     # return untainted PID of swtpm daemon so it can be killed on error
3163     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3164     return $1;
3165 }
3166
3167 sub vga_conf_has_spice {
3168     my ($vga) = @_;
3169
3170     my $vgaconf = parse_vga($vga);
3171     my $vgatype = $vgaconf->{type};
3172     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3173
3174     return $1 || 1;
3175 }
3176
3177 sub is_native($) {
3178     my ($arch) = @_;
3179     return get_host_arch() eq $arch;
3180 }
3181
3182 sub get_vm_arch {
3183     my ($conf) = @_;
3184     return $conf->{arch} // get_host_arch();
3185 }
3186
3187 my $default_machines = {
3188     x86_64 => 'pc',
3189     aarch64 => 'virt',
3190 };
3191
3192 sub get_installed_machine_version {
3193     my ($kvmversion) = @_;
3194     $kvmversion = kvm_user_version() if !defined($kvmversion);
3195     $kvmversion =~ m/^(\d+\.\d+)/;
3196     return $1;
3197 }
3198
3199 sub windows_get_pinned_machine_version {
3200     my ($machine, $base_version, $kvmversion) = @_;
3201
3202     my $pin_version = $base_version;
3203     if (!defined($base_version) ||
3204         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3205     ) {
3206         $pin_version = get_installed_machine_version($kvmversion);
3207     }
3208     if (!$machine || $machine eq 'pc') {
3209         $machine = "pc-i440fx-$pin_version";
3210     } elsif ($machine eq 'q35') {
3211         $machine = "pc-q35-$pin_version";
3212     } elsif ($machine eq 'virt') {
3213         $machine = "virt-$pin_version";
3214     } else {
3215         warn "unknown machine type '$machine', not touching that!\n";
3216     }
3217
3218     return $machine;
3219 }
3220
3221 sub get_vm_machine {
3222     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3223
3224     my $machine = $forcemachine || $conf->{machine};
3225
3226     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3227         $kvmversion //= kvm_user_version();
3228         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3229         # layout which confuses windows quite a bit and may result in various regressions..
3230         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3231         if (windows_version($conf->{ostype})) {
3232             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3233         }
3234         $arch //= 'x86_64';
3235         $machine ||= $default_machines->{$arch};
3236         if ($add_pve_version) {
3237             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3238             $machine .= "+pve$pvever";
3239         }
3240     }
3241
3242     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3243         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3244         $machine = $1 if $is_pxe;
3245
3246         # for version-pinned machines that do not include a pve-version (e.g.
3247         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3248         $machine .= '+pve0';
3249
3250         $machine .= '.pxe' if $is_pxe;
3251     }
3252
3253     return $machine;
3254 }
3255
3256 sub get_ovmf_files($$$) {
3257     my ($arch, $efidisk, $smm) = @_;
3258
3259     my $types = $OVMF->{$arch}
3260         or die "no OVMF images known for architecture '$arch'\n";
3261
3262     my $type = 'default';
3263     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3264         $type = $smm ? "4m" : "4m-no-smm";
3265         $type .= '-ms' if $efidisk->{'pre-enrolled-keys'};
3266     }
3267
3268     return $types->{$type}->@*;
3269 }
3270
3271 my $Arch2Qemu = {
3272     aarch64 => '/usr/bin/qemu-system-aarch64',
3273     x86_64 => '/usr/bin/qemu-system-x86_64',
3274 };
3275 sub get_command_for_arch($) {
3276     my ($arch) = @_;
3277     return '/usr/bin/kvm' if is_native($arch);
3278
3279     my $cmd = $Arch2Qemu->{$arch}
3280         or die "don't know how to emulate architecture '$arch'\n";
3281     return $cmd;
3282 }
3283
3284 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3285 # to use in a QEMU command line (-cpu element), first array_intersect the result
3286 # of query_supported_ with query_understood_. This is necessary because:
3287 #
3288 # a) query_understood_ returns flags the host cannot use and
3289 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3290 #    flags, but CPU settings - with most of them being flags. Those settings
3291 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3292 #
3293 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3294 # expensive. If you need the value returned from this, you can get it much
3295 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3296 # $accel being 'kvm' or 'tcg'.
3297 #
3298 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3299 # changes, automatically populating pmxcfs.
3300 #
3301 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3302 # since kvm and tcg machines support different flags
3303 #
3304 sub query_supported_cpu_flags {
3305     my ($arch) = @_;
3306
3307     $arch //= get_host_arch();
3308     my $default_machine = $default_machines->{$arch};
3309
3310     my $flags = {};
3311
3312     # FIXME: Once this is merged, the code below should work for ARM as well:
3313     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3314     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3315         $arch eq "aarch64";
3316
3317     my $kvm_supported = defined(kvm_version());
3318     my $qemu_cmd = get_command_for_arch($arch);
3319     my $fakevmid = -1;
3320     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3321
3322     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3323     my $query_supported_run_qemu = sub {
3324         my ($kvm) = @_;
3325
3326         my $flags = {};
3327         my $cmd = [
3328             $qemu_cmd,
3329             '-machine', $default_machine,
3330             '-display', 'none',
3331             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3332             '-mon', 'chardev=qmp,mode=control',
3333             '-pidfile', $pidfile,
3334             '-S', '-daemonize'
3335         ];
3336
3337         if (!$kvm) {
3338             push @$cmd, '-accel', 'tcg';
3339         }
3340
3341         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3342         die "QEMU flag querying VM exited with code " . $rc if $rc;
3343
3344         eval {
3345             my $cmd_result = mon_cmd(
3346                 $fakevmid,
3347                 'query-cpu-model-expansion',
3348                 type => 'full',
3349                 model => { name => 'host' }
3350             );
3351
3352             my $props = $cmd_result->{model}->{props};
3353             foreach my $prop (keys %$props) {
3354                 next if $props->{$prop} ne '1';
3355                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3356                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3357                 # We only keep those with underscores, to match /proc/cpuinfo
3358                 $prop =~ s/\.|-/_/g;
3359                 $flags->{$prop} = 1;
3360             }
3361         };
3362         my $err = $@;
3363
3364         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3365         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3366
3367         die $err if $err;
3368
3369         return [ sort keys %$flags ];
3370     };
3371
3372     # We need to query QEMU twice, since KVM and TCG have different supported flags
3373     PVE::QemuConfig->lock_config($fakevmid, sub {
3374         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3375         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3376
3377         if ($kvm_supported) {
3378             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3379             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3380         }
3381     });
3382
3383     return $flags;
3384 }
3385
3386 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3387 my $understood_cpu_flag_dir = "/usr/share/kvm";
3388 sub query_understood_cpu_flags {
3389     my $arch = get_host_arch();
3390     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3391
3392     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3393         if ! -e $filepath;
3394
3395     my $raw = file_get_contents($filepath);
3396     $raw =~ s/^\s+|\s+$//g;
3397     my @flags = split(/\s+/, $raw);
3398
3399     return \@flags;
3400 }
3401
3402 my sub get_cpuunits {
3403     my ($conf) = @_;
3404     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3405 }
3406
3407 # Since commit 277d33454f77ec1d1e0bc04e37621e4dd2424b67 in pve-qemu, smm is not off by default
3408 # anymore. But smm=off seems to be required when using SeaBIOS and serial display.
3409 my sub should_disable_smm {
3410     my ($conf, $vga) = @_;
3411
3412     return (!defined($conf->{bios}) || $conf->{bios} eq 'seabios') &&
3413         $vga->{type} && $vga->{type} =~ m/^(serial\d+|none)$/;
3414 }
3415
3416 sub config_to_command {
3417     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3418         $pbs_backing) = @_;
3419
3420     my $cmd = [];
3421     my ($globalFlags, $machineFlags, $rtcFlags) = ([], [], []);
3422     my $devices = [];
3423     my $bridges = {};
3424     my $ostype = $conf->{ostype};
3425     my $winversion = windows_version($ostype);
3426     my $kvm = $conf->{kvm};
3427     my $nodename = nodename();
3428
3429     my $arch = get_vm_arch($conf);
3430     my $kvm_binary = get_command_for_arch($arch);
3431     my $kvmver = kvm_user_version($kvm_binary);
3432
3433     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3434         $kvmver //= "undefined";
3435         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3436     }
3437
3438     my $add_pve_version = min_version($kvmver, 4, 1);
3439
3440     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3441     my $machine_version = extract_version($machine_type, $kvmver);
3442     $kvm //= 1 if is_native($arch);
3443
3444     $machine_version =~ m/(\d+)\.(\d+)/;
3445     my ($machine_major, $machine_minor) = ($1, $2);
3446
3447     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3448         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3449     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3450         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3451             ." please upgrade node '$nodename'\n"
3452     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3453         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3454         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3455             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3456             ." node '$nodename'\n";
3457     }
3458
3459     # if a specific +pve version is required for a feature, use $version_guard
3460     # instead of min_version to allow machines to be run with the minimum
3461     # required version
3462     my $required_pve_version = 0;
3463     my $version_guard = sub {
3464         my ($major, $minor, $pve) = @_;
3465         return 0 if !min_version($machine_version, $major, $minor, $pve);
3466         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3467         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3468         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3469         return 1;
3470     };
3471
3472     if ($kvm && !defined kvm_version()) {
3473         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3474             ." or enable in BIOS.\n";
3475     }
3476
3477     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3478     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3479     my $use_old_bios_files = undef;
3480     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3481
3482     my $cpuunits = get_cpuunits($conf);
3483
3484     push @$cmd, $kvm_binary;
3485
3486     push @$cmd, '-id', $vmid;
3487
3488     my $vmname = $conf->{name} || "vm$vmid";
3489
3490     push @$cmd, '-name', $vmname;
3491
3492     push @$cmd, '-no-shutdown';
3493
3494     my $use_virtio = 0;
3495
3496     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3497     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3498     push @$cmd, '-mon', "chardev=qmp,mode=control";
3499
3500     if (min_version($machine_version, 2, 12)) {
3501         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3502         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3503     }
3504
3505     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3506
3507     push @$cmd, '-daemonize';
3508
3509     if ($conf->{smbios1}) {
3510         my $smbios_conf = parse_smbios1($conf->{smbios1});
3511         if ($smbios_conf->{base64}) {
3512             # Do not pass base64 flag to qemu
3513             delete $smbios_conf->{base64};
3514             my $smbios_string = "";
3515             foreach my $key (keys %$smbios_conf) {
3516                 my $value;
3517                 if ($key eq "uuid") {
3518                     $value = $smbios_conf->{uuid}
3519                 } else {
3520                     $value = decode_base64($smbios_conf->{$key});
3521                 }
3522                 # qemu accepts any binary data, only commas need escaping by double comma
3523                 $value =~ s/,/,,/g;
3524                 $smbios_string .= "," . $key . "=" . $value if $value;
3525             }
3526             push @$cmd, '-smbios', "type=1" . $smbios_string;
3527         } else {
3528             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3529         }
3530     }
3531
3532     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3533         my $d;
3534         if (my $efidisk = $conf->{efidisk0}) {
3535             $d = parse_drive('efidisk0', $efidisk);
3536         }
3537
3538         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d, $q35);
3539         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3540
3541         my ($path, $format);
3542         my $read_only_str = '';
3543         if ($d) {
3544             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3545             $format = $d->{format};
3546             if ($storeid) {
3547                 $path = PVE::Storage::path($storecfg, $d->{file});
3548                 if (!defined($format)) {
3549                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3550                     $format = qemu_img_format($scfg, $volname);
3551                 }
3552             } else {
3553                 $path = $d->{file};
3554                 die "efidisk format must be specified\n"
3555                     if !defined($format);
3556             }
3557
3558             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3559         } else {
3560             warn "no efidisk configured! Using temporary efivars disk.\n";
3561             $path = "/tmp/$vmid-ovmf.fd";
3562             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3563             $format = 'raw';
3564         }
3565
3566         my $size_str = "";
3567
3568         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3569             $size_str = ",size=" . (-s $ovmf_vars);
3570         }
3571
3572         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3573         my $cache = "";
3574         if ($path =~ m/^rbd:/) {
3575                 $cache = ',cache=writeback';
3576                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3577         }
3578
3579         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3580         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3581     }
3582
3583     if ($q35) { # tell QEMU to load q35 config early
3584         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3585         if (min_version($machine_version, 4, 0)) {
3586             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3587         } else {
3588             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3589         }
3590     }
3591
3592     if (defined(my $fixups = qemu_created_version_fixups($conf, $forcemachine, $kvmver))) {
3593         push @$cmd, $fixups->@*;
3594     }
3595
3596     if ($conf->{vmgenid}) {
3597         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3598     }
3599
3600     # add usb controllers
3601     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3602         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3603     push @$devices, @usbcontrollers if @usbcontrollers;
3604     my $vga = parse_vga($conf->{vga});
3605
3606     my $qxlnum = vga_conf_has_spice($conf->{vga});
3607     $vga->{type} = 'qxl' if $qxlnum;
3608
3609     if (!$vga->{type}) {
3610         if ($arch eq 'aarch64') {
3611             $vga->{type} = 'virtio';
3612         } elsif (min_version($machine_version, 2, 9)) {
3613             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3614         } else {
3615             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3616         }
3617     }
3618
3619     # enable absolute mouse coordinates (needed by vnc)
3620     my $tablet = $conf->{tablet};
3621     if (!defined($tablet)) {
3622         $tablet = $defaults->{tablet};
3623         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3624         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3625     }
3626
3627     if ($tablet) {
3628         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3629         my $kbd = print_keyboarddevice_full($conf, $arch);
3630         push @$devices, '-device', $kbd if defined($kbd);
3631     }
3632
3633     my $bootorder = device_bootorder($conf);
3634
3635     # host pci device passthrough
3636     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3637         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3638
3639     # usb devices
3640     my $usb_dev_features = {};
3641     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3642
3643     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3644         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3645     push @$devices, @usbdevices if @usbdevices;
3646
3647     # serial devices
3648     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3649         my $path = $conf->{"serial$i"} or next;
3650         if ($path eq 'socket') {
3651             my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3652             push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3653             # On aarch64, serial0 is the UART device. Qemu only allows
3654             # connecting UART devices via the '-serial' command line, as
3655             # the device has a fixed slot on the hardware...
3656             if ($arch eq 'aarch64' && $i == 0) {
3657                 push @$devices, '-serial', "chardev:serial$i";
3658             } else {
3659                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3660             }
3661         } else {
3662             die "no such serial device\n" if ! -c $path;
3663             push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3664             push @$devices, '-device', "isa-serial,chardev=serial$i";
3665         }
3666     }
3667
3668     # parallel devices
3669     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3670         if (my $path = $conf->{"parallel$i"}) {
3671             die "no such parallel device\n" if ! -c $path;
3672             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3673             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3674             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3675         }
3676     }
3677
3678     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3679         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3680         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3681         push @$devices, @$audio_devs;
3682     }
3683
3684     add_tpm_device($vmid, $devices, $conf);
3685
3686     my $sockets = 1;
3687     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3688     $sockets = $conf->{sockets} if  $conf->{sockets};
3689
3690     my $cores = $conf->{cores} || 1;
3691
3692     my $maxcpus = $sockets * $cores;
3693
3694     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3695
3696     my $allowed_vcpus = $cpuinfo->{cpus};
3697
3698     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n" if ($allowed_vcpus < $maxcpus);
3699
3700     if ($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3701         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3702         for (my $i = 2; $i <= $vcpus; $i++)  {
3703             my $cpustr = print_cpu_device($conf,$i);
3704             push @$cmd, '-device', $cpustr;
3705         }
3706
3707     } else {
3708
3709         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3710     }
3711     push @$cmd, '-nodefaults';
3712
3713     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3714
3715     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3716
3717     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3718
3719     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3720         push @$devices, '-device', print_vga_device(
3721             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3722         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3723         push @$cmd,  '-vnc', "unix:$socket,password=on";
3724     } else {
3725         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3726         push @$cmd, '-nographic';
3727     }
3728
3729     # time drift fix
3730     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3731     my $useLocaltime = $conf->{localtime};
3732
3733     if ($winversion >= 5) { # windows
3734         $useLocaltime = 1 if !defined($conf->{localtime});
3735
3736         # use time drift fix when acpi is enabled
3737         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3738             $tdf = 1 if !defined($conf->{tdf});
3739         }
3740     }
3741
3742     if ($winversion >= 6) {
3743         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3744         push @$cmd, '-no-hpet';
3745     }
3746
3747     push @$rtcFlags, 'driftfix=slew' if $tdf;
3748
3749     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3750         push @$rtcFlags, "base=$conf->{startdate}";
3751     } elsif ($useLocaltime) {
3752         push @$rtcFlags, 'base=localtime';
3753     }
3754
3755     if ($forcecpu) {
3756         push @$cmd, '-cpu', $forcecpu;
3757     } else {
3758         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3759     }
3760
3761     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3762
3763     push @$cmd, '-S' if $conf->{freeze};
3764
3765     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3766
3767     my $guest_agent = parse_guest_agent($conf);
3768
3769     if ($guest_agent->{enabled}) {
3770         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3771         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3772
3773         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3774             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3775             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3776             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3777         } elsif ($guest_agent->{type} eq 'isa') {
3778             push @$devices, '-device', "isa-serial,chardev=qga0";
3779         }
3780     }
3781
3782     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3783     if ($rng && $version_guard->(4, 1, 2)) {
3784         check_rng_source($rng->{source});
3785
3786         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3787         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3788         my $limiter_str = "";
3789         if ($max_bytes) {
3790             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3791         }
3792
3793         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3794         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3795         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3796     }
3797
3798     my $spice_port;
3799
3800     if ($qxlnum) {
3801         if ($qxlnum > 1) {
3802             if ($winversion){
3803                 for (my $i = 1; $i < $qxlnum; $i++){
3804                     push @$devices, '-device', print_vga_device(
3805                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3806                 }
3807             } else {
3808                 # assume other OS works like Linux
3809                 my ($ram, $vram) = ("134217728", "67108864");
3810                 if ($vga->{memory}) {
3811                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3812                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3813                 }
3814                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3815                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3816             }
3817         }
3818
3819         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3820
3821         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3822         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3823         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3824
3825         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3826         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3827         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3828
3829         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3830         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3831
3832         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3833         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3834         if ($spice_enhancement->{foldersharing}) {
3835             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3836             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3837         }
3838
3839         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3840         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3841             if $spice_enhancement->{videostreaming};
3842
3843         push @$devices, '-spice', "$spice_opts";
3844     }
3845
3846     # enable balloon by default, unless explicitly disabled
3847     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3848         my $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3849         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3850     }
3851
3852     if ($conf->{watchdog}) {
3853         my $wdopts = parse_watchdog($conf->{watchdog});
3854         my $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3855         my $watchdog = $wdopts->{model} || 'i6300esb';
3856         push @$devices, '-device', "$watchdog$pciaddr";
3857         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3858     }
3859
3860     my $vollist = [];
3861     my $scsicontroller = {};
3862     my $ahcicontroller = {};
3863     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3864
3865     # Add iscsi initiator name if available
3866     if (my $initiator = get_initiator_name()) {
3867         push @$devices, '-iscsi', "initiator-name=$initiator";
3868     }
3869
3870     PVE::QemuConfig->foreach_volume($conf, sub {
3871         my ($ds, $drive) = @_;
3872
3873         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3874             check_volume_storage_type($storecfg, $drive->{file});
3875             push @$vollist, $drive->{file};
3876         }
3877
3878         # ignore efidisk here, already added in bios/fw handling code above
3879         return if $drive->{interface} eq 'efidisk';
3880         # similar for TPM
3881         return if $drive->{interface} eq 'tpmstate';
3882
3883         $use_virtio = 1 if $ds =~ m/^virtio/;
3884
3885         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3886
3887         if ($drive->{interface} eq 'virtio'){
3888            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3889         }
3890
3891         if ($drive->{interface} eq 'scsi') {
3892
3893             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3894
3895             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3896                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3897
3898             my $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3899             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3900
3901             my $iothread = '';
3902             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3903                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3904                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3905             } elsif ($drive->{iothread}) {
3906                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3907             }
3908
3909             my $queues = '';
3910             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3911                 $queues = ",num_queues=$drive->{queues}";
3912             }
3913
3914             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3915                 if !$scsicontroller->{$controller};
3916             $scsicontroller->{$controller}=1;
3917         }
3918
3919         if ($drive->{interface} eq 'sata') {
3920             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3921             my $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3922             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3923                 if !$ahcicontroller->{$controller};
3924             $ahcicontroller->{$controller}=1;
3925         }
3926
3927         my $pbs_conf = $pbs_backing->{$ds};
3928         my $pbs_name = undef;
3929         if ($pbs_conf) {
3930             $pbs_name = "drive-$ds-pbs";
3931             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3932         }
3933
3934         my $drive_cmd = print_drive_commandline_full(
3935             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3936
3937         # extra protection for templates, but SATA and IDE don't support it..
3938         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3939
3940         push @$devices, '-drive',$drive_cmd;
3941         push @$devices, '-device', print_drivedevice_full(
3942             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3943     });
3944
3945     for (my $i = 0; $i < $MAX_NETS; $i++) {
3946         my $netname = "net$i";
3947
3948         next if !$conf->{$netname};
3949         my $d = parse_net($conf->{$netname});
3950         next if !$d;
3951
3952         $use_virtio = 1 if $d->{model} eq 'virtio';
3953
3954         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3955
3956         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3957         push @$devices, '-netdev', $netdevfull;
3958
3959         my $netdevicefull = print_netdevice_full(
3960             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3961
3962         push @$devices, '-device', $netdevicefull;
3963     }
3964
3965     if ($conf->{ivshmem}) {
3966         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3967
3968         my $bus;
3969         if ($q35) {
3970             $bus = print_pcie_addr("ivshmem");
3971         } else {
3972             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3973         }
3974
3975         my $ivshmem_name = $ivshmem->{name} // $vmid;
3976         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3977
3978         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3979         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3980             .",size=$ivshmem->{size}M";
3981     }
3982
3983     # pci.4 is nested in pci.1
3984     $bridges->{1} = 1 if $bridges->{4};
3985
3986     if (!$q35) { # add pci bridges
3987         if (min_version($machine_version, 2, 3)) {
3988            $bridges->{1} = 1;
3989            $bridges->{2} = 1;
3990         }
3991         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3992     }
3993
3994     for my $k (sort {$b cmp $a} keys %$bridges) {
3995         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3996
3997         my $k_name = $k;
3998         if ($k == 2 && $legacy_igd) {
3999             $k_name = "$k-igd";
4000         }
4001         my $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
4002         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
4003
4004         if ($q35) { # add after -readconfig pve-q35.cfg
4005             splice @$devices, 2, 0, '-device', $devstr;
4006         } else {
4007             unshift @$devices, '-device', $devstr if $k > 0;
4008         }
4009     }
4010
4011     if (!$kvm) {
4012         push @$machineFlags, 'accel=tcg';
4013     }
4014
4015     push @$machineFlags, 'smm=off' if should_disable_smm($conf, $vga);
4016
4017     my $machine_type_min = $machine_type;
4018     if ($add_pve_version) {
4019         $machine_type_min =~ s/\+pve\d+$//;
4020         $machine_type_min .= "+pve$required_pve_version";
4021     }
4022     push @$machineFlags, "type=${machine_type_min}";
4023
4024     push @$cmd, @$devices;
4025     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
4026     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
4027     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
4028
4029     if (my $vmstate = $conf->{vmstate}) {
4030         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4031         push @$vollist, $vmstate;
4032         push @$cmd, '-loadstate', $statepath;
4033         print "activating and using '$vmstate' as vmstate\n";
4034     }
4035
4036     if (PVE::QemuConfig->is_template($conf)) {
4037         # needed to workaround base volumes being read-only
4038         push @$cmd, '-snapshot';
4039     }
4040
4041     # add custom args
4042     if ($conf->{args}) {
4043         my $aa = PVE::Tools::split_args($conf->{args});
4044         push @$cmd, @$aa;
4045     }
4046
4047     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4048 }
4049
4050 sub check_rng_source {
4051     my ($source) = @_;
4052
4053     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
4054     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
4055         if ! -e $source;
4056
4057     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
4058     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
4059         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
4060         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
4061         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
4062             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
4063             ." to the host.\n";
4064     }
4065 }
4066
4067 sub spice_port {
4068     my ($vmid) = @_;
4069
4070     my $res = mon_cmd($vmid, 'query-spice');
4071
4072     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4073 }
4074
4075 sub vm_devices_list {
4076     my ($vmid) = @_;
4077
4078     my $res = mon_cmd($vmid, 'query-pci');
4079     my $devices_to_check = [];
4080     my $devices = {};
4081     foreach my $pcibus (@$res) {
4082         push @$devices_to_check, @{$pcibus->{devices}},
4083     }
4084
4085     while (@$devices_to_check) {
4086         my $to_check = [];
4087         for my $d (@$devices_to_check) {
4088             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4089             next if !$d->{'pci_bridge'};
4090
4091             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4092             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4093         }
4094         $devices_to_check = $to_check;
4095     }
4096
4097     my $resblock = mon_cmd($vmid, 'query-block');
4098     foreach my $block (@$resblock) {
4099         if($block->{device} =~ m/^drive-(\S+)/){
4100                 $devices->{$1} = 1;
4101         }
4102     }
4103
4104     my $resmice = mon_cmd($vmid, 'query-mice');
4105     foreach my $mice (@$resmice) {
4106         if ($mice->{name} eq 'QEMU HID Tablet') {
4107             $devices->{tablet} = 1;
4108             last;
4109         }
4110     }
4111
4112     # for usb devices there is no query-usb
4113     # but we can iterate over the entries in
4114     # qom-list path=/machine/peripheral
4115     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4116     foreach my $per (@$resperipheral) {
4117         if ($per->{name} =~ m/^usb\d+$/) {
4118             $devices->{$per->{name}} = 1;
4119         }
4120     }
4121
4122     return $devices;
4123 }
4124
4125 sub vm_deviceplug {
4126     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4127
4128     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4129
4130     my $devices_list = vm_devices_list($vmid);
4131     return 1 if defined($devices_list->{$deviceid});
4132
4133     # add PCI bridge if we need it for the device
4134     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4135
4136     if ($deviceid eq 'tablet') {
4137         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4138     } elsif ($deviceid eq 'keyboard') {
4139         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4140     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4141         die "usb hotplug currently not reliable\n";
4142         # since we can't reliably hot unplug all added usb devices and usb
4143         # passthrough breaks live migration we disable usb hotplugging for now
4144         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4145     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4146         qemu_iothread_add($vmid, $deviceid, $device);
4147
4148         qemu_driveadd($storecfg, $vmid, $device);
4149         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4150
4151         qemu_deviceadd($vmid, $devicefull);
4152         eval { qemu_deviceaddverify($vmid, $deviceid); };
4153         if (my $err = $@) {
4154             eval { qemu_drivedel($vmid, $deviceid); };
4155             warn $@ if $@;
4156             die $err;
4157         }
4158     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4160         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4161         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4162
4163         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4164
4165         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4166             qemu_iothread_add($vmid, $deviceid, $device);
4167             $devicefull .= ",iothread=iothread-$deviceid";
4168         }
4169
4170         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4171             $devicefull .= ",num_queues=$device->{queues}";
4172         }
4173
4174         qemu_deviceadd($vmid, $devicefull);
4175         qemu_deviceaddverify($vmid, $deviceid);
4176     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4177         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4178         qemu_driveadd($storecfg, $vmid, $device);
4179
4180         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4181         eval { qemu_deviceadd($vmid, $devicefull); };
4182         if (my $err = $@) {
4183             eval { qemu_drivedel($vmid, $deviceid); };
4184             warn $@ if $@;
4185             die $err;
4186         }
4187     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4188         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4189
4190         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4191         my $use_old_bios_files = undef;
4192         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4193
4194         my $netdevicefull = print_netdevice_full(
4195             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4196         qemu_deviceadd($vmid, $netdevicefull);
4197         eval {
4198             qemu_deviceaddverify($vmid, $deviceid);
4199             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4200         };
4201         if (my $err = $@) {
4202             eval { qemu_netdevdel($vmid, $deviceid); };
4203             warn $@ if $@;
4204             die $err;
4205         }
4206     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4207         my $bridgeid = $2;
4208         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4209         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4210
4211         qemu_deviceadd($vmid, $devicefull);
4212         qemu_deviceaddverify($vmid, $deviceid);
4213     } else {
4214         die "can't hotplug device '$deviceid'\n";
4215     }
4216
4217     return 1;
4218 }
4219
4220 # fixme: this should raise exceptions on error!
4221 sub vm_deviceunplug {
4222     my ($vmid, $conf, $deviceid) = @_;
4223
4224     my $devices_list = vm_devices_list($vmid);
4225     return 1 if !defined($devices_list->{$deviceid});
4226
4227     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4228     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4229
4230     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4231         qemu_devicedel($vmid, $deviceid);
4232     } elsif ($deviceid =~ m/^usb\d+$/) {
4233         die "usb hotplug currently not reliable\n";
4234         # when unplugging usb devices this way, there may be remaining usb
4235         # controllers/hubs so we disable it for now
4236         #qemu_devicedel($vmid, $deviceid);
4237         #qemu_devicedelverify($vmid, $deviceid);
4238     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4239         my $device = parse_drive($deviceid, $conf->{$deviceid});
4240
4241         qemu_devicedel($vmid, $deviceid);
4242         qemu_devicedelverify($vmid, $deviceid);
4243         qemu_drivedel($vmid, $deviceid);
4244         qemu_iothread_del($vmid, $deviceid, $device);
4245     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4246         qemu_devicedel($vmid, $deviceid);
4247         qemu_devicedelverify($vmid, $deviceid);
4248     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4249         my $device = parse_drive($deviceid, $conf->{$deviceid});
4250
4251         qemu_devicedel($vmid, $deviceid);
4252         qemu_drivedel($vmid, $deviceid);
4253         qemu_deletescsihw($conf, $vmid, $deviceid);
4254
4255         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4256             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4257     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4258         qemu_devicedel($vmid, $deviceid);
4259         qemu_devicedelverify($vmid, $deviceid);
4260         qemu_netdevdel($vmid, $deviceid);
4261     } else {
4262         die "can't unplug device '$deviceid'\n";
4263     }
4264
4265     return 1;
4266 }
4267
4268 sub qemu_deviceadd {
4269     my ($vmid, $devicefull) = @_;
4270
4271     $devicefull = "driver=".$devicefull;
4272     my %options =  split(/[=,]/, $devicefull);
4273
4274     mon_cmd($vmid, "device_add" , %options);
4275 }
4276
4277 sub qemu_devicedel {
4278     my ($vmid, $deviceid) = @_;
4279
4280     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4281 }
4282
4283 sub qemu_iothread_add {
4284     my ($vmid, $deviceid, $device) = @_;
4285
4286     if ($device->{iothread}) {
4287         my $iothreads = vm_iothreads_list($vmid);
4288         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4289     }
4290 }
4291
4292 sub qemu_iothread_del {
4293     my ($vmid, $deviceid, $device) = @_;
4294
4295     if ($device->{iothread}) {
4296         my $iothreads = vm_iothreads_list($vmid);
4297         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4298     }
4299 }
4300
4301 sub qemu_objectadd {
4302     my ($vmid, $objectid, $qomtype) = @_;
4303
4304     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4305
4306     return 1;
4307 }
4308
4309 sub qemu_objectdel {
4310     my ($vmid, $objectid) = @_;
4311
4312     mon_cmd($vmid, "object-del", id => $objectid);
4313
4314     return 1;
4315 }
4316
4317 sub qemu_driveadd {
4318     my ($storecfg, $vmid, $device) = @_;
4319
4320     my $kvmver = get_running_qemu_version($vmid);
4321     my $io_uring = min_version($kvmver, 6, 0);
4322     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4323     $drive =~ s/\\/\\\\/g;
4324     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4325
4326     # If the command succeeds qemu prints: "OK"
4327     return 1 if $ret =~ m/OK/s;
4328
4329     die "adding drive failed: $ret\n";
4330 }
4331
4332 sub qemu_drivedel {
4333     my ($vmid, $deviceid) = @_;
4334
4335     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4336     $ret =~ s/^\s+//;
4337
4338     return 1 if $ret eq "";
4339
4340     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4341     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4342
4343     die "deleting drive $deviceid failed : $ret\n";
4344 }
4345
4346 sub qemu_deviceaddverify {
4347     my ($vmid, $deviceid) = @_;
4348
4349     for (my $i = 0; $i <= 5; $i++) {
4350          my $devices_list = vm_devices_list($vmid);
4351          return 1 if defined($devices_list->{$deviceid});
4352          sleep 1;
4353     }
4354
4355     die "error on hotplug device '$deviceid'\n";
4356 }
4357
4358
4359 sub qemu_devicedelverify {
4360     my ($vmid, $deviceid) = @_;
4361
4362     # need to verify that the device is correctly removed as device_del
4363     # is async and empty return is not reliable
4364
4365     for (my $i = 0; $i <= 5; $i++) {
4366          my $devices_list = vm_devices_list($vmid);
4367          return 1 if !defined($devices_list->{$deviceid});
4368          sleep 1;
4369     }
4370
4371     die "error on hot-unplugging device '$deviceid'\n";
4372 }
4373
4374 sub qemu_findorcreatescsihw {
4375     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4376
4377     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4378
4379     my $scsihwid="$controller_prefix$controller";
4380     my $devices_list = vm_devices_list($vmid);
4381
4382     if (!defined($devices_list->{$scsihwid})) {
4383         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4384     }
4385
4386     return 1;
4387 }
4388
4389 sub qemu_deletescsihw {
4390     my ($conf, $vmid, $opt) = @_;
4391
4392     my $device = parse_drive($opt, $conf->{$opt});
4393
4394     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4395         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4396         return 1;
4397     }
4398
4399     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4400
4401     my $devices_list = vm_devices_list($vmid);
4402     foreach my $opt (keys %{$devices_list}) {
4403         if (is_valid_drivename($opt)) {
4404             my $drive = parse_drive($opt, $conf->{$opt});
4405             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4406                 return 1;
4407             }
4408         }
4409     }
4410
4411     my $scsihwid="scsihw$controller";
4412
4413     vm_deviceunplug($vmid, $conf, $scsihwid);
4414
4415     return 1;
4416 }
4417
4418 sub qemu_add_pci_bridge {
4419     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4420
4421     my $bridges = {};
4422
4423     my $bridgeid;
4424
4425     print_pci_addr($device, $bridges, $arch, $machine_type);
4426
4427     while (my ($k, $v) = each %$bridges) {
4428         $bridgeid = $k;
4429     }
4430     return 1 if !defined($bridgeid) || $bridgeid < 1;
4431
4432     my $bridge = "pci.$bridgeid";
4433     my $devices_list = vm_devices_list($vmid);
4434
4435     if (!defined($devices_list->{$bridge})) {
4436         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4437     }
4438
4439     return 1;
4440 }
4441
4442 sub qemu_set_link_status {
4443     my ($vmid, $device, $up) = @_;
4444
4445     mon_cmd($vmid, "set_link", name => $device,
4446                up => $up ? JSON::true : JSON::false);
4447 }
4448
4449 sub qemu_netdevadd {
4450     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4451
4452     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4453     my %options =  split(/[=,]/, $netdev);
4454
4455     if (defined(my $vhost = $options{vhost})) {
4456         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4457     }
4458
4459     if (defined(my $queues = $options{queues})) {
4460         $options{queues} = $queues + 0;
4461     }
4462
4463     mon_cmd($vmid, "netdev_add",  %options);
4464     return 1;
4465 }
4466
4467 sub qemu_netdevdel {
4468     my ($vmid, $deviceid) = @_;
4469
4470     mon_cmd($vmid, "netdev_del", id => $deviceid);
4471 }
4472
4473 sub qemu_usb_hotplug {
4474     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4475
4476     return if !$device;
4477
4478     # remove the old one first
4479     vm_deviceunplug($vmid, $conf, $deviceid);
4480
4481     # check if xhci controller is necessary and available
4482     if ($device->{usb3}) {
4483
4484         my $devicelist = vm_devices_list($vmid);
4485
4486         if (!$devicelist->{xhci}) {
4487             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4488             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4489         }
4490     }
4491     my $d = parse_usb_device($device->{host});
4492     $d->{usb3} = $device->{usb3};
4493
4494     # add the new one
4495     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4496 }
4497
4498 sub qemu_cpu_hotplug {
4499     my ($vmid, $conf, $vcpus) = @_;
4500
4501     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4502
4503     my $sockets = 1;
4504     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4505     $sockets = $conf->{sockets} if  $conf->{sockets};
4506     my $cores = $conf->{cores} || 1;
4507     my $maxcpus = $sockets * $cores;
4508
4509     $vcpus = $maxcpus if !$vcpus;
4510
4511     die "you can't add more vcpus than maxcpus\n"
4512         if $vcpus > $maxcpus;
4513
4514     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4515
4516     if ($vcpus < $currentvcpus) {
4517
4518         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4519
4520             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4521                 qemu_devicedel($vmid, "cpu$i");
4522                 my $retry = 0;
4523                 my $currentrunningvcpus = undef;
4524                 while (1) {
4525                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4526                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4527                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4528                     $retry++;
4529                     sleep 1;
4530                 }
4531                 #update conf after each succesfull cpu unplug
4532                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4533                 PVE::QemuConfig->write_config($vmid, $conf);
4534             }
4535         } else {
4536             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4537         }
4538
4539         return;
4540     }
4541
4542     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4543     die "vcpus in running vm does not match its configuration\n"
4544         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4545
4546     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4547
4548         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4549             my $cpustr = print_cpu_device($conf, $i);
4550             qemu_deviceadd($vmid, $cpustr);
4551
4552             my $retry = 0;
4553             my $currentrunningvcpus = undef;
4554             while (1) {
4555                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4556                 last if scalar(@{$currentrunningvcpus}) == $i;
4557                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4558                 sleep 1;
4559                 $retry++;
4560             }
4561             #update conf after each succesfull cpu hotplug
4562             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4563             PVE::QemuConfig->write_config($vmid, $conf);
4564         }
4565     } else {
4566
4567         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4568             mon_cmd($vmid, "cpu-add", id => int($i));
4569         }
4570     }
4571 }
4572
4573 sub qemu_block_set_io_throttle {
4574     my ($vmid, $deviceid,
4575         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4576         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4577         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4578         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4579
4580     return if !check_running($vmid) ;
4581
4582     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4583         bps => int($bps),
4584         bps_rd => int($bps_rd),
4585         bps_wr => int($bps_wr),
4586         iops => int($iops),
4587         iops_rd => int($iops_rd),
4588         iops_wr => int($iops_wr),
4589         bps_max => int($bps_max),
4590         bps_rd_max => int($bps_rd_max),
4591         bps_wr_max => int($bps_wr_max),
4592         iops_max => int($iops_max),
4593         iops_rd_max => int($iops_rd_max),
4594         iops_wr_max => int($iops_wr_max),
4595         bps_max_length => int($bps_max_length),
4596         bps_rd_max_length => int($bps_rd_max_length),
4597         bps_wr_max_length => int($bps_wr_max_length),
4598         iops_max_length => int($iops_max_length),
4599         iops_rd_max_length => int($iops_rd_max_length),
4600         iops_wr_max_length => int($iops_wr_max_length),
4601     );
4602
4603 }
4604
4605 sub qemu_block_resize {
4606     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4607
4608     my $running = check_running($vmid);
4609
4610     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4611
4612     return if !$running;
4613
4614     my $padding = (1024 - $size % 1024) % 1024;
4615     $size = $size + $padding;
4616
4617     mon_cmd(
4618         $vmid,
4619         "block_resize",
4620         device => $deviceid,
4621         size => int($size),
4622         timeout => 60,
4623     );
4624 }
4625
4626 sub qemu_volume_snapshot {
4627     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4628
4629     my $running = check_running($vmid);
4630
4631     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4632         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4633     } else {
4634         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4635     }
4636 }
4637
4638 sub qemu_volume_snapshot_delete {
4639     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4640
4641     my $running = check_running($vmid);
4642
4643     if($running) {
4644
4645         $running = undef;
4646         my $conf = PVE::QemuConfig->load_config($vmid);
4647         PVE::QemuConfig->foreach_volume($conf, sub {
4648             my ($ds, $drive) = @_;
4649             $running = 1 if $drive->{file} eq $volid;
4650         });
4651     }
4652
4653     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4654         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4655     } else {
4656         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4657     }
4658 }
4659
4660 sub set_migration_caps {
4661     my ($vmid, $savevm) = @_;
4662
4663     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4664
4665     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4666     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4667
4668     my $cap_ref = [];
4669
4670     my $enabled_cap = {
4671         "auto-converge" => 1,
4672         "xbzrle" => 1,
4673         "x-rdma-pin-all" => 0,
4674         "zero-blocks" => 0,
4675         "compress" => 0,
4676         "dirty-bitmaps" => $dirty_bitmaps,
4677     };
4678
4679     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4680
4681     for my $supported_capability (@$supported_capabilities) {
4682         push @$cap_ref, {
4683             capability => $supported_capability->{capability},
4684             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4685         };
4686     }
4687
4688     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4689 }
4690
4691 sub foreach_volid {
4692     my ($conf, $func, @param) = @_;
4693
4694     my $volhash = {};
4695
4696     my $test_volid = sub {
4697         my ($key, $drive, $snapname) = @_;
4698
4699         my $volid = $drive->{file};
4700         return if !$volid;
4701
4702         $volhash->{$volid}->{cdrom} //= 1;
4703         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4704
4705         my $replicate = $drive->{replicate} // 1;
4706         $volhash->{$volid}->{replicate} //= 0;
4707         $volhash->{$volid}->{replicate} = 1 if $replicate;
4708
4709         $volhash->{$volid}->{shared} //= 0;
4710         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4711
4712         $volhash->{$volid}->{referenced_in_config} //= 0;
4713         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4714
4715         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4716             if defined($snapname);
4717
4718         my $size = $drive->{size};
4719         $volhash->{$volid}->{size} //= $size if $size;
4720
4721         $volhash->{$volid}->{is_vmstate} //= 0;
4722         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4723
4724         $volhash->{$volid}->{is_tpmstate} //= 0;
4725         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4726
4727         $volhash->{$volid}->{is_unused} //= 0;
4728         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4729
4730         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4731     };
4732
4733     my $include_opts = {
4734         extra_keys => ['vmstate'],
4735         include_unused => 1,
4736     };
4737
4738     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4739     foreach my $snapname (keys %{$conf->{snapshots}}) {
4740         my $snap = $conf->{snapshots}->{$snapname};
4741         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4742     }
4743
4744     foreach my $volid (keys %$volhash) {
4745         &$func($volid, $volhash->{$volid}, @param);
4746     }
4747 }
4748
4749 my $fast_plug_option = {
4750     'lock' => 1,
4751     'name' => 1,
4752     'onboot' => 1,
4753     'shares' => 1,
4754     'startup' => 1,
4755     'description' => 1,
4756     'protection' => 1,
4757     'vmstatestorage' => 1,
4758     'hookscript' => 1,
4759     'tags' => 1,
4760 };
4761
4762 # hotplug changes in [PENDING]
4763 # $selection hash can be used to only apply specified options, for
4764 # example: { cores => 1 } (only apply changed 'cores')
4765 # $errors ref is used to return error messages
4766 sub vmconfig_hotplug_pending {
4767     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4768
4769     my $defaults = load_defaults();
4770     my $arch = get_vm_arch($conf);
4771     my $machine_type = get_vm_machine($conf, undef, $arch);
4772
4773     # commit values which do not have any impact on running VM first
4774     # Note: those option cannot raise errors, we we do not care about
4775     # $selection and always apply them.
4776
4777     my $add_error = sub {
4778         my ($opt, $msg) = @_;
4779         $errors->{$opt} = "hotplug problem - $msg";
4780     };
4781
4782     my $changes = 0;
4783     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4784         if ($fast_plug_option->{$opt}) {
4785             $conf->{$opt} = $conf->{pending}->{$opt};
4786             delete $conf->{pending}->{$opt};
4787             $changes = 1;
4788         }
4789     }
4790
4791     if ($changes) {
4792         PVE::QemuConfig->write_config($vmid, $conf);
4793     }
4794
4795     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4796
4797     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4798     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4799     foreach my $opt (sort keys %$pending_delete_hash) {
4800         next if $selection && !$selection->{$opt};
4801         my $force = $pending_delete_hash->{$opt}->{force};
4802         eval {
4803             if ($opt eq 'hotplug') {
4804                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4805             } elsif ($opt eq 'tablet') {
4806                 die "skip\n" if !$hotplug_features->{usb};
4807                 if ($defaults->{tablet}) {
4808                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4809                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4810                         if $arch eq 'aarch64';
4811                 } else {
4812                     vm_deviceunplug($vmid, $conf, 'tablet');
4813                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4814                 }
4815             } elsif ($opt =~ m/^usb\d+/) {
4816                 die "skip\n";
4817                 # since we cannot reliably hot unplug usb devices we are disabling it
4818                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4819                 #vm_deviceunplug($vmid, $conf, $opt);
4820             } elsif ($opt eq 'vcpus') {
4821                 die "skip\n" if !$hotplug_features->{cpu};
4822                 qemu_cpu_hotplug($vmid, $conf, undef);
4823             } elsif ($opt eq 'balloon') {
4824                 # enable balloon device is not hotpluggable
4825                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4826                 # here we reset the ballooning value to memory
4827                 my $balloon = $conf->{memory} || $defaults->{memory};
4828                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4829             } elsif ($fast_plug_option->{$opt}) {
4830                 # do nothing
4831             } elsif ($opt =~ m/^net(\d+)$/) {
4832                 die "skip\n" if !$hotplug_features->{network};
4833                 vm_deviceunplug($vmid, $conf, $opt);
4834             } elsif (is_valid_drivename($opt)) {
4835                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4836                 vm_deviceunplug($vmid, $conf, $opt);
4837                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4838             } elsif ($opt =~ m/^memory$/) {
4839                 die "skip\n" if !$hotplug_features->{memory};
4840                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4841             } elsif ($opt eq 'cpuunits') {
4842                 $cgroup->change_cpu_shares(undef, 1024);
4843             } elsif ($opt eq 'cpulimit') {
4844                 $cgroup->change_cpu_quota(undef, undef); # reset, cgroup module can better decide values
4845             } else {
4846                 die "skip\n";
4847             }
4848         };
4849         if (my $err = $@) {
4850             &$add_error($opt, $err) if $err ne "skip\n";
4851         } else {
4852             delete $conf->{$opt};
4853             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4854         }
4855     }
4856
4857     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4858     $apply_pending_cloudinit = sub {
4859         return if $apply_pending_cloudinit_done; # once is enough
4860         $apply_pending_cloudinit_done = 1; # once is enough
4861
4862         my ($key, $value) = @_;
4863
4864         my @cloudinit_opts = keys %$confdesc_cloudinit;
4865         foreach my $opt (keys %{$conf->{pending}}) {
4866             next if !grep { $_ eq $opt } @cloudinit_opts;
4867             $conf->{$opt} = delete $conf->{pending}->{$opt};
4868         }
4869
4870         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4871         foreach my $opt (sort keys %$pending_delete_hash) {
4872             next if !grep { $_ eq $opt } @cloudinit_opts;
4873             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4874             delete $conf->{$opt};
4875         }
4876
4877         my $new_conf = { %$conf };
4878         $new_conf->{$key} = $value;
4879         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4880     };
4881
4882     foreach my $opt (keys %{$conf->{pending}}) {
4883         next if $selection && !$selection->{$opt};
4884         my $value = $conf->{pending}->{$opt};
4885         eval {
4886             if ($opt eq 'hotplug') {
4887                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4888             } elsif ($opt eq 'tablet') {
4889                 die "skip\n" if !$hotplug_features->{usb};
4890                 if ($value == 1) {
4891                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4892                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4893                         if $arch eq 'aarch64';
4894                 } elsif ($value == 0) {
4895                     vm_deviceunplug($vmid, $conf, 'tablet');
4896                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4897                 }
4898             } elsif ($opt =~ m/^usb\d+$/) {
4899                 die "skip\n";
4900                 # since we cannot reliably hot unplug usb devices we disable it for now
4901                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4902                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4903                 #die "skip\n" if !$d;
4904                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4905             } elsif ($opt eq 'vcpus') {
4906                 die "skip\n" if !$hotplug_features->{cpu};
4907                 qemu_cpu_hotplug($vmid, $conf, $value);
4908             } elsif ($opt eq 'balloon') {
4909                 # enable/disable balloning device is not hotpluggable
4910                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4911                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4912                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4913
4914                 # allow manual ballooning if shares is set to zero
4915                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4916                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4917                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4918                 }
4919             } elsif ($opt =~ m/^net(\d+)$/) {
4920                 # some changes can be done without hotplug
4921                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4922                                     $vmid, $opt, $value, $arch, $machine_type);
4923             } elsif (is_valid_drivename($opt)) {
4924                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4925                 # some changes can be done without hotplug
4926                 my $drive = parse_drive($opt, $value);
4927                 if (drive_is_cloudinit($drive)) {
4928                     &$apply_pending_cloudinit($opt, $value);
4929                 }
4930                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4931                                      $vmid, $opt, $value, $arch, $machine_type);
4932             } elsif ($opt =~ m/^memory$/) { #dimms
4933                 die "skip\n" if !$hotplug_features->{memory};
4934                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4935             } elsif ($opt eq 'cpuunits') {
4936                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4937             } elsif ($opt eq 'cpulimit') {
4938                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4939                 $cgroup->change_cpu_quota($cpulimit, 100000);
4940             } elsif ($opt eq 'agent') {
4941                 vmconfig_update_agent($conf, $opt, $value);
4942             } else {
4943                 die "skip\n";  # skip non-hot-pluggable options
4944             }
4945         };
4946         if (my $err = $@) {
4947             &$add_error($opt, $err) if $err ne "skip\n";
4948         } else {
4949             $conf->{$opt} = $value;
4950             delete $conf->{pending}->{$opt};
4951         }
4952     }
4953
4954     PVE::QemuConfig->write_config($vmid, $conf);
4955 }
4956
4957 sub try_deallocate_drive {
4958     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4959
4960     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4961         my $volid = $drive->{file};
4962         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4963             my $sid = PVE::Storage::parse_volume_id($volid);
4964             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4965
4966             # check if the disk is really unused
4967             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4968                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4969             PVE::Storage::vdisk_free($storecfg, $volid);
4970             return 1;
4971         } else {
4972             # If vm is not owner of this disk remove from config
4973             return 1;
4974         }
4975     }
4976
4977     return;
4978 }
4979
4980 sub vmconfig_delete_or_detach_drive {
4981     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4982
4983     my $drive = parse_drive($opt, $conf->{$opt});
4984
4985     my $rpcenv = PVE::RPCEnvironment::get();
4986     my $authuser = $rpcenv->get_user();
4987
4988     if ($force) {
4989         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4990         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4991     } else {
4992         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4993     }
4994 }
4995
4996
4997
4998 sub vmconfig_apply_pending {
4999     my ($vmid, $conf, $storecfg, $errors) = @_;
5000
5001     return if !scalar(keys %{$conf->{pending}});
5002
5003     my $add_apply_error = sub {
5004         my ($opt, $msg) = @_;
5005         my $err_msg = "unable to apply pending change $opt : $msg";
5006         $errors->{$opt} = $err_msg;
5007         warn $err_msg;
5008     };
5009
5010     # cold plug
5011
5012     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5013     foreach my $opt (sort keys %$pending_delete_hash) {
5014         my $force = $pending_delete_hash->{$opt}->{force};
5015         eval {
5016             if ($opt =~ m/^unused/) {
5017                 die "internal error";
5018             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5019                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5020             }
5021         };
5022         if (my $err = $@) {
5023             $add_apply_error->($opt, $err);
5024         } else {
5025             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5026             delete $conf->{$opt};
5027         }
5028     }
5029
5030     PVE::QemuConfig->cleanup_pending($conf);
5031
5032     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5033         next if $opt eq 'delete'; # just to be sure
5034         eval {
5035             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
5036                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5037             }
5038         };
5039         if (my $err = $@) {
5040             $add_apply_error->($opt, $err);
5041         } else {
5042             $conf->{$opt} = delete $conf->{pending}->{$opt};
5043         }
5044     }
5045
5046     # write all changes at once to avoid unnecessary i/o
5047     PVE::QemuConfig->write_config($vmid, $conf);
5048 }
5049
5050 sub vmconfig_update_net {
5051     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5052
5053     my $newnet = parse_net($value);
5054
5055     if ($conf->{$opt}) {
5056         my $oldnet = parse_net($conf->{$opt});
5057
5058         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5059             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5060             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5061             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5062
5063             # for non online change, we try to hot-unplug
5064             die "skip\n" if !$hotplug;
5065             vm_deviceunplug($vmid, $conf, $opt);
5066         } else {
5067
5068             die "internal error" if $opt !~ m/net(\d+)/;
5069             my $iface = "tap${vmid}i$1";
5070
5071             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5072                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5073                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5074                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5075                 PVE::Network::tap_unplug($iface);
5076
5077                 if ($have_sdn) {
5078                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5079                 } else {
5080                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5081                 }
5082             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5083                 # Rate can be applied on its own but any change above needs to
5084                 # include the rate in tap_plug since OVS resets everything.
5085                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5086             }
5087
5088             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5089                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5090             }
5091
5092             return 1;
5093         }
5094     }
5095
5096     if ($hotplug) {
5097         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5098     } else {
5099         die "skip\n";
5100     }
5101 }
5102
5103 sub vmconfig_update_agent {
5104     my ($conf, $opt, $value) = @_;
5105
5106     die "skip\n" if !$conf->{$opt};
5107
5108     my $hotplug_options = { fstrim_cloned_disks => 1 };
5109
5110     my $old_agent = parse_guest_agent($conf);
5111     my $agent = parse_guest_agent({$opt => $value});
5112
5113     for my $option (keys %$agent) { # added/changed options
5114         next if defined($hotplug_options->{$option});
5115         die "skip\n" if safe_string_ne($agent->{$option}, $old_agent->{$option});
5116     }
5117
5118     for my $option (keys %$old_agent) { # removed options
5119         next if defined($hotplug_options->{$option});
5120         die "skip\n" if safe_string_ne($old_agent->{$option}, $agent->{$option});
5121     }
5122
5123     return; # either no actual change (e.g., format string reordered) or just hotpluggable changes
5124 }
5125
5126 sub vmconfig_update_disk {
5127     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5128
5129     my $drive = parse_drive($opt, $value);
5130
5131     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5132         my $media = $drive->{media} || 'disk';
5133         my $oldmedia = $old_drive->{media} || 'disk';
5134         die "unable to change media type\n" if $media ne $oldmedia;
5135
5136         if (!drive_is_cdrom($old_drive)) {
5137
5138             if ($drive->{file} ne $old_drive->{file}) {
5139
5140                 die "skip\n" if !$hotplug;
5141
5142                 # unplug and register as unused
5143                 vm_deviceunplug($vmid, $conf, $opt);
5144                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5145
5146             } else {
5147                 # update existing disk
5148
5149                 # skip non hotpluggable value
5150                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5151                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5152                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5153                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5154                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5155                     die "skip\n";
5156                 }
5157
5158                 # apply throttle
5159                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5160                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5161                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5162                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5163                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5164                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5165                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5166                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5167                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5168                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5169                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5170                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5171                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5172                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5173                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5174                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5175                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5176                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5177
5178                     qemu_block_set_io_throttle(
5179                         $vmid,"drive-$opt",
5180                         ($drive->{mbps} || 0)*1024*1024,
5181                         ($drive->{mbps_rd} || 0)*1024*1024,
5182                         ($drive->{mbps_wr} || 0)*1024*1024,
5183                          $drive->{iops} || 0,
5184                          $drive->{iops_rd} || 0,
5185                          $drive->{iops_wr} || 0,
5186                         ($drive->{mbps_max} || 0)*1024*1024,
5187                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5188                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5189                          $drive->{iops_max} || 0,
5190                          $drive->{iops_rd_max} || 0,
5191                          $drive->{iops_wr_max} || 0,
5192                          $drive->{bps_max_length} || 1,
5193                          $drive->{bps_rd_max_length} || 1,
5194                          $drive->{bps_wr_max_length} || 1,
5195                          $drive->{iops_max_length} || 1,
5196                          $drive->{iops_rd_max_length} || 1,
5197                          $drive->{iops_wr_max_length} || 1,
5198                     );
5199
5200                 }
5201
5202                 return 1;
5203             }
5204
5205         } else { # cdrom
5206
5207             if ($drive->{file} eq 'none') {
5208                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5209                 if (drive_is_cloudinit($old_drive)) {
5210                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5211                 }
5212             } else {
5213                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5214
5215                 # force eject if locked
5216                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5217
5218                 if ($path) {
5219                     mon_cmd($vmid, "blockdev-change-medium",
5220                         id => "$opt", filename => "$path");
5221                 }
5222             }
5223
5224             return 1;
5225         }
5226     }
5227
5228     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5229     # hotplug new disks
5230     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5231     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5232 }
5233
5234 # called in locked context by incoming migration
5235 sub vm_migrate_get_nbd_disks {
5236     my ($storecfg, $conf, $replicated_volumes) = @_;
5237
5238     my $local_volumes = {};
5239     PVE::QemuConfig->foreach_volume($conf, sub {
5240         my ($ds, $drive) = @_;
5241
5242         return if drive_is_cdrom($drive);
5243         return if $ds eq 'tpmstate0';
5244
5245         my $volid = $drive->{file};
5246
5247         return if !$volid;
5248
5249         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5250
5251         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5252         return if $scfg->{shared};
5253
5254         # replicated disks re-use existing state via bitmap
5255         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5256         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5257     });
5258     return $local_volumes;
5259 }
5260
5261 # called in locked context by incoming migration
5262 sub vm_migrate_alloc_nbd_disks {
5263     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5264
5265     my $format = undef;
5266
5267     my $nbd = {};
5268     foreach my $opt (sort keys %$source_volumes) {
5269         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5270
5271         if ($use_existing) {
5272             $nbd->{$opt}->{drivestr} = print_drive($drive);
5273             $nbd->{$opt}->{volid} = $volid;
5274             $nbd->{$opt}->{replicated} = 1;
5275             next;
5276         }
5277
5278         # If a remote storage is specified and the format of the original
5279         # volume is not available there, fall back to the default format.
5280         # Otherwise use the same format as the original.
5281         if (!$storagemap->{identity}) {
5282             $storeid = map_storage($storagemap, $storeid);
5283             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5284             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5285             my $fileFormat = qemu_img_format($scfg, $volname);
5286             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5287         } else {
5288             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5289             $format = qemu_img_format($scfg, $volname);
5290         }
5291
5292         my $size = $drive->{size} / 1024;
5293         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5294         my $newdrive = $drive;
5295         $newdrive->{format} = $format;
5296         $newdrive->{file} = $newvolid;
5297         my $drivestr = print_drive($newdrive);
5298         $nbd->{$opt}->{drivestr} = $drivestr;
5299         $nbd->{$opt}->{volid} = $newvolid;
5300     }
5301
5302     return $nbd;
5303 }
5304
5305 # see vm_start_nolock for parameters, additionally:
5306 # migrate_opts:
5307 #   storagemap = parsed storage map for allocating NBD disks
5308 sub vm_start {
5309     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5310
5311     return PVE::QemuConfig->lock_config($vmid, sub {
5312         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5313
5314         die "you can't start a vm if it's a template\n"
5315             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5316
5317         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5318         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5319
5320         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5321
5322         if ($has_backup_lock && $running) {
5323             # a backup is currently running, attempt to start the guest in the
5324             # existing QEMU instance
5325             return vm_resume($vmid);
5326         }
5327
5328         PVE::QemuConfig->check_lock($conf)
5329             if !($params->{skiplock} || $has_suspended_lock);
5330
5331         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5332
5333         die "VM $vmid already running\n" if $running;
5334
5335         if (my $storagemap = $migrate_opts->{storagemap}) {
5336             my $replicated = $migrate_opts->{replicated_volumes};
5337             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5338             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5339
5340             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5341                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5342             }
5343         }
5344
5345         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5346     });
5347 }
5348
5349
5350 # params:
5351 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5352 #   skiplock => 0/1, skip checking for config lock
5353 #   skiptemplate => 0/1, skip checking whether VM is template
5354 #   forcemachine => to force Qemu machine (rollback/migration)
5355 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5356 #   timeout => in seconds
5357 #   paused => start VM in paused state (backup)
5358 #   resume => resume from hibernation
5359 #   pbs-backing => {
5360 #      sata0 => {
5361 #         repository
5362 #         snapshot
5363 #         keyfile
5364 #         archive
5365 #      },
5366 #      virtio2 => ...
5367 #   }
5368 # migrate_opts:
5369 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5370 #   migratedfrom => source node
5371 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5372 #   network => CIDR of migration network
5373 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5374 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5375 #   replicated_volumes => which volids should be re-used with bitmaps for nbd migration
5376 #   tpmstate_vol => new volid of tpmstate0, not yet contained in config
5377 sub vm_start_nolock {
5378     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5379
5380     my $statefile = $params->{statefile};
5381     my $resume = $params->{resume};
5382
5383     my $migratedfrom = $migrate_opts->{migratedfrom};
5384     my $migration_type = $migrate_opts->{type};
5385
5386     my $res = {};
5387
5388     # clean up leftover reboot request files
5389     eval { clear_reboot_request($vmid); };
5390     warn $@ if $@;
5391
5392     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5393         vmconfig_apply_pending($vmid, $conf, $storecfg);
5394         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5395     }
5396
5397     # don't regenerate the ISO if the VM is started as part of a live migration
5398     # this way we can reuse the old ISO with the correct config
5399     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5400
5401     # override TPM state vol if migrated, conf is out of date still
5402     if (my $tpmvol = $migrate_opts->{tpmstate_vol}) {
5403         my $parsed = parse_drive("tpmstate0", $conf->{tpmstate0});
5404         $parsed->{file} = $tpmvol;
5405         $conf->{tpmstate0} = print_drive($parsed);
5406     }
5407
5408     my $defaults = load_defaults();
5409
5410     # set environment variable useful inside network script
5411     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5412
5413     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5414
5415     my $forcemachine = $params->{forcemachine};
5416     my $forcecpu = $params->{forcecpu};
5417     if ($resume) {
5418         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5419         $forcemachine = $conf->{runningmachine};
5420         $forcecpu = $conf->{runningcpu};
5421         print "Resuming suspended VM\n";
5422     }
5423
5424     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5425         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5426
5427     my $migration_ip;
5428     my $get_migration_ip = sub {
5429         my ($nodename) = @_;
5430
5431         return $migration_ip if defined($migration_ip);
5432
5433         my $cidr = $migrate_opts->{network};
5434
5435         if (!defined($cidr)) {
5436             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5437             $cidr = $dc_conf->{migration}->{network};
5438         }
5439
5440         if (defined($cidr)) {
5441             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5442
5443             die "could not get IP: no address configured on local " .
5444                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5445
5446             die "could not get IP: multiple addresses configured on local " .
5447                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5448
5449             $migration_ip = @$ips[0];
5450         }
5451
5452         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5453             if !defined($migration_ip);
5454
5455         return $migration_ip;
5456     };
5457
5458     my $migrate_uri;
5459     if ($statefile) {
5460         if ($statefile eq 'tcp') {
5461             my $localip = "localhost";
5462             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5463             my $nodename = nodename();
5464
5465             if (!defined($migration_type)) {
5466                 if (defined($datacenterconf->{migration}->{type})) {
5467                     $migration_type = $datacenterconf->{migration}->{type};
5468                 } else {
5469                     $migration_type = 'secure';
5470                 }
5471             }
5472
5473             if ($migration_type eq 'insecure') {
5474                 $localip = $get_migration_ip->($nodename);
5475                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5476             }
5477
5478             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5479             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5480             $migrate_uri = "tcp:${localip}:${migrate_port}";
5481             push @$cmd, '-incoming', $migrate_uri;
5482             push @$cmd, '-S';
5483
5484         } elsif ($statefile eq 'unix') {
5485             # should be default for secure migrations as a ssh TCP forward
5486             # tunnel is not deterministic reliable ready and fails regurarly
5487             # to set up in time, so use UNIX socket forwards
5488             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5489             unlink $socket_addr;
5490
5491             $migrate_uri = "unix:$socket_addr";
5492
5493             push @$cmd, '-incoming', $migrate_uri;
5494             push @$cmd, '-S';
5495
5496         } elsif (-e $statefile) {
5497             push @$cmd, '-loadstate', $statefile;
5498         } else {
5499             my $statepath = PVE::Storage::path($storecfg, $statefile);
5500             push @$vollist, $statefile;
5501             push @$cmd, '-loadstate', $statepath;
5502         }
5503     } elsif ($params->{paused}) {
5504         push @$cmd, '-S';
5505     }
5506
5507     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5508
5509     my $pci_devices = {}; # host pci devices
5510     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5511         my $dev = $conf->{"hostpci$i"} or next;
5512         $pci_devices->{$i} = parse_hostpci($dev);
5513     }
5514
5515     # do not reserve pciid for mediated devices, sysfs will error out for duplicate assignment
5516     my $real_pci_devices = [ grep { !(defined($_->{mdev}) && scalar($_->{pciid}->@*) == 1) } values $pci_devices->%* ];
5517
5518     # map to a flat list of pci ids
5519     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } $real_pci_devices->@* ];
5520
5521     # reserve all PCI IDs before actually doing anything with them
5522     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5523
5524     eval {
5525         for my $id (sort keys %$pci_devices) {
5526             my $d = $pci_devices->{$id};
5527             for my $dev ($d->{pciid}->@*) {
5528                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5529             }
5530         }
5531     };
5532     if (my $err = $@) {
5533         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5534         warn $@ if $@;
5535         die $err;
5536     }
5537
5538     PVE::Storage::activate_volumes($storecfg, $vollist);
5539
5540     eval {
5541         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5542     };
5543     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5544     # timeout should be more than enough here...
5545     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5546
5547     my $cpuunits = get_cpuunits($conf);
5548
5549     my %run_params = (
5550         timeout => $statefile ? undef : $start_timeout,
5551         umask => 0077,
5552         noerr => 1,
5553     );
5554
5555     # when migrating, prefix QEMU output so other side can pick up any
5556     # errors that might occur and show the user
5557     if ($migratedfrom) {
5558         $run_params{quiet} = 1;
5559         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5560     }
5561
5562     my %systemd_properties = (
5563         Slice => 'qemu.slice',
5564         KillMode => 'process',
5565         SendSIGKILL => 0,
5566         TimeoutStopUSec => ULONG_MAX, # infinity
5567     );
5568
5569     if (PVE::CGroup::cgroup_mode() == 2) {
5570         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5571         $systemd_properties{CPUWeight} = $cpuunits;
5572     } else {
5573         $systemd_properties{CPUShares} = $cpuunits;
5574     }
5575
5576     if (my $cpulimit = $conf->{cpulimit}) {
5577         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5578     }
5579     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5580
5581     my $run_qemu = sub {
5582         PVE::Tools::run_fork sub {
5583             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5584
5585             my $tpmpid;
5586             if (my $tpm = $conf->{tpmstate0}) {
5587                 # start the TPM emulator so QEMU can connect on start
5588                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5589             }
5590
5591             my $exitcode = run_command($cmd, %run_params);
5592             if ($exitcode) {
5593                 if ($tpmpid) {
5594                     warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5595                     kill 'TERM', $tpmpid;
5596                 }
5597                 die "QEMU exited with code $exitcode\n";
5598             }
5599         };
5600     };
5601
5602     if ($conf->{hugepages}) {
5603
5604         my $code = sub {
5605             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5606             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5607
5608             PVE::QemuServer::Memory::hugepages_mount();
5609             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5610
5611             eval { $run_qemu->() };
5612             if (my $err = $@) {
5613                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5614                     if !$conf->{keephugepages};
5615                 die $err;
5616             }
5617
5618             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5619                 if !$conf->{keephugepages};
5620         };
5621         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5622
5623     } else {
5624         eval { $run_qemu->() };
5625     }
5626
5627     if (my $err = $@) {
5628         # deactivate volumes if start fails
5629         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5630         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5631
5632         die "start failed: $err";
5633     }
5634
5635     # re-reserve all PCI IDs now that we can know the actual VM PID
5636     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5637     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5638     warn $@ if $@;
5639
5640     print "migration listens on $migrate_uri\n" if $migrate_uri;
5641     $res->{migrate_uri} = $migrate_uri;
5642
5643     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5644         eval { mon_cmd($vmid, "cont"); };
5645         warn $@ if $@;
5646     }
5647
5648     #start nbd server for storage migration
5649     if (my $nbd = $migrate_opts->{nbd}) {
5650         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5651
5652         my $migrate_storage_uri;
5653         # nbd_protocol_version > 0 for unix socket support
5654         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5655             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5656             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5657             $migrate_storage_uri = "nbd:unix:$socket_path";
5658         } else {
5659             my $nodename = nodename();
5660             my $localip = $get_migration_ip->($nodename);
5661             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5662             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5663
5664             mon_cmd($vmid, "nbd-server-start", addr => {
5665                 type => 'inet',
5666                 data => {
5667                     host => "${localip}",
5668                     port => "${storage_migrate_port}",
5669                 },
5670             });
5671             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5672             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5673         }
5674
5675         $res->{migrate_storage_uri} = $migrate_storage_uri;
5676
5677         foreach my $opt (sort keys %$nbd) {
5678             my $drivestr = $nbd->{$opt}->{drivestr};
5679             my $volid = $nbd->{$opt}->{volid};
5680             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5681             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5682             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5683             print "re-using replicated volume: $opt - $volid\n"
5684                 if $nbd->{$opt}->{replicated};
5685
5686             $res->{drives}->{$opt} = $nbd->{$opt};
5687             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5688         }
5689     }
5690
5691     if ($migratedfrom) {
5692         eval {
5693             set_migration_caps($vmid);
5694         };
5695         warn $@ if $@;
5696
5697         if ($spice_port) {
5698             print "spice listens on port $spice_port\n";
5699             $res->{spice_port} = $spice_port;
5700             if ($migrate_opts->{spice_ticket}) {
5701                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5702                     $migrate_opts->{spice_ticket});
5703                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5704             }
5705         }
5706
5707     } else {
5708         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5709             if !$statefile && $conf->{balloon};
5710
5711         foreach my $opt (keys %$conf) {
5712             next if $opt !~  m/^net\d+$/;
5713             my $nicconf = parse_net($conf->{$opt});
5714             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5715         }
5716     }
5717
5718     mon_cmd($vmid, 'qom-set',
5719                 path => "machine/peripheral/balloon0",
5720                 property => "guest-stats-polling-interval",
5721                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5722
5723     if ($resume) {
5724         print "Resumed VM, removing state\n";
5725         if (my $vmstate = $conf->{vmstate}) {
5726             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5727             PVE::Storage::vdisk_free($storecfg, $vmstate);
5728         }
5729         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5730         PVE::QemuConfig->write_config($vmid, $conf);
5731     }
5732
5733     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5734
5735     return $res;
5736 }
5737
5738 sub vm_commandline {
5739     my ($storecfg, $vmid, $snapname) = @_;
5740
5741     my $conf = PVE::QemuConfig->load_config($vmid);
5742
5743     my ($forcemachine, $forcecpu);
5744     if ($snapname) {
5745         my $snapshot = $conf->{snapshots}->{$snapname};
5746         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5747
5748         # check for machine or CPU overrides in snapshot
5749         $forcemachine = $snapshot->{runningmachine};
5750         $forcecpu = $snapshot->{runningcpu};
5751
5752         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5753
5754         $conf = $snapshot;
5755     }
5756
5757     my $defaults = load_defaults();
5758
5759     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5760
5761     return PVE::Tools::cmd2string($cmd);
5762 }
5763
5764 sub vm_reset {
5765     my ($vmid, $skiplock) = @_;
5766
5767     PVE::QemuConfig->lock_config($vmid, sub {
5768
5769         my $conf = PVE::QemuConfig->load_config($vmid);
5770
5771         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5772
5773         mon_cmd($vmid, "system_reset");
5774     });
5775 }
5776
5777 sub get_vm_volumes {
5778     my ($conf) = @_;
5779
5780     my $vollist = [];
5781     foreach_volid($conf, sub {
5782         my ($volid, $attr) = @_;
5783
5784         return if $volid =~ m|^/|;
5785
5786         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5787         return if !$sid;
5788
5789         push @$vollist, $volid;
5790     });
5791
5792     return $vollist;
5793 }
5794
5795 sub vm_stop_cleanup {
5796     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5797
5798     eval {
5799
5800         if (!$keepActive) {
5801             my $vollist = get_vm_volumes($conf);
5802             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5803
5804             if (my $tpmdrive = $conf->{tpmstate0}) {
5805                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5806                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5807                 if ($storeid) {
5808                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5809                 }
5810             }
5811         }
5812
5813         foreach my $ext (qw(mon qmp pid vnc qga)) {
5814             unlink "/var/run/qemu-server/${vmid}.$ext";
5815         }
5816
5817         if ($conf->{ivshmem}) {
5818             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5819             # just delete it for now, VMs which have this already open do not
5820             # are affected, but new VMs will get a separated one. If this
5821             # becomes an issue we either add some sort of ref-counting or just
5822             # add a "don't delete on stop" flag to the ivshmem format.
5823             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5824         }
5825
5826         my $ids = [];
5827         foreach my $key (keys %$conf) {
5828             next if $key !~ m/^hostpci(\d+)$/;
5829             my $hostpciindex = $1;
5830             my $d = parse_hostpci($conf->{$key});
5831             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5832
5833             foreach my $pci (@{$d->{pciid}}) {
5834                 my $pciid = $pci->{id};
5835                 push @$ids, $pci->{id};
5836                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5837             }
5838         }
5839         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5840
5841         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5842     };
5843     warn $@ if $@; # avoid errors - just warn
5844 }
5845
5846 # call only in locked context
5847 sub _do_vm_stop {
5848     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5849
5850     my $pid = check_running($vmid, $nocheck);
5851     return if !$pid;
5852
5853     my $conf;
5854     if (!$nocheck) {
5855         $conf = PVE::QemuConfig->load_config($vmid);
5856         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5857         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5858             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5859             $timeout = $opts->{down} if $opts->{down};
5860         }
5861         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5862     }
5863
5864     eval {
5865         if ($shutdown) {
5866             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5867                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5868             } else {
5869                 mon_cmd($vmid, "system_powerdown");
5870             }
5871         } else {
5872             mon_cmd($vmid, "quit");
5873         }
5874     };
5875     my $err = $@;
5876
5877     if (!$err) {
5878         $timeout = 60 if !defined($timeout);
5879
5880         my $count = 0;
5881         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5882             $count++;
5883             sleep 1;
5884         }
5885
5886         if ($count >= $timeout) {
5887             if ($force) {
5888                 warn "VM still running - terminating now with SIGTERM\n";
5889                 kill 15, $pid;
5890             } else {
5891                 die "VM quit/powerdown failed - got timeout\n";
5892             }
5893         } else {
5894             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5895             return;
5896         }
5897     } else {
5898        if (!check_running($vmid, $nocheck)) {
5899             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5900             return;
5901        }
5902        if ($force) {
5903             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5904             kill 15, $pid;
5905         } else {
5906             die "VM quit/powerdown failed\n";
5907         }
5908     }
5909
5910     # wait again
5911     $timeout = 10;
5912
5913     my $count = 0;
5914     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5915         $count++;
5916         sleep 1;
5917     }
5918
5919     if ($count >= $timeout) {
5920         warn "VM still running - terminating now with SIGKILL\n";
5921         kill 9, $pid;
5922         sleep 1;
5923     }
5924
5925     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5926 }
5927
5928 # Note: use $nocheck to skip tests if VM configuration file exists.
5929 # We need that when migration VMs to other nodes (files already moved)
5930 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5931 sub vm_stop {
5932     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5933
5934     $force = 1 if !defined($force) && !$shutdown;
5935
5936     if ($migratedfrom){
5937         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5938         kill 15, $pid if $pid;
5939         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5940         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5941         return;
5942     }
5943
5944     PVE::QemuConfig->lock_config($vmid, sub {
5945         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5946    });
5947 }
5948
5949 sub vm_reboot {
5950     my ($vmid, $timeout) = @_;
5951
5952     PVE::QemuConfig->lock_config($vmid, sub {
5953         eval {
5954
5955             # only reboot if running, as qmeventd starts it again on a stop event
5956             return if !check_running($vmid);
5957
5958             create_reboot_request($vmid);
5959
5960             my $storecfg = PVE::Storage::config();
5961             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5962
5963         };
5964         if (my $err = $@) {
5965             # avoid that the next normal shutdown will be confused for a reboot
5966             clear_reboot_request($vmid);
5967             die $err;
5968         }
5969    });
5970 }
5971
5972 # note: if using the statestorage parameter, the caller has to check privileges
5973 sub vm_suspend {
5974     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5975
5976     my $conf;
5977     my $path;
5978     my $storecfg;
5979     my $vmstate;
5980
5981     PVE::QemuConfig->lock_config($vmid, sub {
5982
5983         $conf = PVE::QemuConfig->load_config($vmid);
5984
5985         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5986         PVE::QemuConfig->check_lock($conf)
5987             if !($skiplock || $is_backing_up);
5988
5989         die "cannot suspend to disk during backup\n"
5990             if $is_backing_up && $includestate;
5991
5992         if ($includestate) {
5993             $conf->{lock} = 'suspending';
5994             my $date = strftime("%Y-%m-%d", localtime(time()));
5995             $storecfg = PVE::Storage::config();
5996             if (!$statestorage) {
5997                 $statestorage = find_vmstate_storage($conf, $storecfg);
5998                 # check permissions for the storage
5999                 my $rpcenv = PVE::RPCEnvironment::get();
6000                 if ($rpcenv->{type} ne 'cli') {
6001                     my $authuser = $rpcenv->get_user();
6002                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
6003                 }
6004             }
6005
6006
6007             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
6008                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
6009             $path = PVE::Storage::path($storecfg, $vmstate);
6010             PVE::QemuConfig->write_config($vmid, $conf);
6011         } else {
6012             mon_cmd($vmid, "stop");
6013         }
6014     });
6015
6016     if ($includestate) {
6017         # save vm state
6018         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
6019
6020         eval {
6021             set_migration_caps($vmid, 1);
6022             mon_cmd($vmid, "savevm-start", statefile => $path);
6023             for(;;) {
6024                 my $state = mon_cmd($vmid, "query-savevm");
6025                 if (!$state->{status}) {
6026                     die "savevm not active\n";
6027                 } elsif ($state->{status} eq 'active') {
6028                     sleep(1);
6029                     next;
6030                 } elsif ($state->{status} eq 'completed') {
6031                     print "State saved, quitting\n";
6032                     last;
6033                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
6034                     die "query-savevm failed with error '$state->{error}'\n"
6035                 } else {
6036                     die "query-savevm returned status '$state->{status}'\n";
6037                 }
6038             }
6039         };
6040         my $err = $@;
6041
6042         PVE::QemuConfig->lock_config($vmid, sub {
6043             $conf = PVE::QemuConfig->load_config($vmid);
6044             if ($err) {
6045                 # cleanup, but leave suspending lock, to indicate something went wrong
6046                 eval {
6047                     mon_cmd($vmid, "savevm-end");
6048                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
6049                     PVE::Storage::vdisk_free($storecfg, $vmstate);
6050                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
6051                     PVE::QemuConfig->write_config($vmid, $conf);
6052                 };
6053                 warn $@ if $@;
6054                 die $err;
6055             }
6056
6057             die "lock changed unexpectedly\n"
6058                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
6059
6060             mon_cmd($vmid, "quit");
6061             $conf->{lock} = 'suspended';
6062             PVE::QemuConfig->write_config($vmid, $conf);
6063         });
6064     }
6065 }
6066
6067 sub vm_resume {
6068     my ($vmid, $skiplock, $nocheck) = @_;
6069
6070     PVE::QemuConfig->lock_config($vmid, sub {
6071         my $res = mon_cmd($vmid, 'query-status');
6072         my $resume_cmd = 'cont';
6073         my $reset = 0;
6074
6075         if ($res->{status}) {
6076             return if $res->{status} eq 'running'; # job done, go home
6077             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
6078             $reset = 1 if $res->{status} eq 'shutdown';
6079         }
6080
6081         if (!$nocheck) {
6082
6083             my $conf = PVE::QemuConfig->load_config($vmid);
6084
6085             PVE::QemuConfig->check_lock($conf)
6086                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
6087         }
6088
6089         if ($reset) {
6090             # required if a VM shuts down during a backup and we get a resume
6091             # request before the backup finishes for example
6092             mon_cmd($vmid, "system_reset");
6093         }
6094         mon_cmd($vmid, $resume_cmd);
6095     });
6096 }
6097
6098 sub vm_sendkey {
6099     my ($vmid, $skiplock, $key) = @_;
6100
6101     PVE::QemuConfig->lock_config($vmid, sub {
6102
6103         my $conf = PVE::QemuConfig->load_config($vmid);
6104
6105         # there is no qmp command, so we use the human monitor command
6106         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
6107         die $res if $res ne '';
6108     });
6109 }
6110
6111 # vzdump restore implementaion
6112
6113 sub tar_archive_read_firstfile {
6114     my $archive = shift;
6115
6116     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6117
6118     # try to detect archive type first
6119     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6120         die "unable to open file '$archive'\n";
6121     my $firstfile = <$fh>;
6122     kill 15, $pid;
6123     close $fh;
6124
6125     die "ERROR: archive contaions no data\n" if !$firstfile;
6126     chomp $firstfile;
6127
6128     return $firstfile;
6129 }
6130
6131 sub tar_restore_cleanup {
6132     my ($storecfg, $statfile) = @_;
6133
6134     print STDERR "starting cleanup\n";
6135
6136     if (my $fd = IO::File->new($statfile, "r")) {
6137         while (defined(my $line = <$fd>)) {
6138             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6139                 my $volid = $2;
6140                 eval {
6141                     if ($volid =~ m|^/|) {
6142                         unlink $volid || die 'unlink failed\n';
6143                     } else {
6144                         PVE::Storage::vdisk_free($storecfg, $volid);
6145                     }
6146                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6147                 };
6148                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6149             } else {
6150                 print STDERR "unable to parse line in statfile - $line";
6151             }
6152         }
6153         $fd->close();
6154     }
6155 }
6156
6157 sub restore_file_archive {
6158     my ($archive, $vmid, $user, $opts) = @_;
6159
6160     return restore_vma_archive($archive, $vmid, $user, $opts)
6161         if $archive eq '-';
6162
6163     my $info = PVE::Storage::archive_info($archive);
6164     my $format = $opts->{format} // $info->{format};
6165     my $comp = $info->{compression};
6166
6167     # try to detect archive format
6168     if ($format eq 'tar') {
6169         return restore_tar_archive($archive, $vmid, $user, $opts);
6170     } else {
6171         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6172     }
6173 }
6174
6175 # hepler to remove disks that will not be used after restore
6176 my $restore_cleanup_oldconf = sub {
6177     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6178
6179     PVE::QemuConfig->foreach_volume($oldconf, sub {
6180         my ($ds, $drive) = @_;
6181
6182         return if drive_is_cdrom($drive, 1);
6183
6184         my $volid = $drive->{file};
6185         return if !$volid || $volid =~ m|^/|;
6186
6187         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6188         return if !$path || !$owner || ($owner != $vmid);
6189
6190         # Note: only delete disk we want to restore
6191         # other volumes will become unused
6192         if ($virtdev_hash->{$ds}) {
6193             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6194             if (my $err = $@) {
6195                 warn $err;
6196             }
6197         }
6198     });
6199
6200     # delete vmstate files, after the restore we have no snapshots anymore
6201     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6202         my $snap = $oldconf->{snapshots}->{$snapname};
6203         if ($snap->{vmstate}) {
6204             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6205             if (my $err = $@) {
6206                 warn $err;
6207             }
6208         }
6209     }
6210 };
6211
6212 # Helper to parse vzdump backup device hints
6213 #
6214 # $rpcenv: Environment, used to ckeck storage permissions
6215 # $user: User ID, to check storage permissions
6216 # $storecfg: Storage configuration
6217 # $fh: the file handle for reading the configuration
6218 # $devinfo: should contain device sizes for all backu-up'ed devices
6219 # $options: backup options (pool, default storage)
6220 #
6221 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6222 my $parse_backup_hints = sub {
6223     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6224
6225     my $virtdev_hash = {};
6226
6227     while (defined(my $line = <$fh>)) {
6228         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6229             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6230             die "archive does not contain data for drive '$virtdev'\n"
6231                 if !$devinfo->{$devname};
6232
6233             if (defined($options->{storage})) {
6234                 $storeid = $options->{storage} || 'local';
6235             } elsif (!$storeid) {
6236                 $storeid = 'local';
6237             }
6238             $format = 'raw' if !$format;
6239             $devinfo->{$devname}->{devname} = $devname;
6240             $devinfo->{$devname}->{virtdev} = $virtdev;
6241             $devinfo->{$devname}->{format} = $format;
6242             $devinfo->{$devname}->{storeid} = $storeid;
6243
6244             # check permission on storage
6245             my $pool = $options->{pool}; # todo: do we need that?
6246             if ($user ne 'root@pam') {
6247                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6248             }
6249
6250             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6251         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6252             my $virtdev = $1;
6253             my $drive = parse_drive($virtdev, $2);
6254             if (drive_is_cloudinit($drive)) {
6255                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6256                 $storeid = $options->{storage} if defined ($options->{storage});
6257                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6258                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6259
6260                 $virtdev_hash->{$virtdev} = {
6261                     format => $format,
6262                     storeid => $storeid,
6263                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6264                     is_cloudinit => 1,
6265                 };
6266             }
6267         }
6268     }
6269
6270     return $virtdev_hash;
6271 };
6272
6273 # Helper to allocate and activate all volumes required for a restore
6274 #
6275 # $storecfg: Storage configuration
6276 # $virtdev_hash: as returned by parse_backup_hints()
6277 #
6278 # Returns: { $virtdev => $volid }
6279 my $restore_allocate_devices = sub {
6280     my ($storecfg, $virtdev_hash, $vmid) = @_;
6281
6282     my $map = {};
6283     foreach my $virtdev (sort keys %$virtdev_hash) {
6284         my $d = $virtdev_hash->{$virtdev};
6285         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6286         my $storeid = $d->{storeid};
6287         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6288
6289         # test if requested format is supported
6290         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6291         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6292         $d->{format} = $defFormat if !$supported;
6293
6294         my $name;
6295         if ($d->{is_cloudinit}) {
6296             $name = "vm-$vmid-cloudinit";
6297             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6298             if ($scfg->{path}) {
6299                 $name .= ".$d->{format}";
6300             }
6301         }
6302
6303         my $volid = PVE::Storage::vdisk_alloc(
6304             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6305
6306         print STDERR "new volume ID is '$volid'\n";
6307         $d->{volid} = $volid;
6308
6309         PVE::Storage::activate_volumes($storecfg, [$volid]);
6310
6311         $map->{$virtdev} = $volid;
6312     }
6313
6314     return $map;
6315 };
6316
6317 sub restore_update_config_line {
6318     my ($cookie, $map, $line, $unique) = @_;
6319
6320     return '' if $line =~ m/^\#qmdump\#/;
6321     return '' if $line =~ m/^\#vzdump\#/;
6322     return '' if $line =~ m/^lock:/;
6323     return '' if $line =~ m/^unused\d+:/;
6324     return '' if $line =~ m/^parent:/;
6325
6326     my $res = '';
6327
6328     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6329     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6330         # try to convert old 1.X settings
6331         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6332         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6333             my ($model, $macaddr) = split(/\=/, $devconfig);
6334             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6335             my $net = {
6336                 model => $model,
6337                 bridge => "vmbr$ind",
6338                 macaddr => $macaddr,
6339             };
6340             my $netstr = print_net($net);
6341
6342             $res .= "net$cookie->{netcount}: $netstr\n";
6343             $cookie->{netcount}++;
6344         }
6345     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6346         my ($id, $netstr) = ($1, $2);
6347         my $net = parse_net($netstr);
6348         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6349         $netstr = print_net($net);
6350         $res .= "$id: $netstr\n";
6351     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6352         my $virtdev = $1;
6353         my $value = $3;
6354         my $di = parse_drive($virtdev, $value);
6355         if (defined($di->{backup}) && !$di->{backup}) {
6356             $res .= "#$line";
6357         } elsif ($map->{$virtdev}) {
6358             delete $di->{format}; # format can change on restore
6359             $di->{file} = $map->{$virtdev};
6360             $value = print_drive($di);
6361             $res .= "$virtdev: $value\n";
6362         } else {
6363             $res .= $line;
6364         }
6365     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6366         my $vmgenid = $1;
6367         if ($vmgenid ne '0') {
6368             # always generate a new vmgenid if there was a valid one setup
6369             $vmgenid = generate_uuid();
6370         }
6371         $res .= "vmgenid: $vmgenid\n";
6372     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6373         my ($uuid, $uuid_str);
6374         UUID::generate($uuid);
6375         UUID::unparse($uuid, $uuid_str);
6376         my $smbios1 = parse_smbios1($2);
6377         $smbios1->{uuid} = $uuid_str;
6378         $res .= $1.print_smbios1($smbios1)."\n";
6379     } else {
6380         $res .= $line;
6381     }
6382
6383     return $res;
6384 }
6385
6386 my $restore_deactivate_volumes = sub {
6387     my ($storecfg, $devinfo) = @_;
6388
6389     my $vollist = [];
6390     foreach my $devname (keys %$devinfo) {
6391         my $volid = $devinfo->{$devname}->{volid};
6392         push @$vollist, $volid if $volid;
6393     }
6394
6395     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6396 };
6397
6398 my $restore_destroy_volumes = sub {
6399     my ($storecfg, $devinfo) = @_;
6400
6401     foreach my $devname (keys %$devinfo) {
6402         my $volid = $devinfo->{$devname}->{volid};
6403         next if !$volid;
6404         eval {
6405             if ($volid =~ m|^/|) {
6406                 unlink $volid || die 'unlink failed\n';
6407             } else {
6408                 PVE::Storage::vdisk_free($storecfg, $volid);
6409             }
6410             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6411         };
6412         print STDERR "unable to cleanup '$volid' - $@" if $@;
6413     }
6414 };
6415
6416 sub scan_volids {
6417     my ($cfg, $vmid) = @_;
6418
6419     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6420
6421     my $volid_hash = {};
6422     foreach my $storeid (keys %$info) {
6423         foreach my $item (@{$info->{$storeid}}) {
6424             next if !($item->{volid} && $item->{size});
6425             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6426             $volid_hash->{$item->{volid}} = $item;
6427         }
6428     }
6429
6430     return $volid_hash;
6431 }
6432
6433 sub update_disk_config {
6434     my ($vmid, $conf, $volid_hash) = @_;
6435
6436     my $changes;
6437     my $prefix = "VM $vmid";
6438
6439     # used and unused disks
6440     my $referenced = {};
6441
6442     # Note: it is allowed to define multiple storages with same path (alias), so
6443     # we need to check both 'volid' and real 'path' (two different volid can point
6444     # to the same path).
6445
6446     my $referencedpath = {};
6447
6448     # update size info
6449     PVE::QemuConfig->foreach_volume($conf, sub {
6450         my ($opt, $drive) = @_;
6451
6452         my $volid = $drive->{file};
6453         return if !$volid;
6454         my $volume = $volid_hash->{$volid};
6455
6456         # mark volid as "in-use" for next step
6457         $referenced->{$volid} = 1;
6458         if ($volume && (my $path = $volume->{path})) {
6459             $referencedpath->{$path} = 1;
6460         }
6461
6462         return if drive_is_cdrom($drive);
6463         return if !$volume;
6464
6465         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6466         if (defined($updated)) {
6467             $changes = 1;
6468             $conf->{$opt} = print_drive($updated);
6469             print "$prefix ($opt): $msg\n";
6470         }
6471     });
6472
6473     # remove 'unusedX' entry if volume is used
6474     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6475         my ($opt, $drive) = @_;
6476
6477         my $volid = $drive->{file};
6478         return if !$volid;
6479
6480         my $path;
6481         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6482         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6483             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6484             $changes = 1;
6485             delete $conf->{$opt};
6486         }
6487
6488         $referenced->{$volid} = 1;
6489         $referencedpath->{$path} = 1 if $path;
6490     });
6491
6492     foreach my $volid (sort keys %$volid_hash) {
6493         next if $volid =~ m/vm-$vmid-state-/;
6494         next if $referenced->{$volid};
6495         my $path = $volid_hash->{$volid}->{path};
6496         next if !$path; # just to be sure
6497         next if $referencedpath->{$path};
6498         $changes = 1;
6499         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6500         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6501         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6502     }
6503
6504     return $changes;
6505 }
6506
6507 sub rescan {
6508     my ($vmid, $nolock, $dryrun) = @_;
6509
6510     my $cfg = PVE::Storage::config();
6511
6512     print "rescan volumes...\n";
6513     my $volid_hash = scan_volids($cfg, $vmid);
6514
6515     my $updatefn =  sub {
6516         my ($vmid) = @_;
6517
6518         my $conf = PVE::QemuConfig->load_config($vmid);
6519
6520         PVE::QemuConfig->check_lock($conf);
6521
6522         my $vm_volids = {};
6523         foreach my $volid (keys %$volid_hash) {
6524             my $info = $volid_hash->{$volid};
6525             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6526         }
6527
6528         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6529
6530         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6531     };
6532
6533     if (defined($vmid)) {
6534         if ($nolock) {
6535             &$updatefn($vmid);
6536         } else {
6537             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6538         }
6539     } else {
6540         my $vmlist = config_list();
6541         foreach my $vmid (keys %$vmlist) {
6542             if ($nolock) {
6543                 &$updatefn($vmid);
6544             } else {
6545                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6546             }
6547         }
6548     }
6549 }
6550
6551 sub restore_proxmox_backup_archive {
6552     my ($archive, $vmid, $user, $options) = @_;
6553
6554     my $storecfg = PVE::Storage::config();
6555
6556     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6557     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6558
6559     my $fingerprint = $scfg->{fingerprint};
6560     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6561
6562     my $repo = PVE::PBSClient::get_repository($scfg);
6563
6564     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6565     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6566     local $ENV{PBS_PASSWORD} = $password;
6567     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6568
6569     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6570         PVE::Storage::parse_volname($storecfg, $archive);
6571
6572     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6573
6574     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6575
6576     my $tmpdir = "/var/tmp/vzdumptmp$$";
6577     rmtree $tmpdir;
6578     mkpath $tmpdir;
6579
6580     my $conffile = PVE::QemuConfig->config_file($vmid);
6581      # disable interrupts (always do cleanups)
6582     local $SIG{INT} =
6583         local $SIG{TERM} =
6584         local $SIG{QUIT} =
6585         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6586
6587     # Note: $oldconf is undef if VM does not exists
6588     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6589     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6590     my $new_conf_raw = '';
6591
6592     my $rpcenv = PVE::RPCEnvironment::get();
6593     my $devinfo = {};
6594
6595     eval {
6596         # enable interrupts
6597         local $SIG{INT} =
6598             local $SIG{TERM} =
6599             local $SIG{QUIT} =
6600             local $SIG{HUP} =
6601             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6602
6603         my $cfgfn = "$tmpdir/qemu-server.conf";
6604         my $firewall_config_fn = "$tmpdir/fw.conf";
6605         my $index_fn = "$tmpdir/index.json";
6606
6607         my $cmd = "restore";
6608
6609         my $param = [$pbs_backup_name, "index.json", $index_fn];
6610         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6611         my $index = PVE::Tools::file_get_contents($index_fn);
6612         $index = decode_json($index);
6613
6614         # print Dumper($index);
6615         foreach my $info (@{$index->{files}}) {
6616             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6617                 my $devname = $1;
6618                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6619                     $devinfo->{$devname}->{size} = $1;
6620                 } else {
6621                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6622                 }
6623             }
6624         }
6625
6626         my $is_qemu_server_backup = scalar(
6627             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6628         );
6629         if (!$is_qemu_server_backup) {
6630             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6631         }
6632         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6633
6634         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6635         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6636
6637         if ($has_firewall_config) {
6638             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6639             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6640
6641             my $pve_firewall_dir = '/etc/pve/firewall';
6642             mkdir $pve_firewall_dir; # make sure the dir exists
6643             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6644         }
6645
6646         my $fh = IO::File->new($cfgfn, "r") ||
6647             die "unable to read qemu-server.conf - $!\n";
6648
6649         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6650
6651         # fixme: rate limit?
6652
6653         # create empty/temp config
6654         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6655
6656         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6657
6658         # allocate volumes
6659         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6660
6661         foreach my $virtdev (sort keys %$virtdev_hash) {
6662             my $d = $virtdev_hash->{$virtdev};
6663             next if $d->{is_cloudinit}; # no need to restore cloudinit
6664
6665             # this fails if storage is unavailable
6666             my $volid = $d->{volid};
6667             my $path = PVE::Storage::path($storecfg, $volid);
6668
6669             # for live-restore we only want to preload the efidisk and TPM state
6670             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6671
6672             my $pbs_restore_cmd = [
6673                 '/usr/bin/pbs-restore',
6674                 '--repository', $repo,
6675                 $pbs_backup_name,
6676                 "$d->{devname}.img.fidx",
6677                 $path,
6678                 '--verbose',
6679                 ];
6680
6681             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6682             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6683
6684             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6685                 push @$pbs_restore_cmd, '--skip-zero';
6686             }
6687
6688             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6689             print "restore proxmox backup image: $dbg_cmdstring\n";
6690             run_command($pbs_restore_cmd);
6691         }
6692
6693         $fh->seek(0, 0) || die "seek failed - $!\n";
6694
6695         my $cookie = { netcount => 0 };
6696         while (defined(my $line = <$fh>)) {
6697             $new_conf_raw .= restore_update_config_line(
6698                 $cookie,
6699                 $map,
6700                 $line,
6701                 $options->{unique},
6702             );
6703         }
6704
6705         $fh->close();
6706     };
6707     my $err = $@;
6708
6709     if ($err || !$options->{live}) {
6710         $restore_deactivate_volumes->($storecfg, $devinfo);
6711     }
6712
6713     rmtree $tmpdir;
6714
6715     if ($err) {
6716         $restore_destroy_volumes->($storecfg, $devinfo);
6717         die $err;
6718     }
6719
6720     if ($options->{live}) {
6721         # keep lock during live-restore
6722         $new_conf_raw .= "\nlock: create";
6723     }
6724
6725     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6726
6727     PVE::Cluster::cfs_update(); # make sure we read new file
6728
6729     eval { rescan($vmid, 1); };
6730     warn $@ if $@;
6731
6732     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6733
6734     if ($options->{live}) {
6735         # enable interrupts
6736         local $SIG{INT} =
6737             local $SIG{TERM} =
6738             local $SIG{QUIT} =
6739             local $SIG{HUP} =
6740             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6741
6742         my $conf = PVE::QemuConfig->load_config($vmid);
6743         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6744
6745         # these special drives are already restored before start
6746         delete $devinfo->{'drive-efidisk0'};
6747         delete $devinfo->{'drive-tpmstate0-backup'};
6748         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6749
6750         PVE::QemuConfig->remove_lock($vmid, "create");
6751     }
6752 }
6753
6754 sub pbs_live_restore {
6755     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6756
6757     print "starting VM for live-restore\n";
6758     print "repository: '$repo', snapshot: '$snap'\n";
6759
6760     my $pbs_backing = {};
6761     for my $ds (keys %$restored_disks) {
6762         $ds =~ m/^drive-(.*)$/;
6763         my $confname = $1;
6764         $pbs_backing->{$confname} = {
6765             repository => $repo,
6766             snapshot => $snap,
6767             archive => "$ds.img.fidx",
6768         };
6769         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6770
6771         my $drive = parse_drive($confname, $conf->{$confname});
6772         print "restoring '$ds' to '$drive->{file}'\n";
6773     }
6774
6775     my $drives_streamed = 0;
6776     eval {
6777         # make sure HA doesn't interrupt our restore by stopping the VM
6778         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6779             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6780         }
6781
6782         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6783         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6784         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6785
6786         my $qmeventd_fd = register_qmeventd_handle($vmid);
6787
6788         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6789         # this will effectively collapse the backing image chain consisting of
6790         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6791         # removes itself once all backing images vanish with 'auto-remove=on')
6792         my $jobs = {};
6793         for my $ds (sort keys %$restored_disks) {
6794             my $job_id = "restore-$ds";
6795             mon_cmd($vmid, 'block-stream',
6796                 'job-id' => $job_id,
6797                 device => "$ds",
6798             );
6799             $jobs->{$job_id} = {};
6800         }
6801
6802         mon_cmd($vmid, 'cont');
6803         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6804
6805         print "restore-drive jobs finished successfully, removing all tracking block devices"
6806             ." to disconnect from Proxmox Backup Server\n";
6807
6808         for my $ds (sort keys %$restored_disks) {
6809             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6810         }
6811
6812         close($qmeventd_fd);
6813     };
6814
6815     my $err = $@;
6816
6817     if ($err) {
6818         warn "An error occured during live-restore: $err\n";
6819         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6820         die "live-restore failed\n";
6821     }
6822 }
6823
6824 sub restore_vma_archive {
6825     my ($archive, $vmid, $user, $opts, $comp) = @_;
6826
6827     my $readfrom = $archive;
6828
6829     my $cfg = PVE::Storage::config();
6830     my $commands = [];
6831     my $bwlimit = $opts->{bwlimit};
6832
6833     my $dbg_cmdstring = '';
6834     my $add_pipe = sub {
6835         my ($cmd) = @_;
6836         push @$commands, $cmd;
6837         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6838         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6839         $readfrom = '-';
6840     };
6841
6842     my $input = undef;
6843     if ($archive eq '-') {
6844         $input = '<&STDIN';
6845     } else {
6846         # If we use a backup from a PVE defined storage we also consider that
6847         # storage's rate limit:
6848         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6849         if (defined($volid)) {
6850             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6851             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6852             if ($readlimit) {
6853                 print STDERR "applying read rate limit: $readlimit\n";
6854                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6855                 $add_pipe->($cstream);
6856             }
6857         }
6858     }
6859
6860     if ($comp) {
6861         my $info = PVE::Storage::decompressor_info('vma', $comp);
6862         my $cmd = $info->{decompressor};
6863         push @$cmd, $readfrom;
6864         $add_pipe->($cmd);
6865     }
6866
6867     my $tmpdir = "/var/tmp/vzdumptmp$$";
6868     rmtree $tmpdir;
6869
6870     # disable interrupts (always do cleanups)
6871     local $SIG{INT} =
6872         local $SIG{TERM} =
6873         local $SIG{QUIT} =
6874         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6875
6876     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6877     POSIX::mkfifo($mapfifo, 0600);
6878     my $fifofh;
6879     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6880
6881     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6882
6883     my $oldtimeout;
6884     my $timeout = 5;
6885
6886     my $devinfo = {};
6887
6888     my $rpcenv = PVE::RPCEnvironment::get();
6889
6890     my $conffile = PVE::QemuConfig->config_file($vmid);
6891
6892     # Note: $oldconf is undef if VM does not exist
6893     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6894     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6895     my $new_conf_raw = '';
6896
6897     my %storage_limits;
6898
6899     my $print_devmap = sub {
6900         my $cfgfn = "$tmpdir/qemu-server.conf";
6901
6902         # we can read the config - that is already extracted
6903         my $fh = IO::File->new($cfgfn, "r") ||
6904             die "unable to read qemu-server.conf - $!\n";
6905
6906         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6907         if (-f $fwcfgfn) {
6908             my $pve_firewall_dir = '/etc/pve/firewall';
6909             mkdir $pve_firewall_dir; # make sure the dir exists
6910             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6911         }
6912
6913         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6914
6915         foreach my $info (values %{$virtdev_hash}) {
6916             my $storeid = $info->{storeid};
6917             next if defined($storage_limits{$storeid});
6918
6919             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6920             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6921             $storage_limits{$storeid} = $limit * 1024;
6922         }
6923
6924         foreach my $devname (keys %$devinfo) {
6925             die "found no device mapping information for device '$devname'\n"
6926                 if !$devinfo->{$devname}->{virtdev};
6927         }
6928
6929         # create empty/temp config
6930         if ($oldconf) {
6931             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6932             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6933         }
6934
6935         # allocate volumes
6936         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6937
6938         # print restore information to $fifofh
6939         foreach my $virtdev (sort keys %$virtdev_hash) {
6940             my $d = $virtdev_hash->{$virtdev};
6941             next if $d->{is_cloudinit}; # no need to restore cloudinit
6942
6943             my $storeid = $d->{storeid};
6944             my $volid = $d->{volid};
6945
6946             my $map_opts = '';
6947             if (my $limit = $storage_limits{$storeid}) {
6948                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6949             }
6950
6951             my $write_zeros = 1;
6952             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6953                 $write_zeros = 0;
6954             }
6955
6956             my $path = PVE::Storage::path($cfg, $volid);
6957
6958             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6959
6960             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6961         }
6962
6963         $fh->seek(0, 0) || die "seek failed - $!\n";
6964
6965         my $cookie = { netcount => 0 };
6966         while (defined(my $line = <$fh>)) {
6967             $new_conf_raw .= restore_update_config_line(
6968                 $cookie,
6969                 $map,
6970                 $line,
6971                 $opts->{unique},
6972             );
6973         }
6974
6975         $fh->close();
6976     };
6977
6978     eval {
6979         # enable interrupts
6980         local $SIG{INT} =
6981             local $SIG{TERM} =
6982             local $SIG{QUIT} =
6983             local $SIG{HUP} =
6984             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6985         local $SIG{ALRM} = sub { die "got timeout\n"; };
6986
6987         $oldtimeout = alarm($timeout);
6988
6989         my $parser = sub {
6990             my $line = shift;
6991
6992             print "$line\n";
6993
6994             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6995                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6996                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6997             } elsif ($line =~ m/^CTIME: /) {
6998                 # we correctly received the vma config, so we can disable
6999                 # the timeout now for disk allocation (set to 10 minutes, so
7000                 # that we always timeout if something goes wrong)
7001                 alarm(600);
7002                 &$print_devmap();
7003                 print $fifofh "done\n";
7004                 my $tmp = $oldtimeout || 0;
7005                 $oldtimeout = undef;
7006                 alarm($tmp);
7007                 close($fifofh);
7008                 $fifofh = undef;
7009             }
7010         };
7011
7012         print "restore vma archive: $dbg_cmdstring\n";
7013         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
7014     };
7015     my $err = $@;
7016
7017     alarm($oldtimeout) if $oldtimeout;
7018
7019     $restore_deactivate_volumes->($cfg, $devinfo);
7020
7021     close($fifofh) if $fifofh;
7022     unlink $mapfifo;
7023     rmtree $tmpdir;
7024
7025     if ($err) {
7026         $restore_destroy_volumes->($cfg, $devinfo);
7027         die $err;
7028     }
7029
7030     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7031
7032     PVE::Cluster::cfs_update(); # make sure we read new file
7033
7034     eval { rescan($vmid, 1); };
7035     warn $@ if $@;
7036
7037     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
7038 }
7039
7040 sub restore_tar_archive {
7041     my ($archive, $vmid, $user, $opts) = @_;
7042
7043     if ($archive ne '-') {
7044         my $firstfile = tar_archive_read_firstfile($archive);
7045         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
7046             if $firstfile ne 'qemu-server.conf';
7047     }
7048
7049     my $storecfg = PVE::Storage::config();
7050
7051     # avoid zombie disks when restoring over an existing VM -> cleanup first
7052     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
7053     # skiplock=1 because qmrestore has set the 'create' lock itself already
7054     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
7055     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
7056
7057     my $tocmd = "/usr/lib/qemu-server/qmextract";
7058
7059     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
7060     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
7061     $tocmd .= ' --prealloc' if $opts->{prealloc};
7062     $tocmd .= ' --info' if $opts->{info};
7063
7064     # tar option "xf" does not autodetect compression when read from STDIN,
7065     # so we pipe to zcat
7066     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
7067         PVE::Tools::shellquote("--to-command=$tocmd");
7068
7069     my $tmpdir = "/var/tmp/vzdumptmp$$";
7070     mkpath $tmpdir;
7071
7072     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
7073     local $ENV{VZDUMP_VMID} = $vmid;
7074     local $ENV{VZDUMP_USER} = $user;
7075
7076     my $conffile = PVE::QemuConfig->config_file($vmid);
7077     my $new_conf_raw = '';
7078
7079     # disable interrupts (always do cleanups)
7080     local $SIG{INT} =
7081         local $SIG{TERM} =
7082         local $SIG{QUIT} =
7083         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
7084
7085     eval {
7086         # enable interrupts
7087         local $SIG{INT} =
7088             local $SIG{TERM} =
7089             local $SIG{QUIT} =
7090             local $SIG{HUP} =
7091             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
7092
7093         if ($archive eq '-') {
7094             print "extracting archive from STDIN\n";
7095             run_command($cmd, input => "<&STDIN");
7096         } else {
7097             print "extracting archive '$archive'\n";
7098             run_command($cmd);
7099         }
7100
7101         return if $opts->{info};
7102
7103         # read new mapping
7104         my $map = {};
7105         my $statfile = "$tmpdir/qmrestore.stat";
7106         if (my $fd = IO::File->new($statfile, "r")) {
7107             while (defined (my $line = <$fd>)) {
7108                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
7109                     $map->{$1} = $2 if $1;
7110                 } else {
7111                     print STDERR "unable to parse line in statfile - $line\n";
7112                 }
7113             }
7114             $fd->close();
7115         }
7116
7117         my $confsrc = "$tmpdir/qemu-server.conf";
7118
7119         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7120
7121         my $cookie = { netcount => 0 };
7122         while (defined (my $line = <$srcfd>)) {
7123             $new_conf_raw .= restore_update_config_line(
7124                 $cookie,
7125                 $map,
7126                 $line,
7127                 $opts->{unique},
7128             );
7129         }
7130
7131         $srcfd->close();
7132     };
7133     if (my $err = $@) {
7134         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7135         die $err;
7136     }
7137
7138     rmtree $tmpdir;
7139
7140     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7141
7142     PVE::Cluster::cfs_update(); # make sure we read new file
7143
7144     eval { rescan($vmid, 1); };
7145     warn $@ if $@;
7146 };
7147
7148 sub foreach_storage_used_by_vm {
7149     my ($conf, $func) = @_;
7150
7151     my $sidhash = {};
7152
7153     PVE::QemuConfig->foreach_volume($conf, sub {
7154         my ($ds, $drive) = @_;
7155         return if drive_is_cdrom($drive);
7156
7157         my $volid = $drive->{file};
7158
7159         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7160         $sidhash->{$sid} = $sid if $sid;
7161     });
7162
7163     foreach my $sid (sort keys %$sidhash) {
7164         &$func($sid);
7165     }
7166 }
7167
7168 my $qemu_snap_storage = {
7169     rbd => 1,
7170 };
7171 sub do_snapshots_with_qemu {
7172     my ($storecfg, $volid, $deviceid) = @_;
7173
7174     return if $deviceid =~ m/tpmstate0/;
7175
7176     my $storage_name = PVE::Storage::parse_volume_id($volid);
7177     my $scfg = $storecfg->{ids}->{$storage_name};
7178     die "could not find storage '$storage_name'\n" if !defined($scfg);
7179
7180     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7181         return 1;
7182     }
7183
7184     if ($volid =~ m/\.(qcow2|qed)$/){
7185         return 1;
7186     }
7187
7188     return;
7189 }
7190
7191 sub qga_check_running {
7192     my ($vmid, $nowarn) = @_;
7193
7194     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7195     if ($@) {
7196         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7197         return 0;
7198     }
7199     return 1;
7200 }
7201
7202 sub template_create {
7203     my ($vmid, $conf, $disk) = @_;
7204
7205     my $storecfg = PVE::Storage::config();
7206
7207     PVE::QemuConfig->foreach_volume($conf, sub {
7208         my ($ds, $drive) = @_;
7209
7210         return if drive_is_cdrom($drive);
7211         return if $disk && $ds ne $disk;
7212
7213         my $volid = $drive->{file};
7214         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7215
7216         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7217         $drive->{file} = $voliddst;
7218         $conf->{$ds} = print_drive($drive);
7219         PVE::QemuConfig->write_config($vmid, $conf);
7220     });
7221 }
7222
7223 sub convert_iscsi_path {
7224     my ($path) = @_;
7225
7226     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7227         my $portal = $1;
7228         my $target = $2;
7229         my $lun = $3;
7230
7231         my $initiator_name = get_initiator_name();
7232
7233         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7234                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7235     }
7236
7237     die "cannot convert iscsi path '$path', unkown format\n";
7238 }
7239
7240 sub qemu_img_convert {
7241     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7242
7243     my $storecfg = PVE::Storage::config();
7244     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7245     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7246
7247     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7248
7249     my $cachemode;
7250     my $src_path;
7251     my $src_is_iscsi = 0;
7252     my $src_format;
7253
7254     if ($src_storeid) {
7255         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7256         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7257         $src_format = qemu_img_format($src_scfg, $src_volname);
7258         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7259         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7260         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7261     } elsif (-f $src_volid) {
7262         $src_path = $src_volid;
7263         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7264             $src_format = $1;
7265         }
7266     }
7267
7268     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7269
7270     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7271     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7272     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7273     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7274
7275     my $cmd = [];
7276     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7277     push @$cmd, '-l', "snapshot.name=$snapname"
7278         if $snapname && $src_format && $src_format eq "qcow2";
7279     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7280     push @$cmd, '-T', $cachemode if defined($cachemode);
7281
7282     if ($src_is_iscsi) {
7283         push @$cmd, '--image-opts';
7284         $src_path = convert_iscsi_path($src_path);
7285     } elsif ($src_format) {
7286         push @$cmd, '-f', $src_format;
7287     }
7288
7289     if ($dst_is_iscsi) {
7290         push @$cmd, '--target-image-opts';
7291         $dst_path = convert_iscsi_path($dst_path);
7292     } else {
7293         push @$cmd, '-O', $dst_format;
7294     }
7295
7296     push @$cmd, $src_path;
7297
7298     if (!$dst_is_iscsi && $is_zero_initialized) {
7299         push @$cmd, "zeroinit:$dst_path";
7300     } else {
7301         push @$cmd, $dst_path;
7302     }
7303
7304     my $parser = sub {
7305         my $line = shift;
7306         if($line =~ m/\((\S+)\/100\%\)/){
7307             my $percent = $1;
7308             my $transferred = int($size * $percent / 100);
7309             my $total_h = render_bytes($size, 1);
7310             my $transferred_h = render_bytes($transferred, 1);
7311
7312             print "transferred $transferred_h of $total_h ($percent%)\n";
7313         }
7314
7315     };
7316
7317     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7318     my $err = $@;
7319     die "copy failed: $err" if $err;
7320 }
7321
7322 sub qemu_img_format {
7323     my ($scfg, $volname) = @_;
7324
7325     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7326         return $1;
7327     } else {
7328         return "raw";
7329     }
7330 }
7331
7332 sub qemu_drive_mirror {
7333     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7334
7335     $jobs = {} if !$jobs;
7336
7337     my $qemu_target;
7338     my $format;
7339     $jobs->{"drive-$drive"} = {};
7340
7341     if ($dst_volid =~ /^nbd:/) {
7342         $qemu_target = $dst_volid;
7343         $format = "nbd";
7344     } else {
7345         my $storecfg = PVE::Storage::config();
7346         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7347
7348         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7349
7350         $format = qemu_img_format($dst_scfg, $dst_volname);
7351
7352         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7353
7354         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7355     }
7356
7357     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7358     $opts->{format} = $format if $format;
7359
7360     if (defined($src_bitmap)) {
7361         $opts->{sync} = 'incremental';
7362         $opts->{bitmap} = $src_bitmap;
7363         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7364     }
7365
7366     if (defined($bwlimit)) {
7367         $opts->{speed} = $bwlimit * 1024;
7368         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7369     } else {
7370         print "drive mirror is starting for drive-$drive\n";
7371     }
7372
7373     # if a job already runs for this device we get an error, catch it for cleanup
7374     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7375     if (my $err = $@) {
7376         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7377         warn "$@\n" if $@;
7378         die "mirroring error: $err\n";
7379     }
7380
7381     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7382 }
7383
7384 # $completion can be either
7385 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7386 # 'cancel': wait until all jobs are ready, block-job-cancel them
7387 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7388 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7389 sub qemu_drive_mirror_monitor {
7390     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7391
7392     $completion //= 'complete';
7393     $op //= "mirror";
7394
7395     eval {
7396         my $err_complete = 0;
7397
7398         my $starttime = time ();
7399         while (1) {
7400             die "block job ('$op') timed out\n" if $err_complete > 300;
7401
7402             my $stats = mon_cmd($vmid, "query-block-jobs");
7403             my $ctime = time();
7404
7405             my $running_jobs = {};
7406             for my $stat (@$stats) {
7407                 next if $stat->{type} ne $op;
7408                 $running_jobs->{$stat->{device}} = $stat;
7409             }
7410
7411             my $readycounter = 0;
7412
7413             for my $job_id (sort keys %$jobs) {
7414                 my $job = $running_jobs->{$job_id};
7415
7416                 my $vanished = !defined($job);
7417                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7418                 if($complete || ($vanished && $completion eq 'auto')) {
7419                     print "$job_id: $op-job finished\n";
7420                     delete $jobs->{$job_id};
7421                     next;
7422                 }
7423
7424                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7425
7426                 my $busy = $job->{busy};
7427                 my $ready = $job->{ready};
7428                 if (my $total = $job->{len}) {
7429                     my $transferred = $job->{offset} || 0;
7430                     my $remaining = $total - $transferred;
7431                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7432
7433                     my $duration = $ctime - $starttime;
7434                     my $total_h = render_bytes($total, 1);
7435                     my $transferred_h = render_bytes($transferred, 1);
7436
7437                     my $status = sprintf(
7438                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7439                         render_duration($duration),
7440                     );
7441
7442                     if ($ready) {
7443                         if ($busy) {
7444                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7445                         } else {
7446                             $status .= ", ready";
7447                         }
7448                     }
7449                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7450                     $jobs->{$job_id}->{ready} = $ready;
7451                 }
7452
7453                 $readycounter++ if $job->{ready};
7454             }
7455
7456             last if scalar(keys %$jobs) == 0;
7457
7458             if ($readycounter == scalar(keys %$jobs)) {
7459                 print "all '$op' jobs are ready\n";
7460
7461                 # do the complete later (or has already been done)
7462                 last if $completion eq 'skip' || $completion eq 'auto';
7463
7464                 if ($vmiddst && $vmiddst != $vmid) {
7465                     my $agent_running = $qga && qga_check_running($vmid);
7466                     if ($agent_running) {
7467                         print "freeze filesystem\n";
7468                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7469                         warn $@ if $@;
7470                     } else {
7471                         print "suspend vm\n";
7472                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7473                         warn $@ if $@;
7474                     }
7475
7476                     # if we clone a disk for a new target vm, we don't switch the disk
7477                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7478
7479                     if ($agent_running) {
7480                         print "unfreeze filesystem\n";
7481                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7482                         warn $@ if $@;
7483                     } else {
7484                         print "resume vm\n";
7485                         eval { PVE::QemuServer::vm_resume($vmid, 1, 1); };
7486                         warn $@ if $@;
7487                     }
7488
7489                     last;
7490                 } else {
7491
7492                     for my $job_id (sort keys %$jobs) {
7493                         # try to switch the disk if source and destination are on the same guest
7494                         print "$job_id: Completing block job_id...\n";
7495
7496                         my $op;
7497                         if ($completion eq 'complete') {
7498                             $op = 'block-job-complete';
7499                         } elsif ($completion eq 'cancel') {
7500                             $op = 'block-job-cancel';
7501                         } else {
7502                             die "invalid completion value: $completion\n";
7503                         }
7504                         eval { mon_cmd($vmid, $op, device => $job_id) };
7505                         if ($@ =~ m/cannot be completed/) {
7506                             print "$job_id: block job cannot be completed, trying again.\n";
7507                             $err_complete++;
7508                         }else {
7509                             print "$job_id: Completed successfully.\n";
7510                             $jobs->{$job_id}->{complete} = 1;
7511                         }
7512                     }
7513                 }
7514             }
7515             sleep 1;
7516         }
7517     };
7518     my $err = $@;
7519
7520     if ($err) {
7521         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7522         die "block job ($op) error: $err";
7523     }
7524 }
7525
7526 sub qemu_blockjobs_cancel {
7527     my ($vmid, $jobs) = @_;
7528
7529     foreach my $job (keys %$jobs) {
7530         print "$job: Cancelling block job\n";
7531         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7532         $jobs->{$job}->{cancel} = 1;
7533     }
7534
7535     while (1) {
7536         my $stats = mon_cmd($vmid, "query-block-jobs");
7537
7538         my $running_jobs = {};
7539         foreach my $stat (@$stats) {
7540             $running_jobs->{$stat->{device}} = $stat;
7541         }
7542
7543         foreach my $job (keys %$jobs) {
7544
7545             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7546                 print "$job: Done.\n";
7547                 delete $jobs->{$job};
7548             }
7549         }
7550
7551         last if scalar(keys %$jobs) == 0;
7552
7553         sleep 1;
7554     }
7555 }
7556
7557 sub clone_disk {
7558     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7559         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7560
7561     my $newvolid;
7562
7563     if (!$full) {
7564         print "create linked clone of drive $drivename ($drive->{file})\n";
7565         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7566         push @$newvollist, $newvolid;
7567     } else {
7568
7569         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7570         $storeid = $storage if $storage;
7571
7572         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7573
7574         print "create full clone of drive $drivename ($drive->{file})\n";
7575         my $name = undef;
7576         my $size = undef;
7577         if (drive_is_cloudinit($drive)) {
7578             $name = "vm-$newvmid-cloudinit";
7579             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7580             if ($scfg->{path}) {
7581                 $name .= ".$dst_format";
7582             }
7583             $snapname = undef;
7584             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7585         } elsif ($drivename eq 'efidisk0') {
7586             $size = get_efivars_size($conf);
7587         } elsif ($drivename eq 'tpmstate0') {
7588             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7589         } else {
7590             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7591         }
7592         $newvolid = PVE::Storage::vdisk_alloc(
7593             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7594         );
7595         push @$newvollist, $newvolid;
7596
7597         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7598
7599         if (drive_is_cloudinit($drive)) {
7600             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7601             # if this is the case, we have to complete any block-jobs still there from
7602             # previous drive-mirrors
7603             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7604                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7605             }
7606             goto no_data_clone;
7607         }
7608
7609         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7610         if (!$running || $snapname) {
7611             # TODO: handle bwlimits
7612             if ($drivename eq 'efidisk0') {
7613                 # the relevant data on the efidisk may be smaller than the source
7614                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7615                 # that is given by the OVMF_VARS.fd
7616                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7617                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7618
7619                 # better for Ceph if block size is not too small, see bug #3324
7620                 my $bs = 1024*1024;
7621
7622                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7623                     "if=$src_path", "of=$dst_path"]);
7624             } else {
7625                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7626             }
7627         } else {
7628
7629             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7630
7631             my $kvmver = get_running_qemu_version ($vmid);
7632             if (!min_version($kvmver, 2, 7)) {
7633                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7634                     if $drive->{iothread};
7635             }
7636
7637             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7638                 $completion, $qga, $bwlimit);
7639         }
7640     }
7641
7642 no_data_clone:
7643     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7644
7645     my $disk = $drive;
7646     $disk->{format} = undef;
7647     $disk->{file} = $newvolid;
7648     $disk->{size} = $size if defined($size);
7649
7650     return $disk;
7651 }
7652
7653 sub get_running_qemu_version {
7654     my ($vmid) = @_;
7655     my $res = mon_cmd($vmid, "query-version");
7656     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7657 }
7658
7659 sub qemu_use_old_bios_files {
7660     my ($machine_type) = @_;
7661
7662     return if !$machine_type;
7663
7664     my $use_old_bios_files = undef;
7665
7666     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7667         $machine_type = $1;
7668         $use_old_bios_files = 1;
7669     } else {
7670         my $version = extract_version($machine_type, kvm_user_version());
7671         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7672         # load new efi bios files on migration. So this hack is required to allow
7673         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7674         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7675         $use_old_bios_files = !min_version($version, 2, 4);
7676     }
7677
7678     return ($use_old_bios_files, $machine_type);
7679 }
7680
7681 sub get_efivars_size {
7682     my ($conf) = @_;
7683     my $arch = get_vm_arch($conf);
7684     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7685     my $smm = PVE::QemuServer::Machine::machine_type_is_q35($conf);
7686     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7687     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7688     return -s $ovmf_vars;
7689 }
7690
7691 sub update_efidisk_size {
7692     my ($conf) = @_;
7693
7694     return if !defined($conf->{efidisk0});
7695
7696     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7697     $disk->{size} = get_efivars_size($conf);
7698     $conf->{efidisk0} = print_drive($disk);
7699
7700     return;
7701 }
7702
7703 sub update_tpmstate_size {
7704     my ($conf) = @_;
7705
7706     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7707     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7708     $conf->{tpmstate0} = print_drive($disk);
7709 }
7710
7711 sub create_efidisk($$$$$$$) {
7712     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk, $smm) = @_;
7713
7714     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk, $smm);
7715     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7716
7717     my $vars_size_b = -s $ovmf_vars;
7718     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7719     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7720     PVE::Storage::activate_volumes($storecfg, [$volid]);
7721
7722     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7723     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7724
7725     return ($volid, $size/1024);
7726 }
7727
7728 sub vm_iothreads_list {
7729     my ($vmid) = @_;
7730
7731     my $res = mon_cmd($vmid, 'query-iothreads');
7732
7733     my $iothreads = {};
7734     foreach my $iothread (@$res) {
7735         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7736     }
7737
7738     return $iothreads;
7739 }
7740
7741 sub scsihw_infos {
7742     my ($conf, $drive) = @_;
7743
7744     my $maxdev = 0;
7745
7746     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7747         $maxdev = 7;
7748     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7749         $maxdev = 1;
7750     } else {
7751         $maxdev = 256;
7752     }
7753
7754     my $controller = int($drive->{index} / $maxdev);
7755     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7756         ? "virtioscsi"
7757         : "scsihw";
7758
7759     return ($maxdev, $controller, $controller_prefix);
7760 }
7761
7762 sub windows_version {
7763     my ($ostype) = @_;
7764
7765     return 0 if !$ostype;
7766
7767     my $winversion = 0;
7768
7769     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7770         $winversion = 5;
7771     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7772         $winversion = 6;
7773     } elsif ($ostype =~ m/^win(\d+)$/) {
7774         $winversion = $1;
7775     }
7776
7777     return $winversion;
7778 }
7779
7780 sub resolve_dst_disk_format {
7781         my ($storecfg, $storeid, $src_volname, $format) = @_;
7782         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7783
7784         if (!$format) {
7785             # if no target format is specified, use the source disk format as hint
7786             if ($src_volname) {
7787                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7788                 $format = qemu_img_format($scfg, $src_volname);
7789             } else {
7790                 return $defFormat;
7791             }
7792         }
7793
7794         # test if requested format is supported - else use default
7795         my $supported = grep { $_ eq $format } @$validFormats;
7796         $format = $defFormat if !$supported;
7797         return $format;
7798 }
7799
7800 # NOTE: if this logic changes, please update docs & possibly gui logic
7801 sub find_vmstate_storage {
7802     my ($conf, $storecfg) = @_;
7803
7804     # first, return storage from conf if set
7805     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7806
7807     my ($target, $shared, $local);
7808
7809     foreach_storage_used_by_vm($conf, sub {
7810         my ($sid) = @_;
7811         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7812         my $dst = $scfg->{shared} ? \$shared : \$local;
7813         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7814     });
7815
7816     # second, use shared storage where VM has at least one disk
7817     # third, use local storage where VM has at least one disk
7818     # fall back to local storage
7819     $target = $shared // $local // 'local';
7820
7821     return $target;
7822 }
7823
7824 sub generate_uuid {
7825     my ($uuid, $uuid_str);
7826     UUID::generate($uuid);
7827     UUID::unparse($uuid, $uuid_str);
7828     return $uuid_str;
7829 }
7830
7831 sub generate_smbios1_uuid {
7832     return "uuid=".generate_uuid();
7833 }
7834
7835 sub nbd_stop {
7836     my ($vmid) = @_;
7837
7838     mon_cmd($vmid, 'nbd-server-stop');
7839 }
7840
7841 sub create_reboot_request {
7842     my ($vmid) = @_;
7843     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7844         or die "failed to create reboot trigger file: $!\n";
7845     close($fh);
7846 }
7847
7848 sub clear_reboot_request {
7849     my ($vmid) = @_;
7850     my $path = "/run/qemu-server/$vmid.reboot";
7851     my $res = 0;
7852
7853     $res = unlink($path);
7854     die "could not remove reboot request for $vmid: $!"
7855         if !$res && $! != POSIX::ENOENT;
7856
7857     return $res;
7858 }
7859
7860 sub bootorder_from_legacy {
7861     my ($conf, $bootcfg) = @_;
7862
7863     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7864     my $bootindex_hash = {};
7865     my $i = 1;
7866     foreach my $o (split(//, $boot)) {
7867         $bootindex_hash->{$o} = $i*100;
7868         $i++;
7869     }
7870
7871     my $bootorder = {};
7872
7873     PVE::QemuConfig->foreach_volume($conf, sub {
7874         my ($ds, $drive) = @_;
7875
7876         if (drive_is_cdrom ($drive, 1)) {
7877             if ($bootindex_hash->{d}) {
7878                 $bootorder->{$ds} = $bootindex_hash->{d};
7879                 $bootindex_hash->{d} += 1;
7880             }
7881         } elsif ($bootindex_hash->{c}) {
7882             $bootorder->{$ds} = $bootindex_hash->{c}
7883                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7884             $bootindex_hash->{c} += 1;
7885         }
7886     });
7887
7888     if ($bootindex_hash->{n}) {
7889         for (my $i = 0; $i < $MAX_NETS; $i++) {
7890             my $netname = "net$i";
7891             next if !$conf->{$netname};
7892             $bootorder->{$netname} = $bootindex_hash->{n};
7893             $bootindex_hash->{n} += 1;
7894         }
7895     }
7896
7897     return $bootorder;
7898 }
7899
7900 # Generate default device list for 'boot: order=' property. Matches legacy
7901 # default boot order, but with explicit device names. This is important, since
7902 # the fallback for when neither 'order' nor the old format is specified relies
7903 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7904 sub get_default_bootdevices {
7905     my ($conf) = @_;
7906
7907     my @ret = ();
7908
7909     # harddisk
7910     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7911     push @ret, $first if $first;
7912
7913     # cdrom
7914     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7915     push @ret, $first if $first;
7916
7917     # network
7918     for (my $i = 0; $i < $MAX_NETS; $i++) {
7919         my $netname = "net$i";
7920         next if !$conf->{$netname};
7921         push @ret, $netname;
7922         last;
7923     }
7924
7925     return \@ret;
7926 }
7927
7928 sub device_bootorder {
7929     my ($conf) = @_;
7930
7931     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7932
7933     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7934
7935     my $bootorder = {};
7936     if (!defined($boot) || $boot->{legacy}) {
7937         $bootorder = bootorder_from_legacy($conf, $boot);
7938     } elsif ($boot->{order}) {
7939         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7940         for my $dev (PVE::Tools::split_list($boot->{order})) {
7941             $bootorder->{$dev} = $i++;
7942         }
7943     }
7944
7945     return $bootorder;
7946 }
7947
7948 sub register_qmeventd_handle {
7949     my ($vmid) = @_;
7950
7951     my $fh;
7952     my $peer = "/var/run/qmeventd.sock";
7953     my $count = 0;
7954
7955     for (;;) {
7956         $count++;
7957         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7958         last if $fh;
7959         if ($! != EINTR && $! != EAGAIN) {
7960             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7961         }
7962         if ($count > 4) {
7963             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7964               . "after $count retries\n";
7965         }
7966         usleep(25000);
7967     }
7968
7969     # send handshake to mark VM as backing up
7970     print $fh to_json({vzdump => {vmid => "$vmid"}});
7971
7972     # return handle to be closed later when inhibit is no longer required
7973     return $fh;
7974 }
7975
7976 # bash completion helper
7977
7978 sub complete_backup_archives {
7979     my ($cmdname, $pname, $cvalue) = @_;
7980
7981     my $cfg = PVE::Storage::config();
7982
7983     my $storeid;
7984
7985     if ($cvalue =~ m/^([^:]+):/) {
7986         $storeid = $1;
7987     }
7988
7989     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7990
7991     my $res = [];
7992     foreach my $id (keys %$data) {
7993         foreach my $item (@{$data->{$id}}) {
7994             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7995             push @$res, $item->{volid} if defined($item->{volid});
7996         }
7997     }
7998
7999     return $res;
8000 }
8001
8002 my $complete_vmid_full = sub {
8003     my ($running) = @_;
8004
8005     my $idlist = vmstatus();
8006
8007     my $res = [];
8008
8009     foreach my $id (keys %$idlist) {
8010         my $d = $idlist->{$id};
8011         if (defined($running)) {
8012             next if $d->{template};
8013             next if $running && $d->{status} ne 'running';
8014             next if !$running && $d->{status} eq 'running';
8015         }
8016         push @$res, $id;
8017
8018     }
8019     return $res;
8020 };
8021
8022 sub complete_vmid {
8023     return &$complete_vmid_full();
8024 }
8025
8026 sub complete_vmid_stopped {
8027     return &$complete_vmid_full(0);
8028 }
8029
8030 sub complete_vmid_running {
8031     return &$complete_vmid_full(1);
8032 }
8033
8034 sub complete_storage {
8035
8036     my $cfg = PVE::Storage::config();
8037     my $ids = $cfg->{ids};
8038
8039     my $res = [];
8040     foreach my $sid (keys %$ids) {
8041         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
8042         next if !$ids->{$sid}->{content}->{images};
8043         push @$res, $sid;
8044     }
8045
8046     return $res;
8047 }
8048
8049 sub complete_migration_storage {
8050     my ($cmd, $param, $current_value, $all_args) = @_;
8051
8052     my $targetnode = @$all_args[1];
8053
8054     my $cfg = PVE::Storage::config();
8055     my $ids = $cfg->{ids};
8056
8057     my $res = [];
8058     foreach my $sid (keys %$ids) {
8059         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
8060         next if !$ids->{$sid}->{content}->{images};
8061         push @$res, $sid;
8062     }
8063
8064     return $res;
8065 }
8066
8067 sub vm_is_paused {
8068     my ($vmid) = @_;
8069     my $qmpstatus = eval {
8070         PVE::QemuConfig::assert_config_exists_on_node($vmid);
8071         mon_cmd($vmid, "query-status");
8072     };
8073     warn "$@\n" if $@;
8074     return $qmpstatus && $qmpstatus->{status} eq "paused";
8075 }
8076
8077 sub check_volume_storage_type {
8078     my ($storecfg, $vol) = @_;
8079
8080     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
8081     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
8082     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
8083
8084     die "storage '$storeid' does not support content-type '$vtype'\n"
8085         if !$scfg->{content}->{$vtype};
8086
8087     return 1;
8088 }
8089
8090 1;