PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         default => [
  76             "$EDK2_FW_BASE/OVMF_CODE.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS.fd",
  78         ],
  79     },
  80     aarch64 => {
  81         default => [
  82             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  83             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  84         ],
  85     },
  86 };
  87
  88 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  89
  90 # Note about locking: we use flock on the config file protect
  91 # against concurent actions.
  92 # Aditionaly, we have a 'lock' setting in the config file. This
  93 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  94 # allowed when such lock is set. But you can ignore this kind of
  95 # lock with the --skiplock flag.
  96
  97 cfs_register_file('/qemu-server/',
  98                   \&parse_vm_config,
  99                   \&write_vm_config);
 100
 101 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 102     description => "Some command save/restore state from this location.",
 103     type => 'string',
 104     maxLength => 128,
 105     optional => 1,
 106 });
 107
 108 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 109         description => "Specifies the Qemu machine type.",
 110         type => 'string',
 111         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 112         maxLength => 40,
 113         optional => 1,
 114 });
 115
 116
 117 sub map_storage {
 118     my ($map, $source) = @_;
 119
 120     return $source if !defined($map);
 121
 122     return $map->{entries}->{$source}
 123         if $map->{entries} && defined($map->{entries}->{$source});
 124
 125     return $map->{default} if $map->{default};
 126
 127     # identity (fallback)
 128     return $source;
 129 }
 130
 131 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 132     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 133     type => 'string',
 134     format => 'storagepair-list',
 135     optional => 1,
 136 });
 137
 138 #no warnings 'redefine';
 139
 140 my $nodename_cache;
 141 sub nodename {
 142     $nodename_cache //= PVE::INotify::nodename();
 143     return $nodename_cache;
 144 }
 145
 146 my $watchdog_fmt = {
 147     model => {
 148         default_key => 1,
 149         type => 'string',
 150         enum => [qw(i6300esb ib700)],
 151         description => "Watchdog type to emulate.",
 152         default => 'i6300esb',
 153         optional => 1,
 154     },
 155     action => {
 156         type => 'string',
 157         enum => [qw(reset shutdown poweroff pause debug none)],
 158         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 159         optional => 1,
 160     },
 161 };
 162 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 163
 164 my $agent_fmt = {
 165     enabled => {
 166         description => "Enable/disable Qemu GuestAgent.",
 167         type => 'boolean',
 168         default => 0,
 169         default_key => 1,
 170     },
 171     fstrim_cloned_disks => {
 172         description => "Run fstrim after moving a disk or migrating the VM.",
 173         type => 'boolean',
 174         optional => 1,
 175         default => 0
 176     },
 177     type => {
 178         description => "Select the agent type",
 179         type => 'string',
 180         default => 'virtio',
 181         optional => 1,
 182         enum => [qw(virtio isa)],
 183     },
 184 };
 185
 186 my $vga_fmt = {
 187     type => {
 188         description => "Select the VGA type.",
 189         type => 'string',
 190         default => 'std',
 191         optional => 1,
 192         default_key => 1,
 193         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 194     },
 195     memory => {
 196         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 197         type => 'integer',
 198         optional => 1,
 199         minimum => 4,
 200         maximum => 512,
 201     },
 202 };
 203
 204 my $ivshmem_fmt = {
 205     size => {
 206         type => 'integer',
 207         minimum => 1,
 208         description => "The size of the file in MB.",
 209     },
 210     name => {
 211         type => 'string',
 212         pattern => '[a-zA-Z0-9\-]+',
 213         optional => 1,
 214         format_description => 'string',
 215         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 216     },
 217 };
 218
 219 my $audio_fmt = {
 220     device => {
 221         type => 'string',
 222         enum => [qw(ich9-intel-hda intel-hda AC97)],
 223         description =>  "Configure an audio device."
 224     },
 225     driver =>  {
 226         type => 'string',
 227         enum => ['spice', 'none'],
 228         default => 'spice',
 229         optional => 1,
 230         description => "Driver backend for the audio device."
 231     },
 232 };
 233
 234 my $spice_enhancements_fmt = {
 235     foldersharing => {
 236         type => 'boolean',
 237         optional => 1,
 238         default => '0',
 239         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 240     },
 241     videostreaming =>  {
 242         type => 'string',
 243         enum => ['off', 'all', 'filter'],
 244         default => 'off',
 245         optional => 1,
 246         description => "Enable video streaming. Uses compression for detected video streams."
 247     },
 248 };
 249
 250 my $rng_fmt = {
 251     source => {
 252         type => 'string',
 253         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 254         default_key => 1,
 255         description => "The file on the host to gather entropy from. In most"
 256                      . " cases /dev/urandom should be preferred over /dev/random"
 257                      . " to avoid entropy-starvation issues on the host. Using"
 258                      . " urandom does *not* decrease security in any meaningful"
 259                      . " way, as it's still seeded from real entropy, and the"
 260                      . " bytes provided will most likely be mixed with real"
 261                      . " entropy on the guest as well. /dev/hwrng can be used"
 262                      . " to pass through a hardware RNG from the host.",
 263     },
 264     max_bytes => {
 265         type => 'integer',
 266         description => "Maximum bytes of entropy injected into the guest every"
 267                      . " 'period' milliseconds. Prefer a lower value when using"
 268                      . " /dev/random as source. Use 0 to disable limiting"
 269                      . " (potentially dangerous!).",
 270         optional => 1,
 271
 272         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 273         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 274         # of overwhelming the host, provided we're reading from /dev/urandom
 275         default => 1024,
 276     },
 277     period => {
 278         type => 'integer',
 279         description => "Every 'period' milliseconds the entropy-injection quota"
 280                      . " is reset, allowing the guest to retrieve another"
 281                      . " 'max_bytes' of entropy.",
 282         optional => 1,
 283         default => 1000,
 284     },
 285 };
 286
 287 my $confdesc = {
 288     onboot => {
 289         optional => 1,
 290         type => 'boolean',
 291         description => "Specifies whether a VM will be started during system bootup.",
 292         default => 0,
 293     },
 294     autostart => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Automatic restart after crash (currently ignored).",
 298         default => 0,
 299     },
 300     hotplug => {
 301         optional => 1,
 302         type => 'string', format => 'pve-hotplug-features',
 303         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 304         default => 'network,disk,usb',
 305     },
 306     reboot => {
 307         optional => 1,
 308         type => 'boolean',
 309         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 310         default => 1,
 311     },
 312     lock => {
 313         optional => 1,
 314         type => 'string',
 315         description => "Lock/unlock the VM.",
 316         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 317     },
 318     cpulimit => {
 319         optional => 1,
 320         type => 'number',
 321         description => "Limit of CPU usage.",
 322         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 323         minimum => 0,
 324         maximum => 128,
 325         default => 0,
 326     },
 327     cpuunits => {
 328         optional => 1,
 329         type => 'integer',
 330         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 331         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 332             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 333             ." weights of all the other running VMs.",
 334         minimum => 2,
 335         maximum => 262144,
 336         default => 'cgroup v1: 1024, cgroup v2: 100',
 337     },
 338     memory => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 342         minimum => 16,
 343         default => 512,
 344     },
 345     balloon => {
 346         optional => 1,
 347         type => 'integer',
 348         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 349         minimum => 0,
 350     },
 351     shares => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 355         minimum => 0,
 356         maximum => 50000,
 357         default => 1000,
 358     },
 359     keyboard => {
 360         optional => 1,
 361         type => 'string',
 362         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 363                        "It should not be necessary to set it.",
 364         enum => PVE::Tools::kvmkeymaplist(),
 365         default => undef,
 366     },
 367     name => {
 368         optional => 1,
 369         type => 'string', format => 'dns-name',
 370         description => "Set a name for the VM. Only used on the configuration web interface.",
 371     },
 372     scsihw => {
 373         optional => 1,
 374         type => 'string',
 375         description => "SCSI controller model",
 376         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 377         default => 'lsi',
 378     },
 379     description => {
 380         optional => 1,
 381         type => 'string',
 382         description => "Description for the VM. Shown in the web-interface VM's summary."
 383             ." This is saved as comment inside the configuration file.",
 384         maxLength => 1024 * 8,
 385     },
 386     ostype => {
 387         optional => 1,
 388         type => 'string',
 389         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 390         description => "Specify guest operating system.",
 391         verbose_description => <<EODESC,
 392 Specify guest operating system. This is used to enable special
 393 optimization/features for specific operating systems:
 394
 395 [horizontal]
 396 other;; unspecified OS
 397 wxp;; Microsoft Windows XP
 398 w2k;; Microsoft Windows 2000
 399 w2k3;; Microsoft Windows 2003
 400 w2k8;; Microsoft Windows 2008
 401 wvista;; Microsoft Windows Vista
 402 win7;; Microsoft Windows 7
 403 win8;; Microsoft Windows 8/2012/2012r2
 404 win10;; Microsoft Windows 10/2016/2019
 405 l24;; Linux 2.4 Kernel
 406 l26;; Linux 2.6 - 5.X Kernel
 407 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 408 EODESC
 409     },
 410     boot => {
 411         optional => 1,
 412         type => 'string', format => 'pve-qm-boot',
 413         description => "Specify guest boot order. Use with 'order=', usage with"
 414                      . " no key or 'legacy=' is deprecated.",
 415     },
 416     bootdisk => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-bootdisk',
 419         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 420         pattern => '(ide|sata|scsi|virtio)\d+',
 421     },
 422     smp => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of CPUs. Please use option -sockets instead.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     sockets => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPU sockets.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     cores => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of cores per socket.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     numa => {
 444         optional => 1,
 445         type => 'boolean',
 446         description => "Enable/disable NUMA.",
 447         default => 0,
 448     },
 449     hugepages => {
 450         optional => 1,
 451         type => 'string',
 452         description => "Enable/disable hugepages memory.",
 453         enum => [qw(any 2 1024)],
 454     },
 455     keephugepages => {
 456         optional => 1,
 457         type => 'boolean',
 458         default => 0,
 459         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 460             ." after VM shutdown and can be used for subsequent starts.",
 461     },
 462     vcpus => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "Number of hotplugged vcpus.",
 466         minimum => 1,
 467         default => 0,
 468     },
 469     acpi => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable ACPI.",
 473         default => 1,
 474     },
 475     agent => {
 476         optional => 1,
 477         description => "Enable/disable Qemu GuestAgent and its properties.",
 478         type => 'string',
 479         format => $agent_fmt,
 480     },
 481     kvm => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Enable/disable KVM hardware virtualization.",
 485         default => 1,
 486     },
 487     tdf => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Enable/disable time drift fix.",
 491         default => 0,
 492     },
 493     localtime => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Set the real time clock to local time. This is enabled by default if ostype"
 497             ." indicates a Microsoft OS.",
 498     },
 499     freeze => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 503     },
 504     vga => {
 505         optional => 1,
 506         type => 'string', format => $vga_fmt,
 507         description => "Configure the VGA hardware.",
 508         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 509             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 510             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 511             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 512             ." display server. For win* OS you can select how many independent displays you want,"
 513             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 514             ." using a serial device as terminal.",
 515     },
 516     watchdog => {
 517         optional => 1,
 518         type => 'string', format => 'pve-qm-watchdog',
 519         description => "Create a virtual hardware watchdog device.",
 520         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 521             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 522             ." else the watchdog will reset the guest (or execute the respective action specified)",
 523     },
 524     startdate => {
 525         optional => 1,
 526         type => 'string',
 527         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 528         description => "Set the initial date of the real time clock. Valid format for date are:"
 529             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 530         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 531         default => 'now',
 532     },
 533     startup =>  get_standard_option('pve-startup-order'),
 534     template => {
 535         optional => 1,
 536         type => 'boolean',
 537         description => "Enable/disable Template.",
 538         default => 0,
 539     },
 540     args => {
 541         optional => 1,
 542         type => 'string',
 543         description => "Arbitrary arguments passed to kvm.",
 544         verbose_description => <<EODESCR,
 545 Arbitrary arguments passed to kvm, for example:
 546
 547 args: -no-reboot -no-hpet
 548
 549 NOTE: this option is for experts only.
 550 EODESCR
 551     },
 552     tablet => {
 553         optional => 1,
 554         type => 'boolean',
 555         default => 1,
 556         description => "Enable/disable the USB tablet device.",
 557         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 558             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 559             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 560             ." may consider disabling this to save some context switches. This is turned off by"
 561             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 562     },
 563     migrate_speed => {
 564         optional => 1,
 565         type => 'integer',
 566         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 567         minimum => 0,
 568         default => 0,
 569     },
 570     migrate_downtime => {
 571         optional => 1,
 572         type => 'number',
 573         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 574         minimum => 0,
 575         default => 0.1,
 576     },
 577     cdrom => {
 578         optional => 1,
 579         type => 'string', format => 'pve-qm-ide',
 580         typetext => '<volume>',
 581         description => "This is an alias for option -ide2",
 582     },
 583     cpu => {
 584         optional => 1,
 585         description => "Emulated CPU type.",
 586         type => 'string',
 587         format => 'pve-vm-cpu-conf',
 588     },
 589     parent => get_standard_option('pve-snapshot-name', {
 590         optional => 1,
 591         description => "Parent snapshot name. This is used internally, and should not be modified.",
 592     }),
 593     snaptime => {
 594         optional => 1,
 595         description => "Timestamp for snapshots.",
 596         type => 'integer',
 597         minimum => 0,
 598     },
 599     vmstate => {
 600         optional => 1,
 601         type => 'string', format => 'pve-volume-id',
 602         description => "Reference to a volume which stores the VM state. This is used internally"
 603             ." for snapshots.",
 604     },
 605     vmstatestorage => get_standard_option('pve-storage-id', {
 606         description => "Default storage for VM state volumes/files.",
 607         optional => 1,
 608     }),
 609     runningmachine => get_standard_option('pve-qemu-machine', {
 610         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 611             ." for snapshots.",
 612     }),
 613     runningcpu => {
 614         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 615             ." internally for snapshots.",
 616         optional => 1,
 617         type => 'string',
 618         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 619         format_description => 'QEMU -cpu parameter'
 620     },
 621     machine => get_standard_option('pve-qemu-machine'),
 622     arch => {
 623         description => "Virtual processor architecture. Defaults to the host.",
 624         optional => 1,
 625         type => 'string',
 626         enum => [qw(x86_64 aarch64)],
 627     },
 628     smbios1 => {
 629         description => "Specify SMBIOS type 1 fields.",
 630         type => 'string', format => 'pve-qm-smbios1',
 631         maxLength => 512,
 632         optional => 1,
 633     },
 634     protection => {
 635         optional => 1,
 636         type => 'boolean',
 637         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 638             ." remove disk operations.",
 639         default => 0,
 640     },
 641     bios => {
 642         optional => 1,
 643         type => 'string',
 644         enum => [ qw(seabios ovmf) ],
 645         description => "Select BIOS implementation.",
 646         default => 'seabios',
 647     },
 648     vmgenid => {
 649         type => 'string',
 650         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 651         format_description => 'UUID',
 652         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 653             ." to disable explicitly.",
 654         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 655             ." value identifier to the guest OS. This allows to notify the guest operating system"
 656             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 657             ." execution or creation from a template). The guest operating system notices the"
 658             ." change, and is then able to react as appropriate by marking its copies of"
 659             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 660             ."Note that auto-creation only works when done through API/CLI create or update methods"
 661             .", but not when manually editing the config file.",
 662         default => "1 (autogenerated)",
 663         optional => 1,
 664     },
 665     hookscript => {
 666         type => 'string',
 667         format => 'pve-volume-id',
 668         optional => 1,
 669         description => "Script that will be executed during various steps in the vms lifetime.",
 670     },
 671     ivshmem => {
 672         type => 'string',
 673         format => $ivshmem_fmt,
 674         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 675             ." the host.",
 676         optional => 1,
 677     },
 678     audio0 => {
 679         type => 'string',
 680         format => $audio_fmt,
 681         description => "Configure a audio device, useful in combination with QXL/Spice.",
 682         optional => 1
 683     },
 684     spice_enhancements => {
 685         type => 'string',
 686         format => $spice_enhancements_fmt,
 687         description => "Configure additional enhancements for SPICE.",
 688         optional => 1
 689     },
 690     tags => {
 691         type => 'string', format => 'pve-tag-list',
 692         description => 'Tags of the VM. This is only meta information.',
 693         optional => 1,
 694     },
 695     rng0 => {
 696         type => 'string',
 697         format => $rng_fmt,
 698         description => "Configure a VirtIO-based Random Number Generator.",
 699         optional => 1,
 700     },
 701 };
 702
 703 my $cicustom_fmt = {
 704     meta => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all meta data passed to the VM via"
 708             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712     network => {
 713         type => 'string',
 714         optional => 1,
 715         description => 'Specify a custom file containing all network data passed to the VM via'
 716             .' cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via'
 724             .' cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728 };
 729 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 730
 731 my $confdesc_cloudinit = {
 732     citype => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'Specifies the cloud-init configuration format. The default depends on the'
 736             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 737             .' and `configdrive2` for windows.',
 738         enum => ['configdrive2', 'nocloud', 'opennebula'],
 739     },
 740     ciuser => {
 741         optional => 1,
 742         type => 'string',
 743         description => "cloud-init: User name to change ssh keys and password for instead of the"
 744             ." image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not'
 750             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 751             .' support hashed passwords.',
 752     },
 753     cicustom => {
 754         optional => 1,
 755         type => 'string',
 756         description => 'cloud-init: Specify custom files to replace the automatically generated'
 757             .' ones at start.',
 758         format => 'pve-qm-cicustom',
 759     },
 760     searchdomain => {
 761         optional => 1,
 762         type => 'string',
 763         description => "cloud-init: Sets DNS search domains for a container. Create will'
 764             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 765             .' are set.",
 766     },
 767     nameserver => {
 768         optional => 1,
 769         type => 'string', format => 'address-list',
 770         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 771             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 772             .' are set.",
 773     },
 774     sshkeys => {
 775         optional => 1,
 776         type => 'string',
 777         format => 'urlencoded',
 778         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 779     },
 780 };
 781
 782 # what about other qemu settings ?
 783 #cpu => 'string',
 784 #machine => 'string',
 785 #fda => 'file',
 786 #fdb => 'file',
 787 #mtdblock => 'file',
 788 #sd => 'file',
 789 #pflash => 'file',
 790 #snapshot => 'bool',
 791 #bootp => 'file',
 792 ##tftp => 'dir',
 793 ##smb => 'dir',
 794 #kernel => 'file',
 795 #append => 'string',
 796 #initrd => 'file',
 797 ##soundhw => 'string',
 798
 799 while (my ($k, $v) = each %$confdesc) {
 800     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 801 }
 802
 803 my $MAX_USB_DEVICES = 5;
 804 my $MAX_NETS = 32;
 805 my $MAX_SERIAL_PORTS = 4;
 806 my $MAX_PARALLEL_PORTS = 3;
 807 my $MAX_NUMA = 8;
 808
 809 my $numa_fmt = {
 810     cpus => {
 811         type => "string",
 812         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 813         description => "CPUs accessing this NUMA node.",
 814         format_description => "id[-id];...",
 815     },
 816     memory => {
 817         type => "number",
 818         description => "Amount of memory this NUMA node provides.",
 819         optional => 1,
 820     },
 821     hostnodes => {
 822         type => "string",
 823         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 824         description => "Host NUMA nodes to use.",
 825         format_description => "id[-id];...",
 826         optional => 1,
 827     },
 828     policy => {
 829         type => 'string',
 830         enum => [qw(preferred bind interleave)],
 831         description => "NUMA allocation policy.",
 832         optional => 1,
 833     },
 834 };
 835 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 836 my $numadesc = {
 837     optional => 1,
 838     type => 'string', format => $numa_fmt,
 839     description => "NUMA topology.",
 840 };
 841 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 842
 843 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 844     $confdesc->{"numa$i"} = $numadesc;
 845 }
 846
 847 my $nic_model_list = [
 848     'e1000',
 849     'e1000-82540em',
 850     'e1000-82544gc',
 851     'e1000-82545em',
 852     'e1000e',
 853     'i82551',
 854     'i82557b',
 855     'i82559er',
 856     'ne2k_isa',
 857     'ne2k_pci',
 858     'pcnet',
 859     'rtl8139',
 860     'virtio',
 861     'vmxnet3',
 862 ];
 863 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 864
 865 my $net_fmt_bridge_descr = <<__EOD__;
 866 Bridge to attach the network device to. The Proxmox VE standard bridge
 867 is called 'vmbr0'.
 868
 869 If you do not specify a bridge, we create a kvm user (NATed) network
 870 device, which provides DHCP and DNS services. The following addresses
 871 are used:
 872
 873  10.0.2.2   Gateway
 874  10.0.2.3   DNS Server
 875  10.0.2.4   SMB Server
 876
 877 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 878 __EOD__
 879
 880 my $net_fmt = {
 881     macaddr  => get_standard_option('mac-addr', {
 882         description => "MAC address. That address must be unique withing your network. This is"
 883             ." automatically generated if not specified.",
 884     }),
 885     model => {
 886         type => 'string',
 887         description => "Network Card Model. The 'virtio' model provides the best performance with"
 888             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 889             ." best to use 'e1000'.",
 890         enum => $nic_model_list,
 891         default_key => 1,
 892     },
 893     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 894     bridge => {
 895         type => 'string',
 896         description => $net_fmt_bridge_descr,
 897         format_description => 'bridge',
 898         pattern => '[-_.\w\d]+',
 899         optional => 1,
 900     },
 901     queues => {
 902         type => 'integer',
 903         minimum => 0, maximum => 16,
 904         description => 'Number of packet queues to be used on the device.',
 905         optional => 1,
 906     },
 907     rate => {
 908         type => 'number',
 909         minimum => 0,
 910         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 911         optional => 1,
 912     },
 913     tag => {
 914         type => 'integer',
 915         minimum => 1, maximum => 4094,
 916         description => 'VLAN tag to apply to packets on this interface.',
 917         optional => 1,
 918     },
 919     trunks => {
 920         type => 'string',
 921         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 922         description => 'VLAN trunks to pass through this interface.',
 923         format_description => 'vlanid[;vlanid...]',
 924         optional => 1,
 925     },
 926     firewall => {
 927         type => 'boolean',
 928         description => 'Whether this interface should be protected by the firewall.',
 929         optional => 1,
 930     },
 931     link_down => {
 932         type => 'boolean',
 933         description => 'Whether this interface should be disconnected (like pulling the plug).',
 934         optional => 1,
 935     },
 936     mtu => {
 937         type => 'integer',
 938         minimum => 1, maximum => 65520,
 939         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 940         optional => 1,
 941     },
 942 };
 943
 944 my $netdesc = {
 945     optional => 1,
 946     type => 'string', format => $net_fmt,
 947     description => "Specify network devices.",
 948 };
 949
 950 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 951
 952 my $ipconfig_fmt = {
 953     ip => {
 954         type => 'string',
 955         format => 'pve-ipv4-config',
 956         format_description => 'IPv4Format/CIDR',
 957         description => 'IPv4 address in CIDR format.',
 958         optional => 1,
 959         default => 'dhcp',
 960     },
 961     gw => {
 962         type => 'string',
 963         format => 'ipv4',
 964         format_description => 'GatewayIPv4',
 965         description => 'Default gateway for IPv4 traffic.',
 966         optional => 1,
 967         requires => 'ip',
 968     },
 969     ip6 => {
 970         type => 'string',
 971         format => 'pve-ipv6-config',
 972         format_description => 'IPv6Format/CIDR',
 973         description => 'IPv6 address in CIDR format.',
 974         optional => 1,
 975         default => 'dhcp',
 976     },
 977     gw6 => {
 978         type => 'string',
 979         format => 'ipv6',
 980         format_description => 'GatewayIPv6',
 981         description => 'Default gateway for IPv6 traffic.',
 982         optional => 1,
 983         requires => 'ip6',
 984     },
 985 };
 986 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 987 my $ipconfigdesc = {
 988     optional => 1,
 989     type => 'string', format => 'pve-qm-ipconfig',
 990     description => <<'EODESCR',
 991 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 992
 993 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 994
 995 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 996 gateway should be provided.
 997 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 998 cloud-init 19.4 or newer.
 999
1000 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1001 dhcp on IPv4.
1002 EODESCR
1003 };
1004 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1005
1006 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1007     $confdesc->{"net$i"} = $netdesc;
1008     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1009 }
1010
1011 foreach my $key (keys %$confdesc_cloudinit) {
1012     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1013 }
1014
1015 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1016 sub verify_volume_id_or_qm_path {
1017     my ($volid, $noerr) = @_;
1018
1019     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1020         return $volid;
1021     }
1022
1023     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1024     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1025     if ($@) {
1026         return if $noerr;
1027         die $@;
1028     }
1029     return $volid;
1030 }
1031
1032 my $usb_fmt = {
1033     host => {
1034         default_key => 1,
1035         type => 'string', format => 'pve-qm-usb-device',
1036         format_description => 'HOSTUSBDEVICE|spice',
1037         description => <<EODESCR,
1038 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1039
1040  'bus-port(.port)*' (decimal numbers) or
1041  'vendor_id:product_id' (hexadeciaml numbers) or
1042  'spice'
1043
1044 You can use the 'lsusb -t' command to list existing usb devices.
1045
1046 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1047 machines - use with special care.
1048
1049 The value 'spice' can be used to add a usb redirection devices for spice.
1050 EODESCR
1051     },
1052     usb3 => {
1053         optional => 1,
1054         type => 'boolean',
1055         description => "Specifies whether if given host option is a USB3 device or port.",
1056         default => 0,
1057     },
1058 };
1059
1060 my $usbdesc = {
1061     optional => 1,
1062     type => 'string', format => $usb_fmt,
1063     description => "Configure an USB device (n is 0 to 4).",
1064 };
1065 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1066
1067 my $serialdesc = {
1068         optional => 1,
1069         type => 'string',
1070         pattern => '(/dev/.+|socket)',
1071         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1072         verbose_description =>  <<EODESCR,
1073 Create a serial device inside the VM (n is 0 to 3), and pass through a
1074 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1075 host side (use 'qm terminal' to open a terminal connection).
1076
1077 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1078 use with special care.
1079
1080 CAUTION: Experimental! User reported problems with this option.
1081 EODESCR
1082 };
1083
1084 my $paralleldesc= {
1085         optional => 1,
1086         type => 'string',
1087         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1088         description =>  "Map host parallel devices (n is 0 to 2).",
1089         verbose_description =>  <<EODESCR,
1090 Map host parallel devices (n is 0 to 2).
1091
1092 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1093 machines - use with special care.
1094
1095 CAUTION: Experimental! User reported problems with this option.
1096 EODESCR
1097 };
1098
1099 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1100     $confdesc->{"parallel$i"} = $paralleldesc;
1101 }
1102
1103 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1104     $confdesc->{"serial$i"} = $serialdesc;
1105 }
1106
1107 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1108     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1109 }
1110
1111 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1112     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1113 }
1114
1115 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1116     $confdesc->{"usb$i"} = $usbdesc;
1117 }
1118
1119 my $boot_fmt = {
1120     legacy => {
1121         optional => 1,
1122         default_key => 1,
1123         type => 'string',
1124         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1125                      . " Deprecated, use 'order=' instead.",
1126         pattern => '[acdn]{1,4}',
1127         format_description => "[acdn]{1,4}",
1128
1129         # note: this is also the fallback if boot: is not given at all
1130         default => 'cdn',
1131     },
1132     order => {
1133         optional => 1,
1134         type => 'string',
1135         format => 'pve-qm-bootdev-list',
1136         format_description => "device[;device...]",
1137         description => <<EODESC,
1138 The guest will attempt to boot from devices in the order they appear here.
1139
1140 Disks, optical drives and passed-through storage USB devices will be directly
1141 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1142 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1143
1144 Note that only devices in this list will be marked as bootable and thus loaded
1145 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1146 (e.g. software-raid), you need to specify all of them here.
1147
1148 Overrides the deprecated 'legacy=[acdn]*' value when given.
1149 EODESC
1150     },
1151 };
1152 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1153
1154 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1155 sub verify_bootdev {
1156     my ($dev, $noerr) = @_;
1157
1158     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1159     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1160
1161     my $check = sub {
1162         my ($base) = @_;
1163         return 0 if $dev !~ m/^$base\d+$/;
1164         return 0 if !$confdesc->{$dev};
1165         return 1;
1166     };
1167
1168     return $dev if $check->("net");
1169     return $dev if $check->("usb");
1170     return $dev if $check->("hostpci");
1171
1172     return if $noerr;
1173     die "invalid boot device '$dev'\n";
1174 }
1175
1176 sub print_bootorder {
1177     my ($devs) = @_;
1178     return "" if !@$devs;
1179     my $data = { order => join(';', @$devs) };
1180     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1181 }
1182
1183 my $kvm_api_version = 0;
1184
1185 sub kvm_version {
1186     return $kvm_api_version if $kvm_api_version;
1187
1188     open my $fh, '<', '/dev/kvm' or return;
1189
1190     # 0xae00 => KVM_GET_API_VERSION
1191     $kvm_api_version = ioctl($fh, 0xae00, 0);
1192     close($fh);
1193
1194     return $kvm_api_version;
1195 }
1196
1197 my $kvm_user_version = {};
1198 my $kvm_mtime = {};
1199
1200 sub kvm_user_version {
1201     my ($binary) = @_;
1202
1203     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1204     my $st = stat($binary);
1205
1206     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1207     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1208         $cachedmtime == $st->mtime;
1209
1210     $kvm_user_version->{$binary} = 'unknown';
1211     $kvm_mtime->{$binary} = $st->mtime;
1212
1213     my $code = sub {
1214         my $line = shift;
1215         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1216             $kvm_user_version->{$binary} = $2;
1217         }
1218     };
1219
1220     eval { run_command([$binary, '--version'], outfunc => $code); };
1221     warn $@ if $@;
1222
1223     return $kvm_user_version->{$binary};
1224
1225 }
1226 my sub extract_version {
1227     my ($machine_type, $version) = @_;
1228     $version = kvm_user_version() if !defined($version);
1229     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1230 }
1231
1232 sub kernel_has_vhost_net {
1233     return -c '/dev/vhost-net';
1234 }
1235
1236 sub option_exists {
1237     my $key = shift;
1238     return defined($confdesc->{$key});
1239 }
1240
1241 my $cdrom_path;
1242 sub get_cdrom_path {
1243
1244     return  $cdrom_path if $cdrom_path;
1245
1246     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1247     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1248     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub scsi_inquiry {
1354     my($fh, $noerr) = @_;
1355
1356     my $SG_IO = 0x2285;
1357     my $SG_GET_VERSION_NUM = 0x2282;
1358
1359     my $versionbuf = "\x00" x 8;
1360     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1361     if (!$ret) {
1362         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1363         return;
1364     }
1365     my $version = unpack("I", $versionbuf);
1366     if ($version < 30000) {
1367         die "scsi generic interface too old\n"  if !$noerr;
1368         return;
1369     }
1370
1371     my $buf = "\x00" x 36;
1372     my $sensebuf = "\x00" x 8;
1373     my $cmd = pack("C x3 C x1", 0x12, 36);
1374
1375     # see /usr/include/scsi/sg.h
1376     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1377
1378     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1379                       length($sensebuf), 0, length($buf), $buf,
1380                       $cmd, $sensebuf, 6000);
1381
1382     $ret = ioctl($fh, $SG_IO, $packet);
1383     if (!$ret) {
1384         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1385         return;
1386     }
1387
1388     my @res = unpack($sg_io_hdr_t, $packet);
1389     if ($res[17] || $res[18]) {
1390         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1391         return;
1392     }
1393
1394     my $res = {};
1395     (my $byte0, my $byte1, $res->{vendor},
1396      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1397
1398     $res->{removable} = $byte1 & 128 ? 1 : 0;
1399     $res->{type} = $byte0 & 31;
1400
1401     return $res;
1402 }
1403
1404 sub path_is_scsi {
1405     my ($path) = @_;
1406
1407     my $fh = IO::File->new("+<$path") || return;
1408     my $res = scsi_inquiry($fh, 1);
1409     close($fh);
1410
1411     return $res;
1412 }
1413
1414 sub print_tabletdevice_full {
1415     my ($conf, $arch) = @_;
1416
1417     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1418
1419     # we use uhci for old VMs because tablet driver was buggy in older qemu
1420     my $usbbus;
1421     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1422         $usbbus = 'ehci';
1423     } else {
1424         $usbbus = 'uhci';
1425     }
1426
1427     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1428 }
1429
1430 sub print_keyboarddevice_full {
1431     my ($conf, $arch, $machine) = @_;
1432
1433     return if $arch ne 'aarch64';
1434
1435     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1436 }
1437
1438 my sub get_drive_id {
1439     my ($drive) = @_;
1440     return "$drive->{interface}$drive->{index}";
1441 }
1442
1443 sub print_drivedevice_full {
1444     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1445
1446     my $device = '';
1447     my $maxdev = 0;
1448
1449     my $drive_id = get_drive_id($drive);
1450     if ($drive->{interface} eq 'virtio') {
1451         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1452         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1453         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1454     } elsif ($drive->{interface} eq 'scsi') {
1455
1456         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1457         my $unit = $drive->{index} % $maxdev;
1458         my $devicetype = 'hd';
1459         my $path = '';
1460         if (drive_is_cdrom($drive)) {
1461             $devicetype = 'cd';
1462         } else {
1463             if ($drive->{file} =~ m|^/|) {
1464                 $path = $drive->{file};
1465                 if (my $info = path_is_scsi($path)) {
1466                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1467                         $devicetype = 'block';
1468                     } elsif ($info->{type} == 1) { # tape
1469                         $devicetype = 'generic';
1470                     }
1471                 }
1472             } else {
1473                  $path = PVE::Storage::path($storecfg, $drive->{file});
1474             }
1475
1476             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1477             my $version = extract_version($machine_type, kvm_user_version());
1478             if ($path =~ m/^iscsi\:\/\// &&
1479                !min_version($version, 4, 1)) {
1480                 $devicetype = 'generic';
1481             }
1482         }
1483
1484         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1485             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1486         } else {
1487             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1488                 .",lun=$drive->{index}";
1489         }
1490         $device .= ",drive=drive-$drive_id,id=$drive_id";
1491
1492         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1493             $device .= ",rotation_rate=1";
1494         }
1495         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1496
1497     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1498         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1499         my $controller = int($drive->{index} / $maxdev);
1500         my $unit = $drive->{index} % $maxdev;
1501         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1502
1503         $device = "ide-$devicetype";
1504         if ($drive->{interface} eq 'ide') {
1505             $device .= ",bus=ide.$controller,unit=$unit";
1506         } else {
1507             $device .= ",bus=ahci$controller.$unit";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($devicetype eq 'hd') {
1512             if (my $model = $drive->{model}) {
1513                 $model = URI::Escape::uri_unescape($model);
1514                 $device .= ",model=$model";
1515             }
1516             if ($drive->{ssd}) {
1517                 $device .= ",rotation_rate=1";
1518             }
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521     } elsif ($drive->{interface} eq 'usb') {
1522         die "implement me";
1523         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1524     } else {
1525         die "unsupported interface type";
1526     }
1527
1528     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1529
1530     if (my $serial = $drive->{serial}) {
1531         $serial = URI::Escape::uri_unescape($serial);
1532         $device .= ",serial=$serial";
1533     }
1534
1535
1536     return $device;
1537 }
1538
1539 sub get_initiator_name {
1540     my $initiator;
1541
1542     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1543     while (defined(my $line = <$fh>)) {
1544         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1545         $initiator = $1;
1546         last;
1547     }
1548     $fh->close();
1549
1550     return $initiator;
1551 }
1552
1553 sub print_drive_commandline_full {
1554     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1555
1556     my $path;
1557     my $volid = $drive->{file};
1558     my $format = $drive->{format};
1559     my $drive_id = get_drive_id($drive);
1560
1561     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1562     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1563
1564     if (drive_is_cdrom($drive)) {
1565         $path = get_iso_path($storecfg, $vmid, $volid);
1566         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1567     } else {
1568         if ($storeid) {
1569             $path = PVE::Storage::path($storecfg, $volid);
1570             $format //= qemu_img_format($scfg, $volname);
1571         } else {
1572             $path = $volid;
1573             $format //= "raw";
1574         }
1575    }
1576
1577    my $is_rbd = $path =~ m/^rbd:/;
1578
1579     my $opts = '';
1580     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1581     foreach my $o (@qemu_drive_options) {
1582         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1583     }
1584
1585     # snapshot only accepts on|off
1586     if (defined($drive->{snapshot})) {
1587         my $v = $drive->{snapshot} ? 'on' : 'off';
1588         $opts .= ",snapshot=$v";
1589     }
1590
1591     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1592         my ($dir, $qmpname) = @$type;
1593         if (my $v = $drive->{"mbps$dir"}) {
1594             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1595         }
1596         if (my $v = $drive->{"mbps${dir}_max"}) {
1597             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1598         }
1599         if (my $v = $drive->{"bps${dir}_max_length"}) {
1600             $opts .= ",throttling.bps$qmpname-max-length=$v";
1601         }
1602         if (my $v = $drive->{"iops${dir}"}) {
1603             $opts .= ",throttling.iops$qmpname=$v";
1604         }
1605         if (my $v = $drive->{"iops${dir}_max"}) {
1606             $opts .= ",throttling.iops$qmpname-max=$v";
1607         }
1608         if (my $v = $drive->{"iops${dir}_max_length"}) {
1609             $opts .= ",throttling.iops$qmpname-max-length=$v";
1610         }
1611     }
1612
1613     if ($pbs_name) {
1614         $format = "rbd" if $is_rbd;
1615         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1616             if !$format;
1617         $opts .= ",format=alloc-track,file.driver=$format";
1618     } elsif ($format) {
1619         $opts .= ",format=$format";
1620     }
1621
1622     my $cache_direct = 0;
1623
1624     if (my $cache = $drive->{cache}) {
1625         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1626     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1627         $opts .= ",cache=none";
1628         $cache_direct = 1;
1629     }
1630
1631     # io_uring with cache mode writeback or writethrough on krbd will hang...
1632     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1633
1634     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1635     # sometimes, just plain disable...
1636     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1637
1638     if (!$drive->{aio}) {
1639         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1640             # io_uring supports all cache modes
1641             $opts .= ",aio=io_uring";
1642         } else {
1643             # aio native works only with O_DIRECT
1644             if($cache_direct) {
1645                 $opts .= ",aio=native";
1646             } else {
1647                 $opts .= ",aio=threads";
1648             }
1649         }
1650     }
1651
1652     if (!drive_is_cdrom($drive)) {
1653         my $detectzeroes;
1654         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1655             $detectzeroes = 'off';
1656         } elsif ($drive->{discard}) {
1657             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1658         } else {
1659             # This used to be our default with discard not being specified:
1660             $detectzeroes = 'on';
1661         }
1662
1663         # note: 'detect-zeroes' works per blockdev and we want it to persist
1664         # after the alloc-track is removed, so put it on 'file' directly
1665         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1666         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1667     }
1668
1669     if ($pbs_name) {
1670         $opts .= ",backing=$pbs_name";
1671         $opts .= ",auto-remove=on";
1672     }
1673
1674     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1675     my $file_param = "file";
1676     if ($pbs_name) {
1677         # non-rbd drivers require the underlying file to be a seperate block
1678         # node, so add a second .file indirection
1679         $file_param .= ".file" if !$is_rbd;
1680         $file_param .= ".filename";
1681     }
1682     my $pathinfo = $path ? "$file_param=$path," : '';
1683
1684     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1685 }
1686
1687 sub print_pbs_blockdev {
1688     my ($pbs_conf, $pbs_name) = @_;
1689     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1690     $blockdev .= ",repository=$pbs_conf->{repository}";
1691     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1692     $blockdev .= ",archive=$pbs_conf->{archive}";
1693     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1694     return $blockdev;
1695 }
1696
1697 sub print_netdevice_full {
1698     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1699
1700     my $device = $net->{model};
1701     if ($net->{model} eq 'virtio') {
1702          $device = 'virtio-net-pci';
1703      };
1704
1705     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1706     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1707     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1708         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1709         # and out of each queue plus one config interrupt and control vector queue
1710         my $vectors = $net->{queues} * 2 + 2;
1711         $tmpstr .= ",vectors=$vectors,mq=on";
1712     }
1713     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1714
1715     if (my $mtu = $net->{mtu}) {
1716         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1717             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1718             if ($mtu == 1) {
1719                  $mtu = $bridge_mtu;
1720             } elsif ($mtu < 576) {
1721                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1722             } elsif ($mtu > $bridge_mtu) {
1723                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1724             }
1725             $tmpstr .= ",host_mtu=$mtu";
1726         } else {
1727             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1728         }
1729     }
1730
1731     if ($use_old_bios_files) {
1732         my $romfile;
1733         if ($device eq 'virtio-net-pci') {
1734             $romfile = 'pxe-virtio.rom';
1735         } elsif ($device eq 'e1000') {
1736             $romfile = 'pxe-e1000.rom';
1737         } elsif ($device eq 'e1000e') {
1738             $romfile = 'pxe-e1000e.rom';
1739         } elsif ($device eq 'ne2k') {
1740             $romfile = 'pxe-ne2k_pci.rom';
1741         } elsif ($device eq 'pcnet') {
1742             $romfile = 'pxe-pcnet.rom';
1743         } elsif ($device eq 'rtl8139') {
1744             $romfile = 'pxe-rtl8139.rom';
1745         }
1746         $tmpstr .= ",romfile=$romfile" if $romfile;
1747     }
1748
1749     return $tmpstr;
1750 }
1751
1752 sub print_netdev_full {
1753     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1754
1755     my $i = '';
1756     if ($netid =~ m/^net(\d+)$/) {
1757         $i = int($1);
1758     }
1759
1760     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1761
1762     my $ifname = "tap${vmid}i$i";
1763
1764     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1765     die "interface name '$ifname' is too long (max 15 character)\n"
1766         if length($ifname) >= 16;
1767
1768     my $vhostparam = '';
1769     if (is_native($arch)) {
1770         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1771     }
1772
1773     my $vmname = $conf->{name} || "vm$vmid";
1774
1775     my $netdev = "";
1776     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1777
1778     if ($net->{bridge}) {
1779         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1780             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1781     } else {
1782         $netdev = "type=user,id=$netid,hostname=$vmname";
1783     }
1784
1785     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1786
1787     return $netdev;
1788 }
1789
1790 my $vga_map = {
1791     'cirrus' => 'cirrus-vga',
1792     'std' => 'VGA',
1793     'vmware' => 'vmware-svga',
1794     'virtio' => 'virtio-vga',
1795 };
1796
1797 sub print_vga_device {
1798     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1799
1800     my $type = $vga_map->{$vga->{type}};
1801     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1802         $type = 'virtio-gpu';
1803     }
1804     my $vgamem_mb = $vga->{memory};
1805
1806     my $max_outputs = '';
1807     if ($qxlnum) {
1808         $type = $id ? 'qxl' : 'qxl-vga';
1809
1810         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1811             # set max outputs so linux can have up to 4 qxl displays with one device
1812             if (min_version($machine_version, 4, 1)) {
1813                 $max_outputs = ",max_outputs=4";
1814             }
1815         }
1816     }
1817
1818     die "no devicetype for $vga->{type}\n" if !$type;
1819
1820     my $memory = "";
1821     if ($vgamem_mb) {
1822         if ($vga->{type} eq 'virtio') {
1823             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1824             $memory = ",max_hostmem=$bytes";
1825         } elsif ($qxlnum) {
1826             # from https://www.spice-space.org/multiple-monitors.html
1827             $memory = ",vgamem_mb=$vga->{memory}";
1828             my $ram = $vgamem_mb * 4;
1829             my $vram = $vgamem_mb * 2;
1830             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1831         } else {
1832             $memory = ",vgamem_mb=$vga->{memory}";
1833         }
1834     } elsif ($qxlnum && $id) {
1835         $memory = ",ram_size=67108864,vram_size=33554432";
1836     }
1837
1838     my $edidoff = "";
1839     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1840         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1841     }
1842
1843     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1844     my $vgaid = "vga" . ($id // '');
1845     my $pciaddr;
1846
1847     if ($q35 && $vgaid eq 'vga') {
1848         # the first display uses pcie.0 bus on q35 machines
1849         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1850     } else {
1851         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1852     }
1853
1854     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1855 }
1856
1857 sub parse_number_sets {
1858     my ($set) = @_;
1859     my $res = [];
1860     foreach my $part (split(/;/, $set)) {
1861         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1862             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1863             push @$res, [ $1, $2 ];
1864         } else {
1865             die "invalid range: $part\n";
1866         }
1867     }
1868     return $res;
1869 }
1870
1871 sub parse_numa {
1872     my ($data) = @_;
1873
1874     my $res = parse_property_string($numa_fmt, $data);
1875     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1876     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1877     return $res;
1878 }
1879
1880 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1881 sub parse_net {
1882     my ($data) = @_;
1883
1884     my $res = eval { parse_property_string($net_fmt, $data) };
1885     if ($@) {
1886         warn $@;
1887         return;
1888     }
1889     if (!defined($res->{macaddr})) {
1890         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1891         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1892     }
1893     return $res;
1894 }
1895
1896 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1897 sub parse_ipconfig {
1898     my ($data) = @_;
1899
1900     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1901     if ($@) {
1902         warn $@;
1903         return;
1904     }
1905
1906     if ($res->{gw} && !$res->{ip}) {
1907         warn 'gateway specified without specifying an IP address';
1908         return;
1909     }
1910     if ($res->{gw6} && !$res->{ip6}) {
1911         warn 'IPv6 gateway specified without specifying an IPv6 address';
1912         return;
1913     }
1914     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1915         warn 'gateway specified together with DHCP';
1916         return;
1917     }
1918     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1919         # gw6 + auto/dhcp
1920         warn "IPv6 gateway specified together with $res->{ip6} address";
1921         return;
1922     }
1923
1924     if (!$res->{ip} && !$res->{ip6}) {
1925         return { ip => 'dhcp', ip6 => 'dhcp' };
1926     }
1927
1928     return $res;
1929 }
1930
1931 sub print_net {
1932     my $net = shift;
1933
1934     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1935 }
1936
1937 sub add_random_macs {
1938     my ($settings) = @_;
1939
1940     foreach my $opt (keys %$settings) {
1941         next if $opt !~ m/^net(\d+)$/;
1942         my $net = parse_net($settings->{$opt});
1943         next if !$net;
1944         $settings->{$opt} = print_net($net);
1945     }
1946 }
1947
1948 sub vm_is_volid_owner {
1949     my ($storecfg, $vmid, $volid) = @_;
1950
1951     if ($volid !~  m|^/|) {
1952         my ($path, $owner);
1953         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1954         if ($owner && ($owner == $vmid)) {
1955             return 1;
1956         }
1957     }
1958
1959     return;
1960 }
1961
1962 sub vmconfig_register_unused_drive {
1963     my ($storecfg, $vmid, $conf, $drive) = @_;
1964
1965     if (drive_is_cloudinit($drive)) {
1966         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1967         warn $@ if $@;
1968     } elsif (!drive_is_cdrom($drive)) {
1969         my $volid = $drive->{file};
1970         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1971             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1972         }
1973     }
1974 }
1975
1976 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1977 my $smbios1_fmt = {
1978     uuid => {
1979         type => 'string',
1980         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1981         format_description => 'UUID',
1982         description => "Set SMBIOS1 UUID.",
1983         optional => 1,
1984     },
1985     version => {
1986         type => 'string',
1987         pattern => '[A-Za-z0-9+\/]+={0,2}',
1988         format_description => 'Base64 encoded string',
1989         description => "Set SMBIOS1 version.",
1990         optional => 1,
1991     },
1992     serial => {
1993         type => 'string',
1994         pattern => '[A-Za-z0-9+\/]+={0,2}',
1995         format_description => 'Base64 encoded string',
1996         description => "Set SMBIOS1 serial number.",
1997         optional => 1,
1998     },
1999     manufacturer => {
2000         type => 'string',
2001         pattern => '[A-Za-z0-9+\/]+={0,2}',
2002         format_description => 'Base64 encoded string',
2003         description => "Set SMBIOS1 manufacturer.",
2004         optional => 1,
2005     },
2006     product => {
2007         type => 'string',
2008         pattern => '[A-Za-z0-9+\/]+={0,2}',
2009         format_description => 'Base64 encoded string',
2010         description => "Set SMBIOS1 product ID.",
2011         optional => 1,
2012     },
2013     sku => {
2014         type => 'string',
2015         pattern => '[A-Za-z0-9+\/]+={0,2}',
2016         format_description => 'Base64 encoded string',
2017         description => "Set SMBIOS1 SKU string.",
2018         optional => 1,
2019     },
2020     family => {
2021         type => 'string',
2022         pattern => '[A-Za-z0-9+\/]+={0,2}',
2023         format_description => 'Base64 encoded string',
2024         description => "Set SMBIOS1 family string.",
2025         optional => 1,
2026     },
2027     base64 => {
2028         type => 'boolean',
2029         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2030         optional => 1,
2031     },
2032 };
2033
2034 sub parse_smbios1 {
2035     my ($data) = @_;
2036
2037     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2038     warn $@ if $@;
2039     return $res;
2040 }
2041
2042 sub print_smbios1 {
2043     my ($smbios1) = @_;
2044     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2045 }
2046
2047 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2048
2049 sub parse_watchdog {
2050     my ($value) = @_;
2051
2052     return if !$value;
2053
2054     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2055     warn $@ if $@;
2056     return $res;
2057 }
2058
2059 sub parse_guest_agent {
2060     my ($conf) = @_;
2061
2062     return {} if !defined($conf->{agent});
2063
2064     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2065     warn $@ if $@;
2066
2067     # if the agent is disabled ignore the other potentially set properties
2068     return {} if !$res->{enabled};
2069     return $res;
2070 }
2071
2072 sub get_qga_key {
2073     my ($conf, $key) = @_;
2074     return undef if !defined($conf->{agent});
2075
2076     my $agent = parse_guest_agent($conf);
2077     return $agent->{$key};
2078 }
2079
2080 sub parse_vga {
2081     my ($value) = @_;
2082
2083     return {} if !$value;
2084     my $res = eval { parse_property_string($vga_fmt, $value) };
2085     warn $@ if $@;
2086     return $res;
2087 }
2088
2089 sub parse_rng {
2090     my ($value) = @_;
2091
2092     return if !$value;
2093
2094     my $res = eval { parse_property_string($rng_fmt, $value) };
2095     warn $@ if $@;
2096     return $res;
2097 }
2098
2099 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2100 sub verify_usb_device {
2101     my ($value, $noerr) = @_;
2102
2103     return $value if parse_usb_device($value);
2104
2105     return if $noerr;
2106
2107     die "unable to parse usb device\n";
2108 }
2109
2110 # add JSON properties for create and set function
2111 sub json_config_properties {
2112     my $prop = shift;
2113
2114     foreach my $opt (keys %$confdesc) {
2115         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2116             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2117         $prop->{$opt} = $confdesc->{$opt};
2118     }
2119
2120     return $prop;
2121 }
2122
2123 # return copy of $confdesc_cloudinit to generate documentation
2124 sub cloudinit_config_properties {
2125
2126     return dclone($confdesc_cloudinit);
2127 }
2128
2129 sub check_type {
2130     my ($key, $value) = @_;
2131
2132     die "unknown setting '$key'\n" if !$confdesc->{$key};
2133
2134     my $type = $confdesc->{$key}->{type};
2135
2136     if (!defined($value)) {
2137         die "got undefined value\n";
2138     }
2139
2140     if ($value =~ m/[\n\r]/) {
2141         die "property contains a line feed\n";
2142     }
2143
2144     if ($type eq 'boolean') {
2145         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2146         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2147         die "type check ('boolean') failed - got '$value'\n";
2148     } elsif ($type eq 'integer') {
2149         return int($1) if $value =~ m/^(\d+)$/;
2150         die "type check ('integer') failed - got '$value'\n";
2151     } elsif ($type eq 'number') {
2152         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2153         die "type check ('number') failed - got '$value'\n";
2154     } elsif ($type eq 'string') {
2155         if (my $fmt = $confdesc->{$key}->{format}) {
2156             PVE::JSONSchema::check_format($fmt, $value);
2157             return $value;
2158         }
2159         $value =~ s/^\"(.*)\"$/$1/;
2160         return $value;
2161     } else {
2162         die "internal error"
2163     }
2164 }
2165
2166 sub destroy_vm {
2167     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2168
2169     my $conf = PVE::QemuConfig->load_config($vmid);
2170
2171     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2172
2173     if ($conf->{template}) {
2174         # check if any base image is still used by a linked clone
2175         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2176                 my ($ds, $drive) = @_;
2177                 return if drive_is_cdrom($drive);
2178
2179                 my $volid = $drive->{file};
2180                 return if !$volid || $volid =~ m|^/|;
2181
2182                 die "base volume '$volid' is still in use by linked cloned\n"
2183                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2184
2185         });
2186     }
2187
2188     my $volids = {};
2189     my $remove_owned_drive = sub {
2190         my ($ds, $drive) = @_;
2191         return if drive_is_cdrom($drive, 1);
2192
2193         my $volid = $drive->{file};
2194         return if !$volid || $volid =~ m|^/|;
2195         return if $volids->{$volid};
2196
2197         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2198         return if !$path || !$owner || ($owner != $vmid);
2199
2200         $volids->{$volid} = 1;
2201         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2202         warn "Could not remove disk '$volid', check manually: $@" if $@;
2203     };
2204
2205     # only remove disks owned by this VM (referenced in the config)
2206     my $include_opts = {
2207         include_unused => 1,
2208         extra_keys => ['vmstate'],
2209     };
2210     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2211
2212     for my $snap (values %{$conf->{snapshots}}) {
2213         next if !defined($snap->{vmstate});
2214         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2215         next if !defined($drive);
2216         $remove_owned_drive->('vmstate', $drive);
2217     }
2218
2219     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2220
2221     if ($purge_unreferenced) { # also remove unreferenced disk
2222         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2223         PVE::Storage::foreach_volid($vmdisks, sub {
2224             my ($volid, $sid, $volname, $d) = @_;
2225             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2226             warn $@ if $@;
2227         });
2228     }
2229
2230     if (defined $replacement_conf) {
2231         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2232     } else {
2233         PVE::QemuConfig->destroy_config($vmid);
2234     }
2235 }
2236
2237 sub parse_vm_config {
2238     my ($filename, $raw) = @_;
2239
2240     return if !defined($raw);
2241
2242     my $res = {
2243         digest => Digest::SHA::sha1_hex($raw),
2244         snapshots => {},
2245         pending => {},
2246     };
2247
2248     $filename =~ m|/qemu-server/(\d+)\.conf$|
2249         || die "got strange filename '$filename'";
2250
2251     my $vmid = $1;
2252
2253     my $conf = $res;
2254     my $descr;
2255     my $section = '';
2256
2257     my @lines = split(/\n/, $raw);
2258     foreach my $line (@lines) {
2259         next if $line =~ m/^\s*$/;
2260
2261         if ($line =~ m/^\[PENDING\]\s*$/i) {
2262             $section = 'pending';
2263             if (defined($descr)) {
2264                 $descr =~ s/\s+$//;
2265                 $conf->{description} = $descr;
2266             }
2267             $descr = undef;
2268             $conf = $res->{$section} = {};
2269             next;
2270
2271         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2272             $section = $1;
2273             if (defined($descr)) {
2274                 $descr =~ s/\s+$//;
2275                 $conf->{description} = $descr;
2276             }
2277             $descr = undef;
2278             $conf = $res->{snapshots}->{$section} = {};
2279             next;
2280         }
2281
2282         if ($line =~ m/^\#(.*)\s*$/) {
2283             $descr = '' if !defined($descr);
2284             $descr .= PVE::Tools::decode_text($1) . "\n";
2285             next;
2286         }
2287
2288         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2289             $descr = '' if !defined($descr);
2290             $descr .= PVE::Tools::decode_text($2);
2291         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2292             $conf->{snapstate} = $1;
2293         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2294             my $key = $1;
2295             my $value = $2;
2296             $conf->{$key} = $value;
2297         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2298             my $value = $1;
2299             if ($section eq 'pending') {
2300                 $conf->{delete} = $value; # we parse this later
2301             } else {
2302                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2303             }
2304         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2305             my $key = $1;
2306             my $value = $2;
2307             eval { $value = check_type($key, $value); };
2308             if ($@) {
2309                 warn "vm $vmid - unable to parse value of '$key' - $@";
2310             } else {
2311                 $key = 'ide2' if $key eq 'cdrom';
2312                 my $fmt = $confdesc->{$key}->{format};
2313                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2314                     my $v = parse_drive($key, $value);
2315                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2316                         $v->{file} = $volid;
2317                         $value = print_drive($v);
2318                     } else {
2319                         warn "vm $vmid - unable to parse value of '$key'\n";
2320                         next;
2321                     }
2322                 }
2323
2324                 $conf->{$key} = $value;
2325             }
2326         } else {
2327             warn "vm $vmid - unable to parse config: $line\n";
2328         }
2329     }
2330
2331     if (defined($descr)) {
2332         $descr =~ s/\s+$//;
2333         $conf->{description} = $descr;
2334     }
2335     delete $res->{snapstate}; # just to be sure
2336
2337     return $res;
2338 }
2339
2340 sub write_vm_config {
2341     my ($filename, $conf) = @_;
2342
2343     delete $conf->{snapstate}; # just to be sure
2344
2345     if ($conf->{cdrom}) {
2346         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2347         $conf->{ide2} = $conf->{cdrom};
2348         delete $conf->{cdrom};
2349     }
2350
2351     # we do not use 'smp' any longer
2352     if ($conf->{sockets}) {
2353         delete $conf->{smp};
2354     } elsif ($conf->{smp}) {
2355         $conf->{sockets} = $conf->{smp};
2356         delete $conf->{cores};
2357         delete $conf->{smp};
2358     }
2359
2360     my $used_volids = {};
2361
2362     my $cleanup_config = sub {
2363         my ($cref, $pending, $snapname) = @_;
2364
2365         foreach my $key (keys %$cref) {
2366             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2367                 $key eq 'snapstate' || $key eq 'pending';
2368             my $value = $cref->{$key};
2369             if ($key eq 'delete') {
2370                 die "propertry 'delete' is only allowed in [PENDING]\n"
2371                     if !$pending;
2372                 # fixme: check syntax?
2373                 next;
2374             }
2375             eval { $value = check_type($key, $value); };
2376             die "unable to parse value of '$key' - $@" if $@;
2377
2378             $cref->{$key} = $value;
2379
2380             if (!$snapname && is_valid_drivename($key)) {
2381                 my $drive = parse_drive($key, $value);
2382                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2383             }
2384         }
2385     };
2386
2387     &$cleanup_config($conf);
2388
2389     &$cleanup_config($conf->{pending}, 1);
2390
2391     foreach my $snapname (keys %{$conf->{snapshots}}) {
2392         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2393         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2394     }
2395
2396     # remove 'unusedX' settings if we re-add a volume
2397     foreach my $key (keys %$conf) {
2398         my $value = $conf->{$key};
2399         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2400             delete $conf->{$key};
2401         }
2402     }
2403
2404     my $generate_raw_config = sub {
2405         my ($conf, $pending) = @_;
2406
2407         my $raw = '';
2408
2409         # add description as comment to top of file
2410         if (defined(my $descr = $conf->{description})) {
2411             if ($descr) {
2412                 foreach my $cl (split(/\n/, $descr)) {
2413                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2414                 }
2415             } else {
2416                 $raw .= "#\n" if $pending;
2417             }
2418         }
2419
2420         foreach my $key (sort keys %$conf) {
2421             next if $key =~ /^(digest|description|pending|snapshots)$/;
2422             $raw .= "$key: $conf->{$key}\n";
2423         }
2424         return $raw;
2425     };
2426
2427     my $raw = &$generate_raw_config($conf);
2428
2429     if (scalar(keys %{$conf->{pending}})){
2430         $raw .= "\n[PENDING]\n";
2431         $raw .= &$generate_raw_config($conf->{pending}, 1);
2432     }
2433
2434     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2435         $raw .= "\n[$snapname]\n";
2436         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2437     }
2438
2439     return $raw;
2440 }
2441
2442 sub load_defaults {
2443
2444     my $res = {};
2445
2446     # we use static defaults from our JSON schema configuration
2447     foreach my $key (keys %$confdesc) {
2448         if (defined(my $default = $confdesc->{$key}->{default})) {
2449             $res->{$key} = $default;
2450         }
2451     }
2452
2453     return $res;
2454 }
2455
2456 sub config_list {
2457     my $vmlist = PVE::Cluster::get_vmlist();
2458     my $res = {};
2459     return $res if !$vmlist || !$vmlist->{ids};
2460     my $ids = $vmlist->{ids};
2461     my $nodename = nodename();
2462
2463     foreach my $vmid (keys %$ids) {
2464         my $d = $ids->{$vmid};
2465         next if !$d->{node} || $d->{node} ne $nodename;
2466         next if !$d->{type} || $d->{type} ne 'qemu';
2467         $res->{$vmid}->{exists} = 1;
2468     }
2469     return $res;
2470 }
2471
2472 # test if VM uses local resources (to prevent migration)
2473 sub check_local_resources {
2474     my ($conf, $noerr) = @_;
2475
2476     my @loc_res = ();
2477
2478     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2479     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2480
2481     push @loc_res, "ivshmem" if $conf->{ivshmem};
2482
2483     foreach my $k (keys %$conf) {
2484         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2485         # sockets are safe: they will recreated be on the target side post-migrate
2486         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2487         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2488     }
2489
2490     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2491
2492     return \@loc_res;
2493 }
2494
2495 # check if used storages are available on all nodes (use by migrate)
2496 sub check_storage_availability {
2497     my ($storecfg, $conf, $node) = @_;
2498
2499     PVE::QemuConfig->foreach_volume($conf, sub {
2500         my ($ds, $drive) = @_;
2501
2502         my $volid = $drive->{file};
2503         return if !$volid;
2504
2505         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2506         return if !$sid;
2507
2508         # check if storage is available on both nodes
2509         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2510         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2511
2512         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2513
2514         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2515             if !$scfg->{content}->{$vtype};
2516    });
2517 }
2518
2519 # list nodes where all VM images are available (used by has_feature API)
2520 sub shared_nodes {
2521     my ($conf, $storecfg) = @_;
2522
2523     my $nodelist = PVE::Cluster::get_nodelist();
2524     my $nodehash = { map { $_ => 1 } @$nodelist };
2525     my $nodename = nodename();
2526
2527     PVE::QemuConfig->foreach_volume($conf, sub {
2528         my ($ds, $drive) = @_;
2529
2530         my $volid = $drive->{file};
2531         return if !$volid;
2532
2533         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2534         if ($storeid) {
2535             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2536             if ($scfg->{disable}) {
2537                 $nodehash = {};
2538             } elsif (my $avail = $scfg->{nodes}) {
2539                 foreach my $node (keys %$nodehash) {
2540                     delete $nodehash->{$node} if !$avail->{$node};
2541                 }
2542             } elsif (!$scfg->{shared}) {
2543                 foreach my $node (keys %$nodehash) {
2544                     delete $nodehash->{$node} if $node ne $nodename
2545                 }
2546             }
2547         }
2548     });
2549
2550     return $nodehash
2551 }
2552
2553 sub check_local_storage_availability {
2554     my ($conf, $storecfg) = @_;
2555
2556     my $nodelist = PVE::Cluster::get_nodelist();
2557     my $nodehash = { map { $_ => {} } @$nodelist };
2558
2559     PVE::QemuConfig->foreach_volume($conf, sub {
2560         my ($ds, $drive) = @_;
2561
2562         my $volid = $drive->{file};
2563         return if !$volid;
2564
2565         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2566         if ($storeid) {
2567             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2568
2569             if ($scfg->{disable}) {
2570                 foreach my $node (keys %$nodehash) {
2571                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2572                 }
2573             } elsif (my $avail = $scfg->{nodes}) {
2574                 foreach my $node (keys %$nodehash) {
2575                     if (!$avail->{$node}) {
2576                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2577                     }
2578                 }
2579             }
2580         }
2581     });
2582
2583     foreach my $node (values %$nodehash) {
2584         if (my $unavail = $node->{unavailable_storages}) {
2585             $node->{unavailable_storages} = [ sort keys %$unavail ];
2586         }
2587     }
2588
2589     return $nodehash
2590 }
2591
2592 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2593 sub check_running {
2594     my ($vmid, $nocheck, $node) = @_;
2595
2596     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2597     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2598 }
2599
2600 sub vzlist {
2601
2602     my $vzlist = config_list();
2603
2604     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2605
2606     while (defined(my $de = $fd->read)) {
2607         next if $de !~ m/^(\d+)\.pid$/;
2608         my $vmid = $1;
2609         next if !defined($vzlist->{$vmid});
2610         if (my $pid = check_running($vmid)) {
2611             $vzlist->{$vmid}->{pid} = $pid;
2612         }
2613     }
2614
2615     return $vzlist;
2616 }
2617
2618 our $vmstatus_return_properties = {
2619     vmid => get_standard_option('pve-vmid'),
2620     status => {
2621         description => "Qemu process status.",
2622         type => 'string',
2623         enum => ['stopped', 'running'],
2624     },
2625     maxmem => {
2626         description => "Maximum memory in bytes.",
2627         type => 'integer',
2628         optional => 1,
2629         renderer => 'bytes',
2630     },
2631     maxdisk => {
2632         description => "Root disk size in bytes.",
2633         type => 'integer',
2634         optional => 1,
2635         renderer => 'bytes',
2636     },
2637     name => {
2638         description => "VM name.",
2639         type => 'string',
2640         optional => 1,
2641     },
2642     qmpstatus => {
2643         description => "Qemu QMP agent status.",
2644         type => 'string',
2645         optional => 1,
2646     },
2647     pid => {
2648         description => "PID of running qemu process.",
2649         type => 'integer',
2650         optional => 1,
2651     },
2652     uptime => {
2653         description => "Uptime.",
2654         type => 'integer',
2655         optional => 1,
2656         renderer => 'duration',
2657     },
2658     cpus => {
2659         description => "Maximum usable CPUs.",
2660         type => 'number',
2661         optional => 1,
2662     },
2663     lock => {
2664         description => "The current config lock, if any.",
2665         type => 'string',
2666         optional => 1,
2667     },
2668     tags => {
2669         description  => "The current configured tags, if any",
2670         type => 'string',
2671         optional => 1,
2672     },
2673     'running-machine' => {
2674         description => "The currently running machine type (if running).",
2675         type => 'string',
2676         optional => 1,
2677     },
2678     'running-qemu' => {
2679         description => "The currently running QEMU version (if running).",
2680         type => 'string',
2681         optional => 1,
2682     },
2683 };
2684
2685 my $last_proc_pid_stat;
2686
2687 # get VM status information
2688 # This must be fast and should not block ($full == false)
2689 # We only query KVM using QMP if $full == true (this can be slow)
2690 sub vmstatus {
2691     my ($opt_vmid, $full) = @_;
2692
2693     my $res = {};
2694
2695     my $storecfg = PVE::Storage::config();
2696
2697     my $list = vzlist();
2698     my $defaults = load_defaults();
2699
2700     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2701
2702     my $cpucount = $cpuinfo->{cpus} || 1;
2703
2704     foreach my $vmid (keys %$list) {
2705         next if $opt_vmid && ($vmid ne $opt_vmid);
2706
2707         my $conf = PVE::QemuConfig->load_config($vmid);
2708
2709         my $d = { vmid => int($vmid) };
2710         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2711
2712         # fixme: better status?
2713         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2714
2715         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2716         if (defined($size)) {
2717             $d->{disk} = 0; # no info available
2718             $d->{maxdisk} = $size;
2719         } else {
2720             $d->{disk} = 0;
2721             $d->{maxdisk} = 0;
2722         }
2723
2724         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2725             * ($conf->{cores} || $defaults->{cores});
2726         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2727         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2728
2729         $d->{name} = $conf->{name} || "VM $vmid";
2730         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2731             : $defaults->{memory}*(1024*1024);
2732
2733         if ($conf->{balloon}) {
2734             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2735             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2736                 : $defaults->{shares};
2737         }
2738
2739         $d->{uptime} = 0;
2740         $d->{cpu} = 0;
2741         $d->{mem} = 0;
2742
2743         $d->{netout} = 0;
2744         $d->{netin} = 0;
2745
2746         $d->{diskread} = 0;
2747         $d->{diskwrite} = 0;
2748
2749         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2750
2751         $d->{serial} = 1 if conf_has_serial($conf);
2752         $d->{lock} = $conf->{lock} if $conf->{lock};
2753         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2754
2755         $res->{$vmid} = $d;
2756     }
2757
2758     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2759     foreach my $dev (keys %$netdev) {
2760         next if $dev !~ m/^tap([1-9]\d*)i/;
2761         my $vmid = $1;
2762         my $d = $res->{$vmid};
2763         next if !$d;
2764
2765         $d->{netout} += $netdev->{$dev}->{receive};
2766         $d->{netin} += $netdev->{$dev}->{transmit};
2767
2768         if ($full) {
2769             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2770             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2771         }
2772
2773     }
2774
2775     my $ctime = gettimeofday;
2776
2777     foreach my $vmid (keys %$list) {
2778
2779         my $d = $res->{$vmid};
2780         my $pid = $d->{pid};
2781         next if !$pid;
2782
2783         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2784         next if !$pstat; # not running
2785
2786         my $used = $pstat->{utime} + $pstat->{stime};
2787
2788         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2789
2790         if ($pstat->{vsize}) {
2791             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2792         }
2793
2794         my $old = $last_proc_pid_stat->{$pid};
2795         if (!$old) {
2796             $last_proc_pid_stat->{$pid} = {
2797                 time => $ctime,
2798                 used => $used,
2799                 cpu => 0,
2800             };
2801             next;
2802         }
2803
2804         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2805
2806         if ($dtime > 1000) {
2807             my $dutime = $used -  $old->{used};
2808
2809             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2810             $last_proc_pid_stat->{$pid} = {
2811                 time => $ctime,
2812                 used => $used,
2813                 cpu => $d->{cpu},
2814             };
2815         } else {
2816             $d->{cpu} = $old->{cpu};
2817         }
2818     }
2819
2820     return $res if !$full;
2821
2822     my $qmpclient = PVE::QMPClient->new();
2823
2824     my $ballooncb = sub {
2825         my ($vmid, $resp) = @_;
2826
2827         my $info = $resp->{'return'};
2828         return if !$info->{max_mem};
2829
2830         my $d = $res->{$vmid};
2831
2832         # use memory assigned to VM
2833         $d->{maxmem} = $info->{max_mem};
2834         $d->{balloon} = $info->{actual};
2835
2836         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2837             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2838             $d->{freemem} = $info->{free_mem};
2839         }
2840
2841         $d->{ballooninfo} = $info;
2842     };
2843
2844     my $blockstatscb = sub {
2845         my ($vmid, $resp) = @_;
2846         my $data = $resp->{'return'} || [];
2847         my $totalrdbytes = 0;
2848         my $totalwrbytes = 0;
2849
2850         for my $blockstat (@$data) {
2851             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2852             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2853
2854             $blockstat->{device} =~ s/drive-//;
2855             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2856         }
2857         $res->{$vmid}->{diskread} = $totalrdbytes;
2858         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2859     };
2860
2861     my $machinecb = sub {
2862         my ($vmid, $resp) = @_;
2863         my $data = $resp->{'return'} || [];
2864
2865         $res->{$vmid}->{'running-machine'} =
2866             PVE::QemuServer::Machine::current_from_query_machines($data);
2867     };
2868
2869     my $versioncb = sub {
2870         my ($vmid, $resp) = @_;
2871         my $data = $resp->{'return'} // {};
2872         my $version = 'unknown';
2873
2874         if (my $v = $data->{qemu}) {
2875             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2876         }
2877
2878         $res->{$vmid}->{'running-qemu'} = $version;
2879     };
2880
2881     my $statuscb = sub {
2882         my ($vmid, $resp) = @_;
2883
2884         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2885         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2886         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2887         # this fails if ballon driver is not loaded, so this must be
2888         # the last commnand (following command are aborted if this fails).
2889         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2890
2891         my $status = 'unknown';
2892         if (!defined($status = $resp->{'return'}->{status})) {
2893             warn "unable to get VM status\n";
2894             return;
2895         }
2896
2897         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2898     };
2899
2900     foreach my $vmid (keys %$list) {
2901         next if $opt_vmid && ($vmid ne $opt_vmid);
2902         next if !$res->{$vmid}->{pid}; # not running
2903         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2904     }
2905
2906     $qmpclient->queue_execute(undef, 2);
2907
2908     foreach my $vmid (keys %$list) {
2909         next if $opt_vmid && ($vmid ne $opt_vmid);
2910         next if !$res->{$vmid}->{pid}; #not running
2911
2912         # we can't use the $qmpclient since it might have already aborted on
2913         # 'query-balloon', but this might also fail for older versions...
2914         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2915         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2916     }
2917
2918     foreach my $vmid (keys %$list) {
2919         next if $opt_vmid && ($vmid ne $opt_vmid);
2920         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2921     }
2922
2923     return $res;
2924 }
2925
2926 sub conf_has_serial {
2927     my ($conf) = @_;
2928
2929     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2930         if ($conf->{"serial$i"}) {
2931             return 1;
2932         }
2933     }
2934
2935     return 0;
2936 }
2937
2938 sub conf_has_audio {
2939     my ($conf, $id) = @_;
2940
2941     $id //= 0;
2942     my $audio = $conf->{"audio$id"};
2943     return if !defined($audio);
2944
2945     my $audioproperties = parse_property_string($audio_fmt, $audio);
2946     my $audiodriver = $audioproperties->{driver} // 'spice';
2947
2948     return {
2949         dev => $audioproperties->{device},
2950         dev_id => "audiodev$id",
2951         backend => $audiodriver,
2952         backend_id => "$audiodriver-backend${id}",
2953     };
2954 }
2955
2956 sub audio_devs {
2957     my ($audio, $audiopciaddr, $machine_version) = @_;
2958
2959     my $devs = [];
2960
2961     my $id = $audio->{dev_id};
2962     my $audiodev = "";
2963     if (min_version($machine_version, 4, 2)) {
2964         $audiodev = ",audiodev=$audio->{backend_id}";
2965     }
2966
2967     if ($audio->{dev} eq 'AC97') {
2968         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2969     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2970         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2971         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2972         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2973     } else {
2974         die "unkown audio device '$audio->{dev}', implement me!";
2975     }
2976
2977     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2978
2979     return $devs;
2980 }
2981
2982 sub get_tpm_paths {
2983     my ($vmid) = @_;
2984     return {
2985         socket => "/var/run/qemu-server/$vmid.swtpm",
2986         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
2987     };
2988 }
2989
2990 sub add_tpm_device {
2991     my ($vmid, $devices, $conf) = @_;
2992
2993     return if !$conf->{tpmstate0};
2994
2995     my $paths = get_tpm_paths($vmid);
2996
2997     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
2998     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
2999     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3000 }
3001
3002 sub start_swtpm {
3003     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3004
3005     return if !$tpmdrive;
3006
3007     my $state;
3008     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3009     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3010     if ($storeid) {
3011         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3012     } else {
3013         $state = $tpm->{file};
3014     }
3015
3016     my $paths = get_tpm_paths($vmid);
3017
3018     # during migration, we will get state from remote
3019     #
3020     if (!$migration) {
3021         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3022         my $setup_cmd = [
3023             "swtpm_setup",
3024             "--tpmstate",
3025             "file://$state",
3026             "--createek",
3027             "--create-ek-cert",
3028             "--create-platform-cert",
3029             "--lock-nvram",
3030             "--config",
3031             "/etc/swtpm_setup.conf", # do not use XDG configs
3032             "--runas",
3033             "0", # force creation as root, error if not possible
3034             "--not-overwrite", # ignore existing state, do not modify
3035         ];
3036
3037         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3038         # TPM 2.0 supports ECC crypto, use if possible
3039         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3040
3041         run_command($setup_cmd, outfunc => sub {
3042             print "swtpm_setup: $1\n";
3043         });
3044     }
3045
3046     my $emulator_cmd = [
3047         "swtpm",
3048         "socket",
3049         "--tpmstate",
3050         "backend-uri=file://$state,mode=0600",
3051         "--ctrl",
3052         "type=unixio,path=$paths->{socket},mode=0600",
3053         "--pid",
3054         "file=$paths->{pid}",
3055         "--terminate", # terminate on QEMU disconnect
3056         "--daemon",
3057     ];
3058     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3059     run_command($emulator_cmd, outfunc => sub { print $1; });
3060
3061     # return untainted PID of swtpm daemon so it can be killed on error
3062     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3063     return $1;
3064 }
3065
3066 sub vga_conf_has_spice {
3067     my ($vga) = @_;
3068
3069     my $vgaconf = parse_vga($vga);
3070     my $vgatype = $vgaconf->{type};
3071     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3072
3073     return $1 || 1;
3074 }
3075
3076 sub is_native($) {
3077     my ($arch) = @_;
3078     return get_host_arch() eq $arch;
3079 }
3080
3081 sub get_vm_arch {
3082     my ($conf) = @_;
3083     return $conf->{arch} // get_host_arch();
3084 }
3085
3086 my $default_machines = {
3087     x86_64 => 'pc',
3088     aarch64 => 'virt',
3089 };
3090
3091 sub get_installed_machine_version {
3092     my ($kvmversion) = @_;
3093     $kvmversion = kvm_user_version() if !defined($kvmversion);
3094     $kvmversion =~ m/^(\d+\.\d+)/;
3095     return $1;
3096 }
3097
3098 sub windows_get_pinned_machine_version {
3099     my ($machine, $base_version, $kvmversion) = @_;
3100
3101     my $pin_version = $base_version;
3102     if (!defined($base_version) ||
3103         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3104     ) {
3105         $pin_version = get_installed_machine_version($kvmversion);
3106     }
3107     if (!$machine || $machine eq 'pc') {
3108         $machine = "pc-i440fx-$pin_version";
3109     } elsif ($machine eq 'q35') {
3110         $machine = "pc-q35-$pin_version";
3111     } elsif ($machine eq 'virt') {
3112         $machine = "virt-$pin_version";
3113     } else {
3114         warn "unknown machine type '$machine', not touching that!\n";
3115     }
3116
3117     return $machine;
3118 }
3119
3120 sub get_vm_machine {
3121     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3122
3123     my $machine = $forcemachine || $conf->{machine};
3124
3125     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3126         $kvmversion //= kvm_user_version();
3127         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3128         # layout which confuses windows quite a bit and may result in various regressions..
3129         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3130         if (windows_version($conf->{ostype})) {
3131             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3132         }
3133         $arch //= 'x86_64';
3134         $machine ||= $default_machines->{$arch};
3135         if ($add_pve_version) {
3136             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3137             $machine .= "+pve$pvever";
3138         }
3139     }
3140
3141     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3142         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3143         $machine = $1 if $is_pxe;
3144
3145         # for version-pinned machines that do not include a pve-version (e.g.
3146         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3147         $machine .= '+pve0';
3148
3149         $machine .= '.pxe' if $is_pxe;
3150     }
3151
3152     return $machine;
3153 }
3154
3155 sub get_ovmf_files($$) {
3156     my ($arch, $efidisk) = @_;
3157
3158     my $types = $OVMF->{$arch}
3159         or die "no OVMF images known for architecture '$arch'\n";
3160
3161     my $type = 'default';
3162     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3163         $type = $efidisk->{'pre-enrolled-keys'} ? "4m-ms" : "4m";
3164     }
3165
3166     return $types->{$type}->@*;
3167 }
3168
3169 my $Arch2Qemu = {
3170     aarch64 => '/usr/bin/qemu-system-aarch64',
3171     x86_64 => '/usr/bin/qemu-system-x86_64',
3172 };
3173 sub get_command_for_arch($) {
3174     my ($arch) = @_;
3175     return '/usr/bin/kvm' if is_native($arch);
3176
3177     my $cmd = $Arch2Qemu->{$arch}
3178         or die "don't know how to emulate architecture '$arch'\n";
3179     return $cmd;
3180 }
3181
3182 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3183 # to use in a QEMU command line (-cpu element), first array_intersect the result
3184 # of query_supported_ with query_understood_. This is necessary because:
3185 #
3186 # a) query_understood_ returns flags the host cannot use and
3187 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3188 #    flags, but CPU settings - with most of them being flags. Those settings
3189 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3190 #
3191 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3192 # expensive. If you need the value returned from this, you can get it much
3193 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3194 # $accel being 'kvm' or 'tcg'.
3195 #
3196 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3197 # changes, automatically populating pmxcfs.
3198 #
3199 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3200 # since kvm and tcg machines support different flags
3201 #
3202 sub query_supported_cpu_flags {
3203     my ($arch) = @_;
3204
3205     $arch //= get_host_arch();
3206     my $default_machine = $default_machines->{$arch};
3207
3208     my $flags = {};
3209
3210     # FIXME: Once this is merged, the code below should work for ARM as well:
3211     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3212     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3213         $arch eq "aarch64";
3214
3215     my $kvm_supported = defined(kvm_version());
3216     my $qemu_cmd = get_command_for_arch($arch);
3217     my $fakevmid = -1;
3218     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3219
3220     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3221     my $query_supported_run_qemu = sub {
3222         my ($kvm) = @_;
3223
3224         my $flags = {};
3225         my $cmd = [
3226             $qemu_cmd,
3227             '-machine', $default_machine,
3228             '-display', 'none',
3229             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3230             '-mon', 'chardev=qmp,mode=control',
3231             '-pidfile', $pidfile,
3232             '-S', '-daemonize'
3233         ];
3234
3235         if (!$kvm) {
3236             push @$cmd, '-accel', 'tcg';
3237         }
3238
3239         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3240         die "QEMU flag querying VM exited with code " . $rc if $rc;
3241
3242         eval {
3243             my $cmd_result = mon_cmd(
3244                 $fakevmid,
3245                 'query-cpu-model-expansion',
3246                 type => 'full',
3247                 model => { name => 'host' }
3248             );
3249
3250             my $props = $cmd_result->{model}->{props};
3251             foreach my $prop (keys %$props) {
3252                 next if $props->{$prop} ne '1';
3253                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3254                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3255                 # We only keep those with underscores, to match /proc/cpuinfo
3256                 $prop =~ s/\.|-/_/g;
3257                 $flags->{$prop} = 1;
3258             }
3259         };
3260         my $err = $@;
3261
3262         # force stop with 10 sec timeout and 'nocheck'
3263         # always stop, even if QMP failed
3264         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3265
3266         die $err if $err;
3267
3268         return [ sort keys %$flags ];
3269     };
3270
3271     # We need to query QEMU twice, since KVM and TCG have different supported flags
3272     PVE::QemuConfig->lock_config($fakevmid, sub {
3273         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3274         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3275
3276         if ($kvm_supported) {
3277             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3278             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3279         }
3280     });
3281
3282     return $flags;
3283 }
3284
3285 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3286 my $understood_cpu_flag_dir = "/usr/share/kvm";
3287 sub query_understood_cpu_flags {
3288     my $arch = get_host_arch();
3289     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3290
3291     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3292         if ! -e $filepath;
3293
3294     my $raw = file_get_contents($filepath);
3295     $raw =~ s/^\s+|\s+$//g;
3296     my @flags = split(/\s+/, $raw);
3297
3298     return \@flags;
3299 }
3300
3301 my sub get_cpuunits {
3302     my ($conf) = @_;
3303     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3304 }
3305 sub config_to_command {
3306     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3307         $pbs_backing) = @_;
3308
3309     my $cmd = [];
3310     my $globalFlags = [];
3311     my $machineFlags = [];
3312     my $rtcFlags = [];
3313     my $devices = [];
3314     my $pciaddr = '';
3315     my $bridges = {};
3316     my $ostype = $conf->{ostype};
3317     my $winversion = windows_version($ostype);
3318     my $kvm = $conf->{kvm};
3319     my $nodename = nodename();
3320
3321     my $arch = get_vm_arch($conf);
3322     my $kvm_binary = get_command_for_arch($arch);
3323     my $kvmver = kvm_user_version($kvm_binary);
3324
3325     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3326         $kvmver //= "undefined";
3327         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3328     }
3329
3330     my $add_pve_version = min_version($kvmver, 4, 1);
3331
3332     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3333     my $machine_version = extract_version($machine_type, $kvmver);
3334     $kvm //= 1 if is_native($arch);
3335
3336     $machine_version =~ m/(\d+)\.(\d+)/;
3337     my ($machine_major, $machine_minor) = ($1, $2);
3338
3339     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3340         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3341     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3342         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3343             ." please upgrade node '$nodename'\n"
3344     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3345         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3346         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3347             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3348             ." node '$nodename'\n";
3349     }
3350
3351     # if a specific +pve version is required for a feature, use $version_guard
3352     # instead of min_version to allow machines to be run with the minimum
3353     # required version
3354     my $required_pve_version = 0;
3355     my $version_guard = sub {
3356         my ($major, $minor, $pve) = @_;
3357         return 0 if !min_version($machine_version, $major, $minor, $pve);
3358         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3359         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3360         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3361         return 1;
3362     };
3363
3364     if ($kvm && !defined kvm_version()) {
3365         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3366             ." or enable in BIOS.\n";
3367     }
3368
3369     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3370     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3371     my $use_old_bios_files = undef;
3372     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3373
3374     my $cpuunits = get_cpuunits($conf);
3375
3376     push @$cmd, $kvm_binary;
3377
3378     push @$cmd, '-id', $vmid;
3379
3380     my $vmname = $conf->{name} || "vm$vmid";
3381
3382     push @$cmd, '-name', $vmname;
3383
3384     push @$cmd, '-no-shutdown';
3385
3386     my $use_virtio = 0;
3387
3388     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3389     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3390     push @$cmd, '-mon', "chardev=qmp,mode=control";
3391
3392     if (min_version($machine_version, 2, 12)) {
3393         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3394         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3395     }
3396
3397     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3398
3399     push @$cmd, '-daemonize';
3400
3401     if ($conf->{smbios1}) {
3402         my $smbios_conf = parse_smbios1($conf->{smbios1});
3403         if ($smbios_conf->{base64}) {
3404             # Do not pass base64 flag to qemu
3405             delete $smbios_conf->{base64};
3406             my $smbios_string = "";
3407             foreach my $key (keys %$smbios_conf) {
3408                 my $value;
3409                 if ($key eq "uuid") {
3410                     $value = $smbios_conf->{uuid}
3411                 } else {
3412                     $value = decode_base64($smbios_conf->{$key});
3413                 }
3414                 # qemu accepts any binary data, only commas need escaping by double comma
3415                 $value =~ s/,/,,/g;
3416                 $smbios_string .= "," . $key . "=" . $value if $value;
3417             }
3418             push @$cmd, '-smbios', "type=1" . $smbios_string;
3419         } else {
3420             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3421         }
3422     }
3423
3424     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3425         my $d;
3426         if (my $efidisk = $conf->{efidisk0}) {
3427             $d = parse_drive('efidisk0', $efidisk);
3428         }
3429
3430         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d);
3431         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3432
3433         my ($path, $format);
3434         my $read_only_str = '';
3435         if ($d) {
3436             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3437             $format = $d->{format};
3438             if ($storeid) {
3439                 $path = PVE::Storage::path($storecfg, $d->{file});
3440                 if (!defined($format)) {
3441                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3442                     $format = qemu_img_format($scfg, $volname);
3443                 }
3444             } else {
3445                 $path = $d->{file};
3446                 die "efidisk format must be specified\n"
3447                     if !defined($format);
3448             }
3449
3450             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3451         } else {
3452             warn "no efidisk configured! Using temporary efivars disk.\n";
3453             $path = "/tmp/$vmid-ovmf.fd";
3454             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3455             $format = 'raw';
3456         }
3457
3458         my $size_str = "";
3459
3460         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3461             $size_str = ",size=" . (-s $ovmf_vars);
3462         }
3463
3464         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3465         my $cache = "";
3466         if ($path =~ m/^rbd:/) {
3467                 $cache = ',cache=writeback';
3468                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3469         }
3470
3471         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3472         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3473     }
3474
3475     # load q35 config
3476     if ($q35) {
3477         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3478         if (min_version($machine_version, 4, 0)) {
3479             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3480         } else {
3481             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3482         }
3483     }
3484
3485     if ($conf->{vmgenid}) {
3486         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3487     }
3488
3489     # add usb controllers
3490     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3491         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3492     push @$devices, @usbcontrollers if @usbcontrollers;
3493     my $vga = parse_vga($conf->{vga});
3494
3495     my $qxlnum = vga_conf_has_spice($conf->{vga});
3496     $vga->{type} = 'qxl' if $qxlnum;
3497
3498     if (!$vga->{type}) {
3499         if ($arch eq 'aarch64') {
3500             $vga->{type} = 'virtio';
3501         } elsif (min_version($machine_version, 2, 9)) {
3502             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3503         } else {
3504             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3505         }
3506     }
3507
3508     # enable absolute mouse coordinates (needed by vnc)
3509     my $tablet;
3510     if (defined($conf->{tablet})) {
3511         $tablet = $conf->{tablet};
3512     } else {
3513         $tablet = $defaults->{tablet};
3514         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3515         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3516     }
3517
3518     if ($tablet) {
3519         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3520         my $kbd = print_keyboarddevice_full($conf, $arch);
3521         push @$devices, '-device', $kbd if defined($kbd);
3522     }
3523
3524     my $bootorder = device_bootorder($conf);
3525
3526     # host pci device passthrough
3527     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3528         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3529
3530     # usb devices
3531     my $usb_dev_features = {};
3532     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3533
3534     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3535         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3536     push @$devices, @usbdevices if @usbdevices;
3537
3538     # serial devices
3539     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3540         if (my $path = $conf->{"serial$i"}) {
3541             if ($path eq 'socket') {
3542                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3543                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3544                 # On aarch64, serial0 is the UART device. Qemu only allows
3545                 # connecting UART devices via the '-serial' command line, as
3546                 # the device has a fixed slot on the hardware...
3547                 if ($arch eq 'aarch64' && $i == 0) {
3548                     push @$devices, '-serial', "chardev:serial$i";
3549                 } else {
3550                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3551                 }
3552             } else {
3553                 die "no such serial device\n" if ! -c $path;
3554                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3555                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3556             }
3557         }
3558     }
3559
3560     # parallel devices
3561     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3562         if (my $path = $conf->{"parallel$i"}) {
3563             die "no such parallel device\n" if ! -c $path;
3564             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3565             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3566             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3567         }
3568     }
3569
3570     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3571         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3572         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3573         push @$devices, @$audio_devs;
3574     }
3575
3576     add_tpm_device($vmid, $devices, $conf);
3577
3578     my $sockets = 1;
3579     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3580     $sockets = $conf->{sockets} if  $conf->{sockets};
3581
3582     my $cores = $conf->{cores} || 1;
3583
3584     my $maxcpus = $sockets * $cores;
3585
3586     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3587
3588     my $allowed_vcpus = $cpuinfo->{cpus};
3589
3590     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3591         if ($allowed_vcpus < $maxcpus);
3592
3593     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3594
3595         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3596         for (my $i = 2; $i <= $vcpus; $i++)  {
3597             my $cpustr = print_cpu_device($conf,$i);
3598             push @$cmd, '-device', $cpustr;
3599         }
3600
3601     } else {
3602
3603         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3604     }
3605     push @$cmd, '-nodefaults';
3606
3607     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3608
3609     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3610
3611     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3612
3613     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3614         push @$devices, '-device', print_vga_device(
3615             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3616         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3617         push @$cmd,  '-vnc', "unix:$socket,password=on";
3618     } else {
3619         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3620         push @$cmd, '-nographic';
3621     }
3622
3623     # time drift fix
3624     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3625     my $useLocaltime = $conf->{localtime};
3626
3627     if ($winversion >= 5) { # windows
3628         $useLocaltime = 1 if !defined($conf->{localtime});
3629
3630         # use time drift fix when acpi is enabled
3631         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3632             $tdf = 1 if !defined($conf->{tdf});
3633         }
3634     }
3635
3636     if ($winversion >= 6) {
3637         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3638         push @$cmd, '-no-hpet';
3639     }
3640
3641     push @$rtcFlags, 'driftfix=slew' if $tdf;
3642
3643     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3644         push @$rtcFlags, "base=$conf->{startdate}";
3645     } elsif ($useLocaltime) {
3646         push @$rtcFlags, 'base=localtime';
3647     }
3648
3649     if ($forcecpu) {
3650         push @$cmd, '-cpu', $forcecpu;
3651     } else {
3652         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3653     }
3654
3655     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3656
3657     push @$cmd, '-S' if $conf->{freeze};
3658
3659     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3660
3661     my $guest_agent = parse_guest_agent($conf);
3662
3663     if ($guest_agent->{enabled}) {
3664         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3665         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3666
3667         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3668             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3669             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3670             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3671         } elsif ($guest_agent->{type} eq 'isa') {
3672             push @$devices, '-device', "isa-serial,chardev=qga0";
3673         }
3674     }
3675
3676     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3677     if ($rng && $version_guard->(4, 1, 2)) {
3678         check_rng_source($rng->{source});
3679
3680         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3681         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3682         my $limiter_str = "";
3683         if ($max_bytes) {
3684             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3685         }
3686
3687         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3688         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3689         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3690     }
3691
3692     my $spice_port;
3693
3694     if ($qxlnum) {
3695         if ($qxlnum > 1) {
3696             if ($winversion){
3697                 for (my $i = 1; $i < $qxlnum; $i++){
3698                     push @$devices, '-device', print_vga_device(
3699                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3700                 }
3701             } else {
3702                 # assume other OS works like Linux
3703                 my ($ram, $vram) = ("134217728", "67108864");
3704                 if ($vga->{memory}) {
3705                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3706                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3707                 }
3708                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3709                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3710             }
3711         }
3712
3713         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3714
3715         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3716         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3717         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3718
3719         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3720         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3721         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3722
3723         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3724         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3725
3726         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3727         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3728         if ($spice_enhancement->{foldersharing}) {
3729             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3730             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3731         }
3732
3733         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3734         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3735             if $spice_enhancement->{videostreaming};
3736
3737         push @$devices, '-spice', "$spice_opts";
3738     }
3739
3740     # enable balloon by default, unless explicitly disabled
3741     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3742         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3743         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3744     }
3745
3746     if ($conf->{watchdog}) {
3747         my $wdopts = parse_watchdog($conf->{watchdog});
3748         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3749         my $watchdog = $wdopts->{model} || 'i6300esb';
3750         push @$devices, '-device', "$watchdog$pciaddr";
3751         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3752     }
3753
3754     my $vollist = [];
3755     my $scsicontroller = {};
3756     my $ahcicontroller = {};
3757     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3758
3759     # Add iscsi initiator name if available
3760     if (my $initiator = get_initiator_name()) {
3761         push @$devices, '-iscsi', "initiator-name=$initiator";
3762     }
3763
3764     PVE::QemuConfig->foreach_volume($conf, sub {
3765         my ($ds, $drive) = @_;
3766
3767         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3768             check_volume_storage_type($storecfg, $drive->{file});
3769             push @$vollist, $drive->{file};
3770         }
3771
3772         # ignore efidisk here, already added in bios/fw handling code above
3773         return if $drive->{interface} eq 'efidisk';
3774         # similar for TPM
3775         return if $drive->{interface} eq 'tpmstate';
3776
3777         $use_virtio = 1 if $ds =~ m/^virtio/;
3778
3779         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3780
3781         if ($drive->{interface} eq 'virtio'){
3782            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3783         }
3784
3785         if ($drive->{interface} eq 'scsi') {
3786
3787             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3788
3789             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3790                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3791
3792             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3793             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3794
3795             my $iothread = '';
3796             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3797                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3798                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3799             } elsif ($drive->{iothread}) {
3800                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3801             }
3802
3803             my $queues = '';
3804             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3805                 $queues = ",num_queues=$drive->{queues}";
3806             }
3807
3808             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3809                 if !$scsicontroller->{$controller};
3810             $scsicontroller->{$controller}=1;
3811         }
3812
3813         if ($drive->{interface} eq 'sata') {
3814             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3815             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3816             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3817                 if !$ahcicontroller->{$controller};
3818             $ahcicontroller->{$controller}=1;
3819         }
3820
3821         my $pbs_conf = $pbs_backing->{$ds};
3822         my $pbs_name = undef;
3823         if ($pbs_conf) {
3824             $pbs_name = "drive-$ds-pbs";
3825             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3826         }
3827
3828         my $drive_cmd = print_drive_commandline_full(
3829             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3830
3831         # extra protection for templates, but SATA and IDE don't support it..
3832         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3833
3834         push @$devices, '-drive',$drive_cmd;
3835         push @$devices, '-device', print_drivedevice_full(
3836             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3837     });
3838
3839     for (my $i = 0; $i < $MAX_NETS; $i++) {
3840         my $netname = "net$i";
3841
3842         next if !$conf->{$netname};
3843         my $d = parse_net($conf->{$netname});
3844         next if !$d;
3845
3846         $use_virtio = 1 if $d->{model} eq 'virtio';
3847
3848         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3849
3850         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3851         push @$devices, '-netdev', $netdevfull;
3852
3853         my $netdevicefull = print_netdevice_full(
3854             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3855
3856         push @$devices, '-device', $netdevicefull;
3857     }
3858
3859     if ($conf->{ivshmem}) {
3860         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3861
3862         my $bus;
3863         if ($q35) {
3864             $bus = print_pcie_addr("ivshmem");
3865         } else {
3866             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3867         }
3868
3869         my $ivshmem_name = $ivshmem->{name} // $vmid;
3870         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3871
3872         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3873         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3874             .",size=$ivshmem->{size}M";
3875     }
3876
3877     # pci.4 is nested in pci.1
3878     $bridges->{1} = 1 if $bridges->{4};
3879
3880     if (!$q35) {
3881         # add pci bridges
3882         if (min_version($machine_version, 2, 3)) {
3883            $bridges->{1} = 1;
3884            $bridges->{2} = 1;
3885         }
3886
3887         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3888
3889     }
3890
3891     for my $k (sort {$b cmp $a} keys %$bridges) {
3892         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3893
3894         my $k_name = $k;
3895         if ($k == 2 && $legacy_igd) {
3896             $k_name = "$k-igd";
3897         }
3898         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3899
3900         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3901         if ($q35) {
3902             # add after -readconfig pve-q35.cfg
3903             splice @$devices, 2, 0, '-device', $devstr;
3904         } else {
3905             unshift @$devices, '-device', $devstr if $k > 0;
3906         }
3907     }
3908
3909     if (!$kvm) {
3910         push @$machineFlags, 'accel=tcg';
3911     }
3912
3913     my $machine_type_min = $machine_type;
3914     if ($add_pve_version) {
3915         $machine_type_min =~ s/\+pve\d+$//;
3916         $machine_type_min .= "+pve$required_pve_version";
3917     }
3918     push @$machineFlags, "type=${machine_type_min}";
3919
3920     push @$cmd, @$devices;
3921     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3922     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3923     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3924
3925     if (my $vmstate = $conf->{vmstate}) {
3926         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3927         push @$vollist, $vmstate;
3928         push @$cmd, '-loadstate', $statepath;
3929         print "activating and using '$vmstate' as vmstate\n";
3930     }
3931
3932     if (PVE::QemuConfig->is_template($conf)) {
3933         # needed to workaround base volumes being read-only
3934         push @$cmd, '-snapshot';
3935     }
3936
3937     # add custom args
3938     if ($conf->{args}) {
3939         my $aa = PVE::Tools::split_args($conf->{args});
3940         push @$cmd, @$aa;
3941     }
3942
3943     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3944 }
3945
3946 sub check_rng_source {
3947     my ($source) = @_;
3948
3949     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3950     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3951         if ! -e $source;
3952
3953     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3954     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3955         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3956         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3957         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3958             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3959             ." to the host.\n";
3960     }
3961 }
3962
3963 sub spice_port {
3964     my ($vmid) = @_;
3965
3966     my $res = mon_cmd($vmid, 'query-spice');
3967
3968     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3969 }
3970
3971 sub vm_devices_list {
3972     my ($vmid) = @_;
3973
3974     my $res = mon_cmd($vmid, 'query-pci');
3975     my $devices_to_check = [];
3976     my $devices = {};
3977     foreach my $pcibus (@$res) {
3978         push @$devices_to_check, @{$pcibus->{devices}},
3979     }
3980
3981     while (@$devices_to_check) {
3982         my $to_check = [];
3983         for my $d (@$devices_to_check) {
3984             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3985             next if !$d->{'pci_bridge'};
3986
3987             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3988             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3989         }
3990         $devices_to_check = $to_check;
3991     }
3992
3993     my $resblock = mon_cmd($vmid, 'query-block');
3994     foreach my $block (@$resblock) {
3995         if($block->{device} =~ m/^drive-(\S+)/){
3996                 $devices->{$1} = 1;
3997         }
3998     }
3999
4000     my $resmice = mon_cmd($vmid, 'query-mice');
4001     foreach my $mice (@$resmice) {
4002         if ($mice->{name} eq 'QEMU HID Tablet') {
4003             $devices->{tablet} = 1;
4004             last;
4005         }
4006     }
4007
4008     # for usb devices there is no query-usb
4009     # but we can iterate over the entries in
4010     # qom-list path=/machine/peripheral
4011     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4012     foreach my $per (@$resperipheral) {
4013         if ($per->{name} =~ m/^usb\d+$/) {
4014             $devices->{$per->{name}} = 1;
4015         }
4016     }
4017
4018     return $devices;
4019 }
4020
4021 sub vm_deviceplug {
4022     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4023
4024     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4025
4026     my $devices_list = vm_devices_list($vmid);
4027     return 1 if defined($devices_list->{$deviceid});
4028
4029     # add PCI bridge if we need it for the device
4030     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4031
4032     if ($deviceid eq 'tablet') {
4033
4034         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4035
4036     } elsif ($deviceid eq 'keyboard') {
4037
4038         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4039
4040     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4041
4042         die "usb hotplug currently not reliable\n";
4043         # since we can't reliably hot unplug all added usb devices and usb
4044         # passthrough breaks live migration we disable usb hotplugging for now
4045         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4046
4047     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4048
4049         qemu_iothread_add($vmid, $deviceid, $device);
4050
4051         qemu_driveadd($storecfg, $vmid, $device);
4052         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4053
4054         qemu_deviceadd($vmid, $devicefull);
4055         eval { qemu_deviceaddverify($vmid, $deviceid); };
4056         if (my $err = $@) {
4057             eval { qemu_drivedel($vmid, $deviceid); };
4058             warn $@ if $@;
4059             die $err;
4060         }
4061
4062     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4063
4064
4065         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4066         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4067         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4068
4069         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4070
4071         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4072             qemu_iothread_add($vmid, $deviceid, $device);
4073             $devicefull .= ",iothread=iothread-$deviceid";
4074         }
4075
4076         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4077             $devicefull .= ",num_queues=$device->{queues}";
4078         }
4079
4080         qemu_deviceadd($vmid, $devicefull);
4081         qemu_deviceaddverify($vmid, $deviceid);
4082
4083     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4084
4085         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4086         qemu_driveadd($storecfg, $vmid, $device);
4087
4088         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4089         eval { qemu_deviceadd($vmid, $devicefull); };
4090         if (my $err = $@) {
4091             eval { qemu_drivedel($vmid, $deviceid); };
4092             warn $@ if $@;
4093             die $err;
4094         }
4095
4096     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4097
4098         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4099
4100         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4101         my $use_old_bios_files = undef;
4102         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4103
4104         my $netdevicefull = print_netdevice_full(
4105             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4106         qemu_deviceadd($vmid, $netdevicefull);
4107         eval {
4108             qemu_deviceaddverify($vmid, $deviceid);
4109             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4110         };
4111         if (my $err = $@) {
4112             eval { qemu_netdevdel($vmid, $deviceid); };
4113             warn $@ if $@;
4114             die $err;
4115         }
4116
4117     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4118
4119         my $bridgeid = $2;
4120         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4121         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4122
4123         qemu_deviceadd($vmid, $devicefull);
4124         qemu_deviceaddverify($vmid, $deviceid);
4125
4126     } else {
4127         die "can't hotplug device '$deviceid'\n";
4128     }
4129
4130     return 1;
4131 }
4132
4133 # fixme: this should raise exceptions on error!
4134 sub vm_deviceunplug {
4135     my ($vmid, $conf, $deviceid) = @_;
4136
4137     my $devices_list = vm_devices_list($vmid);
4138     return 1 if !defined($devices_list->{$deviceid});
4139
4140     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4141     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4142
4143     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4144         qemu_devicedel($vmid, $deviceid);
4145     } elsif ($deviceid =~ m/^usb\d+$/) {
4146         die "usb hotplug currently not reliable\n";
4147         # when unplugging usb devices this way, there may be remaining usb
4148         # controllers/hubs so we disable it for now
4149         #qemu_devicedel($vmid, $deviceid);
4150         #qemu_devicedelverify($vmid, $deviceid);
4151     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4152         my $device = parse_drive($deviceid, $conf->{$deviceid});
4153
4154         qemu_devicedel($vmid, $deviceid);
4155         qemu_devicedelverify($vmid, $deviceid);
4156         qemu_drivedel($vmid, $deviceid);
4157         qemu_iothread_del($vmid, $deviceid, $device);
4158     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4159         qemu_devicedel($vmid, $deviceid);
4160         qemu_devicedelverify($vmid, $deviceid);
4161     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4162         my $device = parse_drive($deviceid, $conf->{$deviceid});
4163
4164         qemu_devicedel($vmid, $deviceid);
4165         qemu_drivedel($vmid, $deviceid);
4166         qemu_deletescsihw($conf, $vmid, $deviceid);
4167
4168         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4169             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4170     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4171         qemu_devicedel($vmid, $deviceid);
4172         qemu_devicedelverify($vmid, $deviceid);
4173         qemu_netdevdel($vmid, $deviceid);
4174     } else {
4175         die "can't unplug device '$deviceid'\n";
4176     }
4177
4178     return 1;
4179 }
4180
4181 sub qemu_deviceadd {
4182     my ($vmid, $devicefull) = @_;
4183
4184     $devicefull = "driver=".$devicefull;
4185     my %options =  split(/[=,]/, $devicefull);
4186
4187     mon_cmd($vmid, "device_add" , %options);
4188 }
4189
4190 sub qemu_devicedel {
4191     my ($vmid, $deviceid) = @_;
4192
4193     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4194 }
4195
4196 sub qemu_iothread_add {
4197     my ($vmid, $deviceid, $device) = @_;
4198
4199     if ($device->{iothread}) {
4200         my $iothreads = vm_iothreads_list($vmid);
4201         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4202     }
4203 }
4204
4205 sub qemu_iothread_del {
4206     my ($vmid, $deviceid, $device) = @_;
4207
4208     if ($device->{iothread}) {
4209         my $iothreads = vm_iothreads_list($vmid);
4210         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4211     }
4212 }
4213
4214 sub qemu_objectadd {
4215     my ($vmid, $objectid, $qomtype) = @_;
4216
4217     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4218
4219     return 1;
4220 }
4221
4222 sub qemu_objectdel {
4223     my ($vmid, $objectid) = @_;
4224
4225     mon_cmd($vmid, "object-del", id => $objectid);
4226
4227     return 1;
4228 }
4229
4230 sub qemu_driveadd {
4231     my ($storecfg, $vmid, $device) = @_;
4232
4233     my $kvmver = get_running_qemu_version($vmid);
4234     my $io_uring = min_version($kvmver, 6, 0);
4235     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4236     $drive =~ s/\\/\\\\/g;
4237     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4238
4239     # If the command succeeds qemu prints: "OK"
4240     return 1 if $ret =~ m/OK/s;
4241
4242     die "adding drive failed: $ret\n";
4243 }
4244
4245 sub qemu_drivedel {
4246     my ($vmid, $deviceid) = @_;
4247
4248     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4249     $ret =~ s/^\s+//;
4250
4251     return 1 if $ret eq "";
4252
4253     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4254     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4255
4256     die "deleting drive $deviceid failed : $ret\n";
4257 }
4258
4259 sub qemu_deviceaddverify {
4260     my ($vmid, $deviceid) = @_;
4261
4262     for (my $i = 0; $i <= 5; $i++) {
4263          my $devices_list = vm_devices_list($vmid);
4264          return 1 if defined($devices_list->{$deviceid});
4265          sleep 1;
4266     }
4267
4268     die "error on hotplug device '$deviceid'\n";
4269 }
4270
4271
4272 sub qemu_devicedelverify {
4273     my ($vmid, $deviceid) = @_;
4274
4275     # need to verify that the device is correctly removed as device_del
4276     # is async and empty return is not reliable
4277
4278     for (my $i = 0; $i <= 5; $i++) {
4279          my $devices_list = vm_devices_list($vmid);
4280          return 1 if !defined($devices_list->{$deviceid});
4281          sleep 1;
4282     }
4283
4284     die "error on hot-unplugging device '$deviceid'\n";
4285 }
4286
4287 sub qemu_findorcreatescsihw {
4288     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4289
4290     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4291
4292     my $scsihwid="$controller_prefix$controller";
4293     my $devices_list = vm_devices_list($vmid);
4294
4295     if (!defined($devices_list->{$scsihwid})) {
4296         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4297     }
4298
4299     return 1;
4300 }
4301
4302 sub qemu_deletescsihw {
4303     my ($conf, $vmid, $opt) = @_;
4304
4305     my $device = parse_drive($opt, $conf->{$opt});
4306
4307     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4308         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4309         return 1;
4310     }
4311
4312     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4313
4314     my $devices_list = vm_devices_list($vmid);
4315     foreach my $opt (keys %{$devices_list}) {
4316         if (is_valid_drivename($opt)) {
4317             my $drive = parse_drive($opt, $conf->{$opt});
4318             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4319                 return 1;
4320             }
4321         }
4322     }
4323
4324     my $scsihwid="scsihw$controller";
4325
4326     vm_deviceunplug($vmid, $conf, $scsihwid);
4327
4328     return 1;
4329 }
4330
4331 sub qemu_add_pci_bridge {
4332     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4333
4334     my $bridges = {};
4335
4336     my $bridgeid;
4337
4338     print_pci_addr($device, $bridges, $arch, $machine_type);
4339
4340     while (my ($k, $v) = each %$bridges) {
4341         $bridgeid = $k;
4342     }
4343     return 1 if !defined($bridgeid) || $bridgeid < 1;
4344
4345     my $bridge = "pci.$bridgeid";
4346     my $devices_list = vm_devices_list($vmid);
4347
4348     if (!defined($devices_list->{$bridge})) {
4349         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4350     }
4351
4352     return 1;
4353 }
4354
4355 sub qemu_set_link_status {
4356     my ($vmid, $device, $up) = @_;
4357
4358     mon_cmd($vmid, "set_link", name => $device,
4359                up => $up ? JSON::true : JSON::false);
4360 }
4361
4362 sub qemu_netdevadd {
4363     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4364
4365     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4366     my %options =  split(/[=,]/, $netdev);
4367
4368     if (defined(my $vhost = $options{vhost})) {
4369         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4370     }
4371
4372     if (defined(my $queues = $options{queues})) {
4373         $options{queues} = $queues + 0;
4374     }
4375
4376     mon_cmd($vmid, "netdev_add",  %options);
4377     return 1;
4378 }
4379
4380 sub qemu_netdevdel {
4381     my ($vmid, $deviceid) = @_;
4382
4383     mon_cmd($vmid, "netdev_del", id => $deviceid);
4384 }
4385
4386 sub qemu_usb_hotplug {
4387     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4388
4389     return if !$device;
4390
4391     # remove the old one first
4392     vm_deviceunplug($vmid, $conf, $deviceid);
4393
4394     # check if xhci controller is necessary and available
4395     if ($device->{usb3}) {
4396
4397         my $devicelist = vm_devices_list($vmid);
4398
4399         if (!$devicelist->{xhci}) {
4400             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4401             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4402         }
4403     }
4404     my $d = parse_usb_device($device->{host});
4405     $d->{usb3} = $device->{usb3};
4406
4407     # add the new one
4408     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4409 }
4410
4411 sub qemu_cpu_hotplug {
4412     my ($vmid, $conf, $vcpus) = @_;
4413
4414     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4415
4416     my $sockets = 1;
4417     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4418     $sockets = $conf->{sockets} if  $conf->{sockets};
4419     my $cores = $conf->{cores} || 1;
4420     my $maxcpus = $sockets * $cores;
4421
4422     $vcpus = $maxcpus if !$vcpus;
4423
4424     die "you can't add more vcpus than maxcpus\n"
4425         if $vcpus > $maxcpus;
4426
4427     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4428
4429     if ($vcpus < $currentvcpus) {
4430
4431         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4432
4433             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4434                 qemu_devicedel($vmid, "cpu$i");
4435                 my $retry = 0;
4436                 my $currentrunningvcpus = undef;
4437                 while (1) {
4438                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4439                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4440                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4441                     $retry++;
4442                     sleep 1;
4443                 }
4444                 #update conf after each succesfull cpu unplug
4445                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4446                 PVE::QemuConfig->write_config($vmid, $conf);
4447             }
4448         } else {
4449             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4450         }
4451
4452         return;
4453     }
4454
4455     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4456     die "vcpus in running vm does not match its configuration\n"
4457         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4458
4459     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4460
4461         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4462             my $cpustr = print_cpu_device($conf, $i);
4463             qemu_deviceadd($vmid, $cpustr);
4464
4465             my $retry = 0;
4466             my $currentrunningvcpus = undef;
4467             while (1) {
4468                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4469                 last if scalar(@{$currentrunningvcpus}) == $i;
4470                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4471                 sleep 1;
4472                 $retry++;
4473             }
4474             #update conf after each succesfull cpu hotplug
4475             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4476             PVE::QemuConfig->write_config($vmid, $conf);
4477         }
4478     } else {
4479
4480         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4481             mon_cmd($vmid, "cpu-add", id => int($i));
4482         }
4483     }
4484 }
4485
4486 sub qemu_block_set_io_throttle {
4487     my ($vmid, $deviceid,
4488         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4489         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4490         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4491         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4492
4493     return if !check_running($vmid) ;
4494
4495     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4496         bps => int($bps),
4497         bps_rd => int($bps_rd),
4498         bps_wr => int($bps_wr),
4499         iops => int($iops),
4500         iops_rd => int($iops_rd),
4501         iops_wr => int($iops_wr),
4502         bps_max => int($bps_max),
4503         bps_rd_max => int($bps_rd_max),
4504         bps_wr_max => int($bps_wr_max),
4505         iops_max => int($iops_max),
4506         iops_rd_max => int($iops_rd_max),
4507         iops_wr_max => int($iops_wr_max),
4508         bps_max_length => int($bps_max_length),
4509         bps_rd_max_length => int($bps_rd_max_length),
4510         bps_wr_max_length => int($bps_wr_max_length),
4511         iops_max_length => int($iops_max_length),
4512         iops_rd_max_length => int($iops_rd_max_length),
4513         iops_wr_max_length => int($iops_wr_max_length),
4514     );
4515
4516 }
4517
4518 sub qemu_block_resize {
4519     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4520
4521     my $running = check_running($vmid);
4522
4523     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4524
4525     return if !$running;
4526
4527     my $padding = (1024 - $size % 1024) % 1024;
4528     $size = $size + $padding;
4529
4530     mon_cmd(
4531         $vmid,
4532         "block_resize",
4533         device => $deviceid,
4534         size => int($size),
4535         timeout => 60,
4536     );
4537 }
4538
4539 sub qemu_volume_snapshot {
4540     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4541
4542     my $running = check_running($vmid);
4543
4544     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4545         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4546     } else {
4547         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4548     }
4549 }
4550
4551 sub qemu_volume_snapshot_delete {
4552     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4553
4554     my $running = check_running($vmid);
4555
4556     if($running) {
4557
4558         $running = undef;
4559         my $conf = PVE::QemuConfig->load_config($vmid);
4560         PVE::QemuConfig->foreach_volume($conf, sub {
4561             my ($ds, $drive) = @_;
4562             $running = 1 if $drive->{file} eq $volid;
4563         });
4564     }
4565
4566     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4567         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4568     } else {
4569         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4570     }
4571 }
4572
4573 sub set_migration_caps {
4574     my ($vmid, $savevm) = @_;
4575
4576     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4577
4578     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4579     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4580
4581     my $cap_ref = [];
4582
4583     my $enabled_cap = {
4584         "auto-converge" => 1,
4585         "xbzrle" => 1,
4586         "x-rdma-pin-all" => 0,
4587         "zero-blocks" => 0,
4588         "compress" => 0,
4589         "dirty-bitmaps" => $dirty_bitmaps,
4590     };
4591
4592     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4593
4594     for my $supported_capability (@$supported_capabilities) {
4595         push @$cap_ref, {
4596             capability => $supported_capability->{capability},
4597             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4598         };
4599     }
4600
4601     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4602 }
4603
4604 sub foreach_volid {
4605     my ($conf, $func, @param) = @_;
4606
4607     my $volhash = {};
4608
4609     my $test_volid = sub {
4610         my ($key, $drive, $snapname) = @_;
4611
4612         my $volid = $drive->{file};
4613         return if !$volid;
4614
4615         $volhash->{$volid}->{cdrom} //= 1;
4616         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4617
4618         my $replicate = $drive->{replicate} // 1;
4619         $volhash->{$volid}->{replicate} //= 0;
4620         $volhash->{$volid}->{replicate} = 1 if $replicate;
4621
4622         $volhash->{$volid}->{shared} //= 0;
4623         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4624
4625         $volhash->{$volid}->{referenced_in_config} //= 0;
4626         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4627
4628         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4629             if defined($snapname);
4630
4631         my $size = $drive->{size};
4632         $volhash->{$volid}->{size} //= $size if $size;
4633
4634         $volhash->{$volid}->{is_vmstate} //= 0;
4635         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4636
4637         $volhash->{$volid}->{is_tpmstate} //= 0;
4638         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4639
4640         $volhash->{$volid}->{is_unused} //= 0;
4641         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4642
4643         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4644     };
4645
4646     my $include_opts = {
4647         extra_keys => ['vmstate'],
4648         include_unused => 1,
4649     };
4650
4651     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4652     foreach my $snapname (keys %{$conf->{snapshots}}) {
4653         my $snap = $conf->{snapshots}->{$snapname};
4654         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4655     }
4656
4657     foreach my $volid (keys %$volhash) {
4658         &$func($volid, $volhash->{$volid}, @param);
4659     }
4660 }
4661
4662 my $fast_plug_option = {
4663     'lock' => 1,
4664     'name' => 1,
4665     'onboot' => 1,
4666     'shares' => 1,
4667     'startup' => 1,
4668     'description' => 1,
4669     'protection' => 1,
4670     'vmstatestorage' => 1,
4671     'hookscript' => 1,
4672     'tags' => 1,
4673 };
4674
4675 # hotplug changes in [PENDING]
4676 # $selection hash can be used to only apply specified options, for
4677 # example: { cores => 1 } (only apply changed 'cores')
4678 # $errors ref is used to return error messages
4679 sub vmconfig_hotplug_pending {
4680     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4681
4682     my $defaults = load_defaults();
4683     my $arch = get_vm_arch($conf);
4684     my $machine_type = get_vm_machine($conf, undef, $arch);
4685
4686     # commit values which do not have any impact on running VM first
4687     # Note: those option cannot raise errors, we we do not care about
4688     # $selection and always apply them.
4689
4690     my $add_error = sub {
4691         my ($opt, $msg) = @_;
4692         $errors->{$opt} = "hotplug problem - $msg";
4693     };
4694
4695     my $changes = 0;
4696     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4697         if ($fast_plug_option->{$opt}) {
4698             $conf->{$opt} = $conf->{pending}->{$opt};
4699             delete $conf->{pending}->{$opt};
4700             $changes = 1;
4701         }
4702     }
4703
4704     if ($changes) {
4705         PVE::QemuConfig->write_config($vmid, $conf);
4706     }
4707
4708     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4709
4710     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4711     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4712     foreach my $opt (sort keys %$pending_delete_hash) {
4713         next if $selection && !$selection->{$opt};
4714         my $force = $pending_delete_hash->{$opt}->{force};
4715         eval {
4716             if ($opt eq 'hotplug') {
4717                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4718             } elsif ($opt eq 'tablet') {
4719                 die "skip\n" if !$hotplug_features->{usb};
4720                 if ($defaults->{tablet}) {
4721                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4722                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4723                         if $arch eq 'aarch64';
4724                 } else {
4725                     vm_deviceunplug($vmid, $conf, 'tablet');
4726                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4727                 }
4728             } elsif ($opt =~ m/^usb\d+/) {
4729                 die "skip\n";
4730                 # since we cannot reliably hot unplug usb devices we are disabling it
4731                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4732                 #vm_deviceunplug($vmid, $conf, $opt);
4733             } elsif ($opt eq 'vcpus') {
4734                 die "skip\n" if !$hotplug_features->{cpu};
4735                 qemu_cpu_hotplug($vmid, $conf, undef);
4736             } elsif ($opt eq 'balloon') {
4737                 # enable balloon device is not hotpluggable
4738                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4739                 # here we reset the ballooning value to memory
4740                 my $balloon = $conf->{memory} || $defaults->{memory};
4741                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4742             } elsif ($fast_plug_option->{$opt}) {
4743                 # do nothing
4744             } elsif ($opt =~ m/^net(\d+)$/) {
4745                 die "skip\n" if !$hotplug_features->{network};
4746                 vm_deviceunplug($vmid, $conf, $opt);
4747             } elsif (is_valid_drivename($opt)) {
4748                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4749                 vm_deviceunplug($vmid, $conf, $opt);
4750                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4751             } elsif ($opt =~ m/^memory$/) {
4752                 die "skip\n" if !$hotplug_features->{memory};
4753                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4754             } elsif ($opt eq 'cpuunits') {
4755                 $cgroup->change_cpu_shares(undef, 1024);
4756             } elsif ($opt eq 'cpulimit') {
4757                 $cgroup->change_cpu_quota(-1, 100000);
4758             } else {
4759                 die "skip\n";
4760             }
4761         };
4762         if (my $err = $@) {
4763             &$add_error($opt, $err) if $err ne "skip\n";
4764         } else {
4765             delete $conf->{$opt};
4766             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4767         }
4768     }
4769
4770     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4771     $apply_pending_cloudinit = sub {
4772         return if $apply_pending_cloudinit_done; # once is enough
4773         $apply_pending_cloudinit_done = 1; # once is enough
4774
4775         my ($key, $value) = @_;
4776
4777         my @cloudinit_opts = keys %$confdesc_cloudinit;
4778         foreach my $opt (keys %{$conf->{pending}}) {
4779             next if !grep { $_ eq $opt } @cloudinit_opts;
4780             $conf->{$opt} = delete $conf->{pending}->{$opt};
4781         }
4782
4783         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4784         foreach my $opt (sort keys %$pending_delete_hash) {
4785             next if !grep { $_ eq $opt } @cloudinit_opts;
4786             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4787             delete $conf->{$opt};
4788         }
4789
4790         my $new_conf = { %$conf };
4791         $new_conf->{$key} = $value;
4792         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4793     };
4794
4795     foreach my $opt (keys %{$conf->{pending}}) {
4796         next if $selection && !$selection->{$opt};
4797         my $value = $conf->{pending}->{$opt};
4798         eval {
4799             if ($opt eq 'hotplug') {
4800                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4801             } elsif ($opt eq 'tablet') {
4802                 die "skip\n" if !$hotplug_features->{usb};
4803                 if ($value == 1) {
4804                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4805                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4806                         if $arch eq 'aarch64';
4807                 } elsif ($value == 0) {
4808                     vm_deviceunplug($vmid, $conf, 'tablet');
4809                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4810                 }
4811             } elsif ($opt =~ m/^usb\d+$/) {
4812                 die "skip\n";
4813                 # since we cannot reliably hot unplug usb devices we disable it for now
4814                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4815                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4816                 #die "skip\n" if !$d;
4817                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4818             } elsif ($opt eq 'vcpus') {
4819                 die "skip\n" if !$hotplug_features->{cpu};
4820                 qemu_cpu_hotplug($vmid, $conf, $value);
4821             } elsif ($opt eq 'balloon') {
4822                 # enable/disable balloning device is not hotpluggable
4823                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4824                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4825                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4826
4827                 # allow manual ballooning if shares is set to zero
4828                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4829                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4830                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4831                 }
4832             } elsif ($opt =~ m/^net(\d+)$/) {
4833                 # some changes can be done without hotplug
4834                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4835                                     $vmid, $opt, $value, $arch, $machine_type);
4836             } elsif (is_valid_drivename($opt)) {
4837                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4838                 # some changes can be done without hotplug
4839                 my $drive = parse_drive($opt, $value);
4840                 if (drive_is_cloudinit($drive)) {
4841                     &$apply_pending_cloudinit($opt, $value);
4842                 }
4843                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4844                                      $vmid, $opt, $value, $arch, $machine_type);
4845             } elsif ($opt =~ m/^memory$/) { #dimms
4846                 die "skip\n" if !$hotplug_features->{memory};
4847                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4848             } elsif ($opt eq 'cpuunits') {
4849                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4850             } elsif ($opt eq 'cpulimit') {
4851                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4852                 $cgroup->change_cpu_quota($cpulimit, 100000);
4853             } else {
4854                 die "skip\n";  # skip non-hot-pluggable options
4855             }
4856         };
4857         if (my $err = $@) {
4858             &$add_error($opt, $err) if $err ne "skip\n";
4859         } else {
4860             $conf->{$opt} = $value;
4861             delete $conf->{pending}->{$opt};
4862         }
4863     }
4864
4865     PVE::QemuConfig->write_config($vmid, $conf);
4866 }
4867
4868 sub try_deallocate_drive {
4869     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4870
4871     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4872         my $volid = $drive->{file};
4873         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4874             my $sid = PVE::Storage::parse_volume_id($volid);
4875             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4876
4877             # check if the disk is really unused
4878             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4879                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4880             PVE::Storage::vdisk_free($storecfg, $volid);
4881             return 1;
4882         } else {
4883             # If vm is not owner of this disk remove from config
4884             return 1;
4885         }
4886     }
4887
4888     return;
4889 }
4890
4891 sub vmconfig_delete_or_detach_drive {
4892     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4893
4894     my $drive = parse_drive($opt, $conf->{$opt});
4895
4896     my $rpcenv = PVE::RPCEnvironment::get();
4897     my $authuser = $rpcenv->get_user();
4898
4899     if ($force) {
4900         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4901         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4902     } else {
4903         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4904     }
4905 }
4906
4907
4908
4909 sub vmconfig_apply_pending {
4910     my ($vmid, $conf, $storecfg, $errors) = @_;
4911
4912     my $add_apply_error = sub {
4913         my ($opt, $msg) = @_;
4914         my $err_msg = "unable to apply pending change $opt : $msg";
4915         $errors->{$opt} = $err_msg;
4916         warn $err_msg;
4917     };
4918
4919     # cold plug
4920
4921     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4922     foreach my $opt (sort keys %$pending_delete_hash) {
4923         my $force = $pending_delete_hash->{$opt}->{force};
4924         eval {
4925             if ($opt =~ m/^unused/) {
4926                 die "internal error";
4927             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4928                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4929             }
4930         };
4931         if (my $err = $@) {
4932             $add_apply_error->($opt, $err);
4933         } else {
4934             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4935             delete $conf->{$opt};
4936         }
4937     }
4938
4939     PVE::QemuConfig->cleanup_pending($conf);
4940
4941     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4942         next if $opt eq 'delete'; # just to be sure
4943         eval {
4944             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4945                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4946             }
4947         };
4948         if (my $err = $@) {
4949             $add_apply_error->($opt, $err);
4950         } else {
4951             $conf->{$opt} = delete $conf->{pending}->{$opt};
4952         }
4953     }
4954
4955     # write all changes at once to avoid unnecessary i/o
4956     PVE::QemuConfig->write_config($vmid, $conf);
4957 }
4958
4959 sub vmconfig_update_net {
4960     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4961
4962     my $newnet = parse_net($value);
4963
4964     if ($conf->{$opt}) {
4965         my $oldnet = parse_net($conf->{$opt});
4966
4967         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4968             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4969             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4970             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4971
4972             # for non online change, we try to hot-unplug
4973             die "skip\n" if !$hotplug;
4974             vm_deviceunplug($vmid, $conf, $opt);
4975         } else {
4976
4977             die "internal error" if $opt !~ m/net(\d+)/;
4978             my $iface = "tap${vmid}i$1";
4979
4980             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4981                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4982                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4983                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4984                 PVE::Network::tap_unplug($iface);
4985
4986                 if ($have_sdn) {
4987                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4988                 } else {
4989                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4990                 }
4991             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4992                 # Rate can be applied on its own but any change above needs to
4993                 # include the rate in tap_plug since OVS resets everything.
4994                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4995             }
4996
4997             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4998                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4999             }
5000
5001             return 1;
5002         }
5003     }
5004
5005     if ($hotplug) {
5006         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5007     } else {
5008         die "skip\n";
5009     }
5010 }
5011
5012 sub vmconfig_update_disk {
5013     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5014
5015     my $drive = parse_drive($opt, $value);
5016
5017     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5018         my $media = $drive->{media} || 'disk';
5019         my $oldmedia = $old_drive->{media} || 'disk';
5020         die "unable to change media type\n" if $media ne $oldmedia;
5021
5022         if (!drive_is_cdrom($old_drive)) {
5023
5024             if ($drive->{file} ne $old_drive->{file}) {
5025
5026                 die "skip\n" if !$hotplug;
5027
5028                 # unplug and register as unused
5029                 vm_deviceunplug($vmid, $conf, $opt);
5030                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5031
5032             } else {
5033                 # update existing disk
5034
5035                 # skip non hotpluggable value
5036                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5037                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5038                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5039                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5040                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5041                     die "skip\n";
5042                 }
5043
5044                 # apply throttle
5045                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5046                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5047                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5048                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5049                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5050                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5051                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5052                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5053                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5054                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5055                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5056                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5057                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5058                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5059                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5060                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5061                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5062                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5063
5064                     qemu_block_set_io_throttle(
5065                         $vmid,"drive-$opt",
5066                         ($drive->{mbps} || 0)*1024*1024,
5067                         ($drive->{mbps_rd} || 0)*1024*1024,
5068                         ($drive->{mbps_wr} || 0)*1024*1024,
5069                          $drive->{iops} || 0,
5070                          $drive->{iops_rd} || 0,
5071                          $drive->{iops_wr} || 0,
5072                         ($drive->{mbps_max} || 0)*1024*1024,
5073                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5074                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5075                          $drive->{iops_max} || 0,
5076                          $drive->{iops_rd_max} || 0,
5077                          $drive->{iops_wr_max} || 0,
5078                          $drive->{bps_max_length} || 1,
5079                          $drive->{bps_rd_max_length} || 1,
5080                          $drive->{bps_wr_max_length} || 1,
5081                          $drive->{iops_max_length} || 1,
5082                          $drive->{iops_rd_max_length} || 1,
5083                          $drive->{iops_wr_max_length} || 1,
5084                     );
5085
5086                 }
5087
5088                 return 1;
5089             }
5090
5091         } else { # cdrom
5092
5093             if ($drive->{file} eq 'none') {
5094                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5095                 if (drive_is_cloudinit($old_drive)) {
5096                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5097                 }
5098             } else {
5099                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5100
5101                 # force eject if locked
5102                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5103
5104                 if ($path) {
5105                     mon_cmd($vmid, "blockdev-change-medium",
5106                         id => "$opt", filename => "$path");
5107                 }
5108             }
5109
5110             return 1;
5111         }
5112     }
5113
5114     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5115     # hotplug new disks
5116     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5117     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5118 }
5119
5120 # called in locked context by incoming migration
5121 sub vm_migrate_get_nbd_disks {
5122     my ($storecfg, $conf, $replicated_volumes) = @_;
5123
5124     my $local_volumes = {};
5125     PVE::QemuConfig->foreach_volume($conf, sub {
5126         my ($ds, $drive) = @_;
5127
5128         return if drive_is_cdrom($drive);
5129
5130         my $volid = $drive->{file};
5131
5132         return if !$volid;
5133
5134         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5135
5136         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5137         return if $scfg->{shared};
5138
5139         # replicated disks re-use existing state via bitmap
5140         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5141         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5142     });
5143     return $local_volumes;
5144 }
5145
5146 # called in locked context by incoming migration
5147 sub vm_migrate_alloc_nbd_disks {
5148     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5149
5150     my $format = undef;
5151
5152     my $nbd = {};
5153     foreach my $opt (sort keys %$source_volumes) {
5154         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5155
5156         if ($use_existing) {
5157             $nbd->{$opt}->{drivestr} = print_drive($drive);
5158             $nbd->{$opt}->{volid} = $volid;
5159             $nbd->{$opt}->{replicated} = 1;
5160             next;
5161         }
5162
5163         # If a remote storage is specified and the format of the original
5164         # volume is not available there, fall back to the default format.
5165         # Otherwise use the same format as the original.
5166         if (!$storagemap->{identity}) {
5167             $storeid = map_storage($storagemap, $storeid);
5168             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5169             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5170             my $fileFormat = qemu_img_format($scfg, $volname);
5171             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5172         } else {
5173             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5174             $format = qemu_img_format($scfg, $volname);
5175         }
5176
5177         my $size = $drive->{size} / 1024;
5178         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5179         my $newdrive = $drive;
5180         $newdrive->{format} = $format;
5181         $newdrive->{file} = $newvolid;
5182         my $drivestr = print_drive($newdrive);
5183         $nbd->{$opt}->{drivestr} = $drivestr;
5184         $nbd->{$opt}->{volid} = $newvolid;
5185     }
5186
5187     return $nbd;
5188 }
5189
5190 # see vm_start_nolock for parameters, additionally:
5191 # migrate_opts:
5192 #   storagemap = parsed storage map for allocating NBD disks
5193 sub vm_start {
5194     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5195
5196     return PVE::QemuConfig->lock_config($vmid, sub {
5197         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5198
5199         die "you can't start a vm if it's a template\n"
5200             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5201
5202         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5203         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5204
5205         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5206
5207         if ($has_backup_lock && $running) {
5208             # a backup is currently running, attempt to start the guest in the
5209             # existing QEMU instance
5210             return vm_resume($vmid);
5211         }
5212
5213         PVE::QemuConfig->check_lock($conf)
5214             if !($params->{skiplock} || $has_suspended_lock);
5215
5216         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5217
5218         die "VM $vmid already running\n" if $running;
5219
5220         if (my $storagemap = $migrate_opts->{storagemap}) {
5221             my $replicated = $migrate_opts->{replicated_volumes};
5222             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5223             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5224
5225             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5226                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5227             }
5228         }
5229
5230         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5231     });
5232 }
5233
5234
5235 # params:
5236 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5237 #   skiplock => 0/1, skip checking for config lock
5238 #   skiptemplate => 0/1, skip checking whether VM is template
5239 #   forcemachine => to force Qemu machine (rollback/migration)
5240 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5241 #   timeout => in seconds
5242 #   paused => start VM in paused state (backup)
5243 #   resume => resume from hibernation
5244 #   pbs-backing => {
5245 #      sata0 => {
5246 #         repository
5247 #         snapshot
5248 #         keyfile
5249 #         archive
5250 #      },
5251 #      virtio2 => ...
5252 #   }
5253 # migrate_opts:
5254 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5255 #   migratedfrom => source node
5256 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5257 #   network => CIDR of migration network
5258 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5259 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5260 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5261 sub vm_start_nolock {
5262     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5263
5264     my $statefile = $params->{statefile};
5265     my $resume = $params->{resume};
5266
5267     my $migratedfrom = $migrate_opts->{migratedfrom};
5268     my $migration_type = $migrate_opts->{type};
5269
5270     my $res = {};
5271
5272     # clean up leftover reboot request files
5273     eval { clear_reboot_request($vmid); };
5274     warn $@ if $@;
5275
5276     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5277         vmconfig_apply_pending($vmid, $conf, $storecfg);
5278         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5279     }
5280
5281     # don't regenerate the ISO if the VM is started as part of a live migration
5282     # this way we can reuse the old ISO with the correct config
5283     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5284
5285     my $defaults = load_defaults();
5286
5287     # set environment variable useful inside network script
5288     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5289
5290     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5291
5292     my $forcemachine = $params->{forcemachine};
5293     my $forcecpu = $params->{forcecpu};
5294     if ($resume) {
5295         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5296         $forcemachine = $conf->{runningmachine};
5297         $forcecpu = $conf->{runningcpu};
5298         print "Resuming suspended VM\n";
5299     }
5300
5301     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5302         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5303
5304     my $migration_ip;
5305     my $get_migration_ip = sub {
5306         my ($nodename) = @_;
5307
5308         return $migration_ip if defined($migration_ip);
5309
5310         my $cidr = $migrate_opts->{network};
5311
5312         if (!defined($cidr)) {
5313             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5314             $cidr = $dc_conf->{migration}->{network};
5315         }
5316
5317         if (defined($cidr)) {
5318             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5319
5320             die "could not get IP: no address configured on local " .
5321                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5322
5323             die "could not get IP: multiple addresses configured on local " .
5324                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5325
5326             $migration_ip = @$ips[0];
5327         }
5328
5329         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5330             if !defined($migration_ip);
5331
5332         return $migration_ip;
5333     };
5334
5335     my $migrate_uri;
5336     if ($statefile) {
5337         if ($statefile eq 'tcp') {
5338             my $localip = "localhost";
5339             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5340             my $nodename = nodename();
5341
5342             if (!defined($migration_type)) {
5343                 if (defined($datacenterconf->{migration}->{type})) {
5344                     $migration_type = $datacenterconf->{migration}->{type};
5345                 } else {
5346                     $migration_type = 'secure';
5347                 }
5348             }
5349
5350             if ($migration_type eq 'insecure') {
5351                 $localip = $get_migration_ip->($nodename);
5352                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5353             }
5354
5355             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5356             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5357             $migrate_uri = "tcp:${localip}:${migrate_port}";
5358             push @$cmd, '-incoming', $migrate_uri;
5359             push @$cmd, '-S';
5360
5361         } elsif ($statefile eq 'unix') {
5362             # should be default for secure migrations as a ssh TCP forward
5363             # tunnel is not deterministic reliable ready and fails regurarly
5364             # to set up in time, so use UNIX socket forwards
5365             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5366             unlink $socket_addr;
5367
5368             $migrate_uri = "unix:$socket_addr";
5369
5370             push @$cmd, '-incoming', $migrate_uri;
5371             push @$cmd, '-S';
5372
5373         } elsif (-e $statefile) {
5374             push @$cmd, '-loadstate', $statefile;
5375         } else {
5376             my $statepath = PVE::Storage::path($storecfg, $statefile);
5377             push @$vollist, $statefile;
5378             push @$cmd, '-loadstate', $statepath;
5379         }
5380     } elsif ($params->{paused}) {
5381         push @$cmd, '-S';
5382     }
5383
5384     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5385
5386     my $pci_devices = {}; # host pci devices
5387     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5388         my $dev = $conf->{"hostpci$i"} or next;
5389         $pci_devices->{$i} = parse_hostpci($dev);
5390     }
5391
5392     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } values $pci_devices->%* ];
5393     # reserve all PCI IDs before actually doing anything with them
5394     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5395
5396     eval {
5397         for my $id (sort keys %$pci_devices) {
5398             my $d = $pci_devices->{$id};
5399             for my $dev ($d->{pciid}->@*) {
5400                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5401             }
5402         }
5403     };
5404     if (my $err = $@) {
5405         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5406         warn $@ if $@;
5407         die $err;
5408     }
5409
5410     PVE::Storage::activate_volumes($storecfg, $vollist);
5411
5412     eval {
5413         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5414             outfunc => sub {}, errfunc => sub {});
5415     };
5416     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5417     # timeout should be more than enough here...
5418     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5419
5420     my $cpuunits = get_cpuunits($conf);
5421
5422     my %run_params = (
5423         timeout => $statefile ? undef : $start_timeout,
5424         umask => 0077,
5425         noerr => 1,
5426     );
5427
5428     # when migrating, prefix QEMU output so other side can pick up any
5429     # errors that might occur and show the user
5430     if ($migratedfrom) {
5431         $run_params{quiet} = 1;
5432         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5433     }
5434
5435     my %systemd_properties = (
5436         Slice => 'qemu.slice',
5437         KillMode => 'process',
5438         SendSIGKILL => 0,
5439         TimeoutStopUSec => ULONG_MAX, # infinity
5440     );
5441
5442     if (PVE::CGroup::cgroup_mode() == 2) {
5443         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5444         $systemd_properties{CPUWeight} = $cpuunits;
5445     } else {
5446         $systemd_properties{CPUShares} = $cpuunits;
5447     }
5448
5449     if (my $cpulimit = $conf->{cpulimit}) {
5450         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5451     }
5452     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5453
5454     my $run_qemu = sub {
5455         PVE::Tools::run_fork sub {
5456             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5457
5458             my $tpmpid;
5459             if (my $tpm = $conf->{tpmstate0}) {
5460                 # start the TPM emulator so QEMU can connect on start
5461                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5462             }
5463
5464             my $exitcode = run_command($cmd, %run_params);
5465             if ($exitcode) {
5466                 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5467                 kill 'TERM', $tpmpid if $tpmpid;
5468                 die "QEMU exited with code $exitcode\n";
5469             }
5470         };
5471     };
5472
5473     if ($conf->{hugepages}) {
5474
5475         my $code = sub {
5476             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5477             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5478
5479             PVE::QemuServer::Memory::hugepages_mount();
5480             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5481
5482             eval { $run_qemu->() };
5483             if (my $err = $@) {
5484                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5485                     if !$conf->{keephugepages};
5486                 die $err;
5487             }
5488
5489             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5490                 if !$conf->{keephugepages};
5491         };
5492         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5493
5494     } else {
5495         eval { $run_qemu->() };
5496     }
5497
5498     if (my $err = $@) {
5499         # deactivate volumes if start fails
5500         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5501         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5502
5503         die "start failed: $err";
5504     }
5505
5506     # re-reserve all PCI IDs now that we can know the actual VM PID
5507     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5508     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5509     warn $@ if $@;
5510
5511     print "migration listens on $migrate_uri\n" if $migrate_uri;
5512     $res->{migrate_uri} = $migrate_uri;
5513
5514     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5515         eval { mon_cmd($vmid, "cont"); };
5516         warn $@ if $@;
5517     }
5518
5519     #start nbd server for storage migration
5520     if (my $nbd = $migrate_opts->{nbd}) {
5521         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5522
5523         my $migrate_storage_uri;
5524         # nbd_protocol_version > 0 for unix socket support
5525         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5526             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5527             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5528             $migrate_storage_uri = "nbd:unix:$socket_path";
5529         } else {
5530             my $nodename = nodename();
5531             my $localip = $get_migration_ip->($nodename);
5532             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5533             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5534
5535             mon_cmd($vmid, "nbd-server-start", addr => {
5536                 type => 'inet',
5537                 data => {
5538                     host => "${localip}",
5539                     port => "${storage_migrate_port}",
5540                 },
5541             });
5542             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5543             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5544         }
5545
5546         $res->{migrate_storage_uri} = $migrate_storage_uri;
5547
5548         foreach my $opt (sort keys %$nbd) {
5549             my $drivestr = $nbd->{$opt}->{drivestr};
5550             my $volid = $nbd->{$opt}->{volid};
5551             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5552             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5553             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5554             print "re-using replicated volume: $opt - $volid\n"
5555                 if $nbd->{$opt}->{replicated};
5556
5557             $res->{drives}->{$opt} = $nbd->{$opt};
5558             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5559         }
5560     }
5561
5562     if ($migratedfrom) {
5563         eval {
5564             set_migration_caps($vmid);
5565         };
5566         warn $@ if $@;
5567
5568         if ($spice_port) {
5569             print "spice listens on port $spice_port\n";
5570             $res->{spice_port} = $spice_port;
5571             if ($migrate_opts->{spice_ticket}) {
5572                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5573                     $migrate_opts->{spice_ticket});
5574                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5575             }
5576         }
5577
5578     } else {
5579         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5580             if !$statefile && $conf->{balloon};
5581
5582         foreach my $opt (keys %$conf) {
5583             next if $opt !~  m/^net\d+$/;
5584             my $nicconf = parse_net($conf->{$opt});
5585             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5586         }
5587     }
5588
5589     mon_cmd($vmid, 'qom-set',
5590                 path => "machine/peripheral/balloon0",
5591                 property => "guest-stats-polling-interval",
5592                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5593
5594     if ($resume) {
5595         print "Resumed VM, removing state\n";
5596         if (my $vmstate = $conf->{vmstate}) {
5597             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5598             PVE::Storage::vdisk_free($storecfg, $vmstate);
5599         }
5600         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5601         PVE::QemuConfig->write_config($vmid, $conf);
5602     }
5603
5604     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5605
5606     return $res;
5607 }
5608
5609 sub vm_commandline {
5610     my ($storecfg, $vmid, $snapname) = @_;
5611
5612     my $conf = PVE::QemuConfig->load_config($vmid);
5613     my $forcemachine;
5614     my $forcecpu;
5615
5616     if ($snapname) {
5617         my $snapshot = $conf->{snapshots}->{$snapname};
5618         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5619
5620         # check for machine or CPU overrides in snapshot
5621         $forcemachine = $snapshot->{runningmachine};
5622         $forcecpu = $snapshot->{runningcpu};
5623
5624         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5625
5626         $conf = $snapshot;
5627     }
5628
5629     my $defaults = load_defaults();
5630
5631     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5632         $forcemachine, $forcecpu);
5633
5634     return PVE::Tools::cmd2string($cmd);
5635 }
5636
5637 sub vm_reset {
5638     my ($vmid, $skiplock) = @_;
5639
5640     PVE::QemuConfig->lock_config($vmid, sub {
5641
5642         my $conf = PVE::QemuConfig->load_config($vmid);
5643
5644         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5645
5646         mon_cmd($vmid, "system_reset");
5647     });
5648 }
5649
5650 sub get_vm_volumes {
5651     my ($conf) = @_;
5652
5653     my $vollist = [];
5654     foreach_volid($conf, sub {
5655         my ($volid, $attr) = @_;
5656
5657         return if $volid =~ m|^/|;
5658
5659         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5660         return if !$sid;
5661
5662         push @$vollist, $volid;
5663     });
5664
5665     return $vollist;
5666 }
5667
5668 sub vm_stop_cleanup {
5669     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5670
5671     eval {
5672
5673         if (!$keepActive) {
5674             my $vollist = get_vm_volumes($conf);
5675             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5676
5677             if (my $tpmdrive = $conf->{tpmstate0}) {
5678                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5679                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5680                 if ($storeid) {
5681                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5682                 }
5683             }
5684         }
5685
5686         foreach my $ext (qw(mon qmp pid vnc qga)) {
5687             unlink "/var/run/qemu-server/${vmid}.$ext";
5688         }
5689
5690         if ($conf->{ivshmem}) {
5691             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5692             # just delete it for now, VMs which have this already open do not
5693             # are affected, but new VMs will get a separated one. If this
5694             # becomes an issue we either add some sort of ref-counting or just
5695             # add a "don't delete on stop" flag to the ivshmem format.
5696             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5697         }
5698
5699         my $ids = [];
5700         foreach my $key (keys %$conf) {
5701             next if $key !~ m/^hostpci(\d+)$/;
5702             my $hostpciindex = $1;
5703             my $d = parse_hostpci($conf->{$key});
5704             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5705
5706             foreach my $pci (@{$d->{pciid}}) {
5707                 my $pciid = $pci->{id};
5708                 push @$ids, $pci->{id};
5709                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5710             }
5711         }
5712         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5713
5714         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5715     };
5716     warn $@ if $@; # avoid errors - just warn
5717 }
5718
5719 # call only in locked context
5720 sub _do_vm_stop {
5721     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5722
5723     my $pid = check_running($vmid, $nocheck);
5724     return if !$pid;
5725
5726     my $conf;
5727     if (!$nocheck) {
5728         $conf = PVE::QemuConfig->load_config($vmid);
5729         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5730         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5731             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5732             $timeout = $opts->{down} if $opts->{down};
5733         }
5734         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5735     }
5736
5737     eval {
5738         if ($shutdown) {
5739             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5740                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5741             } else {
5742                 mon_cmd($vmid, "system_powerdown");
5743             }
5744         } else {
5745             mon_cmd($vmid, "quit");
5746         }
5747     };
5748     my $err = $@;
5749
5750     if (!$err) {
5751         $timeout = 60 if !defined($timeout);
5752
5753         my $count = 0;
5754         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5755             $count++;
5756             sleep 1;
5757         }
5758
5759         if ($count >= $timeout) {
5760             if ($force) {
5761                 warn "VM still running - terminating now with SIGTERM\n";
5762                 kill 15, $pid;
5763             } else {
5764                 die "VM quit/powerdown failed - got timeout\n";
5765             }
5766         } else {
5767             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5768             return;
5769         }
5770     } else {
5771        if (!check_running($vmid, $nocheck)) {
5772             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5773             return;
5774        }
5775        if ($force) {
5776             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5777             kill 15, $pid;
5778         } else {
5779             die "VM quit/powerdown failed\n";
5780         }
5781     }
5782
5783     # wait again
5784     $timeout = 10;
5785
5786     my $count = 0;
5787     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5788         $count++;
5789         sleep 1;
5790     }
5791
5792     if ($count >= $timeout) {
5793         warn "VM still running - terminating now with SIGKILL\n";
5794         kill 9, $pid;
5795         sleep 1;
5796     }
5797
5798     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5799 }
5800
5801 # Note: use $nocheck to skip tests if VM configuration file exists.
5802 # We need that when migration VMs to other nodes (files already moved)
5803 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5804 sub vm_stop {
5805     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5806
5807     $force = 1 if !defined($force) && !$shutdown;
5808
5809     if ($migratedfrom){
5810         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5811         kill 15, $pid if $pid;
5812         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5813         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5814         return;
5815     }
5816
5817     PVE::QemuConfig->lock_config($vmid, sub {
5818         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5819    });
5820 }
5821
5822 sub vm_reboot {
5823     my ($vmid, $timeout) = @_;
5824
5825     PVE::QemuConfig->lock_config($vmid, sub {
5826         eval {
5827
5828             # only reboot if running, as qmeventd starts it again on a stop event
5829             return if !check_running($vmid);
5830
5831             create_reboot_request($vmid);
5832
5833             my $storecfg = PVE::Storage::config();
5834             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5835
5836         };
5837         if (my $err = $@) {
5838             # avoid that the next normal shutdown will be confused for a reboot
5839             clear_reboot_request($vmid);
5840             die $err;
5841         }
5842    });
5843 }
5844
5845 # note: if using the statestorage parameter, the caller has to check privileges
5846 sub vm_suspend {
5847     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5848
5849     my $conf;
5850     my $path;
5851     my $storecfg;
5852     my $vmstate;
5853
5854     PVE::QemuConfig->lock_config($vmid, sub {
5855
5856         $conf = PVE::QemuConfig->load_config($vmid);
5857
5858         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5859         PVE::QemuConfig->check_lock($conf)
5860             if !($skiplock || $is_backing_up);
5861
5862         die "cannot suspend to disk during backup\n"
5863             if $is_backing_up && $includestate;
5864
5865         if ($includestate) {
5866             $conf->{lock} = 'suspending';
5867             my $date = strftime("%Y-%m-%d", localtime(time()));
5868             $storecfg = PVE::Storage::config();
5869             if (!$statestorage) {
5870                 $statestorage = find_vmstate_storage($conf, $storecfg);
5871                 # check permissions for the storage
5872                 my $rpcenv = PVE::RPCEnvironment::get();
5873                 if ($rpcenv->{type} ne 'cli') {
5874                     my $authuser = $rpcenv->get_user();
5875                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5876                 }
5877             }
5878
5879
5880             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5881                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5882             $path = PVE::Storage::path($storecfg, $vmstate);
5883             PVE::QemuConfig->write_config($vmid, $conf);
5884         } else {
5885             mon_cmd($vmid, "stop");
5886         }
5887     });
5888
5889     if ($includestate) {
5890         # save vm state
5891         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5892
5893         eval {
5894             set_migration_caps($vmid, 1);
5895             mon_cmd($vmid, "savevm-start", statefile => $path);
5896             for(;;) {
5897                 my $state = mon_cmd($vmid, "query-savevm");
5898                 if (!$state->{status}) {
5899                     die "savevm not active\n";
5900                 } elsif ($state->{status} eq 'active') {
5901                     sleep(1);
5902                     next;
5903                 } elsif ($state->{status} eq 'completed') {
5904                     print "State saved, quitting\n";
5905                     last;
5906                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5907                     die "query-savevm failed with error '$state->{error}'\n"
5908                 } else {
5909                     die "query-savevm returned status '$state->{status}'\n";
5910                 }
5911             }
5912         };
5913         my $err = $@;
5914
5915         PVE::QemuConfig->lock_config($vmid, sub {
5916             $conf = PVE::QemuConfig->load_config($vmid);
5917             if ($err) {
5918                 # cleanup, but leave suspending lock, to indicate something went wrong
5919                 eval {
5920                     mon_cmd($vmid, "savevm-end");
5921                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5922                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5923                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5924                     PVE::QemuConfig->write_config($vmid, $conf);
5925                 };
5926                 warn $@ if $@;
5927                 die $err;
5928             }
5929
5930             die "lock changed unexpectedly\n"
5931                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5932
5933             mon_cmd($vmid, "quit");
5934             $conf->{lock} = 'suspended';
5935             PVE::QemuConfig->write_config($vmid, $conf);
5936         });
5937     }
5938 }
5939
5940 sub vm_resume {
5941     my ($vmid, $skiplock, $nocheck) = @_;
5942
5943     PVE::QemuConfig->lock_config($vmid, sub {
5944         my $res = mon_cmd($vmid, 'query-status');
5945         my $resume_cmd = 'cont';
5946         my $reset = 0;
5947
5948         if ($res->{status}) {
5949             return if $res->{status} eq 'running'; # job done, go home
5950             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5951             $reset = 1 if $res->{status} eq 'shutdown';
5952         }
5953
5954         if (!$nocheck) {
5955
5956             my $conf = PVE::QemuConfig->load_config($vmid);
5957
5958             PVE::QemuConfig->check_lock($conf)
5959                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5960         }
5961
5962         if ($reset) {
5963             # required if a VM shuts down during a backup and we get a resume
5964             # request before the backup finishes for example
5965             mon_cmd($vmid, "system_reset");
5966         }
5967         mon_cmd($vmid, $resume_cmd);
5968     });
5969 }
5970
5971 sub vm_sendkey {
5972     my ($vmid, $skiplock, $key) = @_;
5973
5974     PVE::QemuConfig->lock_config($vmid, sub {
5975
5976         my $conf = PVE::QemuConfig->load_config($vmid);
5977
5978         # there is no qmp command, so we use the human monitor command
5979         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5980         die $res if $res ne '';
5981     });
5982 }
5983
5984 # vzdump restore implementaion
5985
5986 sub tar_archive_read_firstfile {
5987     my $archive = shift;
5988
5989     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5990
5991     # try to detect archive type first
5992     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5993         die "unable to open file '$archive'\n";
5994     my $firstfile = <$fh>;
5995     kill 15, $pid;
5996     close $fh;
5997
5998     die "ERROR: archive contaions no data\n" if !$firstfile;
5999     chomp $firstfile;
6000
6001     return $firstfile;
6002 }
6003
6004 sub tar_restore_cleanup {
6005     my ($storecfg, $statfile) = @_;
6006
6007     print STDERR "starting cleanup\n";
6008
6009     if (my $fd = IO::File->new($statfile, "r")) {
6010         while (defined(my $line = <$fd>)) {
6011             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6012                 my $volid = $2;
6013                 eval {
6014                     if ($volid =~ m|^/|) {
6015                         unlink $volid || die 'unlink failed\n';
6016                     } else {
6017                         PVE::Storage::vdisk_free($storecfg, $volid);
6018                     }
6019                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6020                 };
6021                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6022             } else {
6023                 print STDERR "unable to parse line in statfile - $line";
6024             }
6025         }
6026         $fd->close();
6027     }
6028 }
6029
6030 sub restore_file_archive {
6031     my ($archive, $vmid, $user, $opts) = @_;
6032
6033     return restore_vma_archive($archive, $vmid, $user, $opts)
6034         if $archive eq '-';
6035
6036     my $info = PVE::Storage::archive_info($archive);
6037     my $format = $opts->{format} // $info->{format};
6038     my $comp = $info->{compression};
6039
6040     # try to detect archive format
6041     if ($format eq 'tar') {
6042         return restore_tar_archive($archive, $vmid, $user, $opts);
6043     } else {
6044         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6045     }
6046 }
6047
6048 # hepler to remove disks that will not be used after restore
6049 my $restore_cleanup_oldconf = sub {
6050     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6051
6052     PVE::QemuConfig->foreach_volume($oldconf, sub {
6053         my ($ds, $drive) = @_;
6054
6055         return if drive_is_cdrom($drive, 1);
6056
6057         my $volid = $drive->{file};
6058         return if !$volid || $volid =~ m|^/|;
6059
6060         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6061         return if !$path || !$owner || ($owner != $vmid);
6062
6063         # Note: only delete disk we want to restore
6064         # other volumes will become unused
6065         if ($virtdev_hash->{$ds}) {
6066             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6067             if (my $err = $@) {
6068                 warn $err;
6069             }
6070         }
6071     });
6072
6073     # delete vmstate files, after the restore we have no snapshots anymore
6074     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6075         my $snap = $oldconf->{snapshots}->{$snapname};
6076         if ($snap->{vmstate}) {
6077             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6078             if (my $err = $@) {
6079                 warn $err;
6080             }
6081         }
6082     }
6083 };
6084
6085 # Helper to parse vzdump backup device hints
6086 #
6087 # $rpcenv: Environment, used to ckeck storage permissions
6088 # $user: User ID, to check storage permissions
6089 # $storecfg: Storage configuration
6090 # $fh: the file handle for reading the configuration
6091 # $devinfo: should contain device sizes for all backu-up'ed devices
6092 # $options: backup options (pool, default storage)
6093 #
6094 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6095 my $parse_backup_hints = sub {
6096     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6097
6098     my $virtdev_hash = {};
6099
6100     while (defined(my $line = <$fh>)) {
6101         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6102             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6103             die "archive does not contain data for drive '$virtdev'\n"
6104                 if !$devinfo->{$devname};
6105
6106             if (defined($options->{storage})) {
6107                 $storeid = $options->{storage} || 'local';
6108             } elsif (!$storeid) {
6109                 $storeid = 'local';
6110             }
6111             $format = 'raw' if !$format;
6112             $devinfo->{$devname}->{devname} = $devname;
6113             $devinfo->{$devname}->{virtdev} = $virtdev;
6114             $devinfo->{$devname}->{format} = $format;
6115             $devinfo->{$devname}->{storeid} = $storeid;
6116
6117             # check permission on storage
6118             my $pool = $options->{pool}; # todo: do we need that?
6119             if ($user ne 'root@pam') {
6120                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6121             }
6122
6123             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6124         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6125             my $virtdev = $1;
6126             my $drive = parse_drive($virtdev, $2);
6127             if (drive_is_cloudinit($drive)) {
6128                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6129                 $storeid = $options->{storage} if defined ($options->{storage});
6130                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6131                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6132
6133                 $virtdev_hash->{$virtdev} = {
6134                     format => $format,
6135                     storeid => $storeid,
6136                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6137                     is_cloudinit => 1,
6138                 };
6139             }
6140         }
6141     }
6142
6143     return $virtdev_hash;
6144 };
6145
6146 # Helper to allocate and activate all volumes required for a restore
6147 #
6148 # $storecfg: Storage configuration
6149 # $virtdev_hash: as returned by parse_backup_hints()
6150 #
6151 # Returns: { $virtdev => $volid }
6152 my $restore_allocate_devices = sub {
6153     my ($storecfg, $virtdev_hash, $vmid) = @_;
6154
6155     my $map = {};
6156     foreach my $virtdev (sort keys %$virtdev_hash) {
6157         my $d = $virtdev_hash->{$virtdev};
6158         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6159         my $storeid = $d->{storeid};
6160         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6161
6162         # test if requested format is supported
6163         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6164         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6165         $d->{format} = $defFormat if !$supported;
6166
6167         my $name;
6168         if ($d->{is_cloudinit}) {
6169             $name = "vm-$vmid-cloudinit";
6170             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6171             if ($scfg->{path}) {
6172                 $name .= ".$d->{format}";
6173             }
6174         }
6175
6176         my $volid = PVE::Storage::vdisk_alloc(
6177             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6178
6179         print STDERR "new volume ID is '$volid'\n";
6180         $d->{volid} = $volid;
6181
6182         PVE::Storage::activate_volumes($storecfg, [$volid]);
6183
6184         $map->{$virtdev} = $volid;
6185     }
6186
6187     return $map;
6188 };
6189
6190 sub restore_update_config_line {
6191     my ($cookie, $map, $line, $unique) = @_;
6192
6193     return '' if $line =~ m/^\#qmdump\#/;
6194     return '' if $line =~ m/^\#vzdump\#/;
6195     return '' if $line =~ m/^lock:/;
6196     return '' if $line =~ m/^unused\d+:/;
6197     return '' if $line =~ m/^parent:/;
6198
6199     my $res = '';
6200
6201     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6202     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6203         # try to convert old 1.X settings
6204         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6205         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6206             my ($model, $macaddr) = split(/\=/, $devconfig);
6207             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6208             my $net = {
6209                 model => $model,
6210                 bridge => "vmbr$ind",
6211                 macaddr => $macaddr,
6212             };
6213             my $netstr = print_net($net);
6214
6215             $res .= "net$cookie->{netcount}: $netstr\n";
6216             $cookie->{netcount}++;
6217         }
6218     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6219         my ($id, $netstr) = ($1, $2);
6220         my $net = parse_net($netstr);
6221         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6222         $netstr = print_net($net);
6223         $res .= "$id: $netstr\n";
6224     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6225         my $virtdev = $1;
6226         my $value = $3;
6227         my $di = parse_drive($virtdev, $value);
6228         if (defined($di->{backup}) && !$di->{backup}) {
6229             $res .= "#$line";
6230         } elsif ($map->{$virtdev}) {
6231             delete $di->{format}; # format can change on restore
6232             $di->{file} = $map->{$virtdev};
6233             $value = print_drive($di);
6234             $res .= "$virtdev: $value\n";
6235         } else {
6236             $res .= $line;
6237         }
6238     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6239         my $vmgenid = $1;
6240         if ($vmgenid ne '0') {
6241             # always generate a new vmgenid if there was a valid one setup
6242             $vmgenid = generate_uuid();
6243         }
6244         $res .= "vmgenid: $vmgenid\n";
6245     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6246         my ($uuid, $uuid_str);
6247         UUID::generate($uuid);
6248         UUID::unparse($uuid, $uuid_str);
6249         my $smbios1 = parse_smbios1($2);
6250         $smbios1->{uuid} = $uuid_str;
6251         $res .= $1.print_smbios1($smbios1)."\n";
6252     } else {
6253         $res .= $line;
6254     }
6255
6256     return $res;
6257 }
6258
6259 my $restore_deactivate_volumes = sub {
6260     my ($storecfg, $devinfo) = @_;
6261
6262     my $vollist = [];
6263     foreach my $devname (keys %$devinfo) {
6264         my $volid = $devinfo->{$devname}->{volid};
6265         push @$vollist, $volid if $volid;
6266     }
6267
6268     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6269 };
6270
6271 my $restore_destroy_volumes = sub {
6272     my ($storecfg, $devinfo) = @_;
6273
6274     foreach my $devname (keys %$devinfo) {
6275         my $volid = $devinfo->{$devname}->{volid};
6276         next if !$volid;
6277         eval {
6278             if ($volid =~ m|^/|) {
6279                 unlink $volid || die 'unlink failed\n';
6280             } else {
6281                 PVE::Storage::vdisk_free($storecfg, $volid);
6282             }
6283             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6284         };
6285         print STDERR "unable to cleanup '$volid' - $@" if $@;
6286     }
6287 };
6288
6289 sub scan_volids {
6290     my ($cfg, $vmid) = @_;
6291
6292     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6293
6294     my $volid_hash = {};
6295     foreach my $storeid (keys %$info) {
6296         foreach my $item (@{$info->{$storeid}}) {
6297             next if !($item->{volid} && $item->{size});
6298             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6299             $volid_hash->{$item->{volid}} = $item;
6300         }
6301     }
6302
6303     return $volid_hash;
6304 }
6305
6306 sub update_disk_config {
6307     my ($vmid, $conf, $volid_hash) = @_;
6308
6309     my $changes;
6310     my $prefix = "VM $vmid";
6311
6312     # used and unused disks
6313     my $referenced = {};
6314
6315     # Note: it is allowed to define multiple storages with same path (alias), so
6316     # we need to check both 'volid' and real 'path' (two different volid can point
6317     # to the same path).
6318
6319     my $referencedpath = {};
6320
6321     # update size info
6322     PVE::QemuConfig->foreach_volume($conf, sub {
6323         my ($opt, $drive) = @_;
6324
6325         my $volid = $drive->{file};
6326         return if !$volid;
6327         my $volume = $volid_hash->{$volid};
6328
6329         # mark volid as "in-use" for next step
6330         $referenced->{$volid} = 1;
6331         if ($volume && (my $path = $volume->{path})) {
6332             $referencedpath->{$path} = 1;
6333         }
6334
6335         return if drive_is_cdrom($drive);
6336         return if !$volume;
6337
6338         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6339         if (defined($updated)) {
6340             $changes = 1;
6341             $conf->{$opt} = print_drive($updated);
6342             print "$prefix ($opt): $msg\n";
6343         }
6344     });
6345
6346     # remove 'unusedX' entry if volume is used
6347     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6348         my ($opt, $drive) = @_;
6349
6350         my $volid = $drive->{file};
6351         return if !$volid;
6352
6353         my $path;
6354         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6355         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6356             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6357             $changes = 1;
6358             delete $conf->{$opt};
6359         }
6360
6361         $referenced->{$volid} = 1;
6362         $referencedpath->{$path} = 1 if $path;
6363     });
6364
6365     foreach my $volid (sort keys %$volid_hash) {
6366         next if $volid =~ m/vm-$vmid-state-/;
6367         next if $referenced->{$volid};
6368         my $path = $volid_hash->{$volid}->{path};
6369         next if !$path; # just to be sure
6370         next if $referencedpath->{$path};
6371         $changes = 1;
6372         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6373         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6374         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6375     }
6376
6377     return $changes;
6378 }
6379
6380 sub rescan {
6381     my ($vmid, $nolock, $dryrun) = @_;
6382
6383     my $cfg = PVE::Storage::config();
6384
6385     print "rescan volumes...\n";
6386     my $volid_hash = scan_volids($cfg, $vmid);
6387
6388     my $updatefn =  sub {
6389         my ($vmid) = @_;
6390
6391         my $conf = PVE::QemuConfig->load_config($vmid);
6392
6393         PVE::QemuConfig->check_lock($conf);
6394
6395         my $vm_volids = {};
6396         foreach my $volid (keys %$volid_hash) {
6397             my $info = $volid_hash->{$volid};
6398             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6399         }
6400
6401         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6402
6403         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6404     };
6405
6406     if (defined($vmid)) {
6407         if ($nolock) {
6408             &$updatefn($vmid);
6409         } else {
6410             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6411         }
6412     } else {
6413         my $vmlist = config_list();
6414         foreach my $vmid (keys %$vmlist) {
6415             if ($nolock) {
6416                 &$updatefn($vmid);
6417             } else {
6418                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6419             }
6420         }
6421     }
6422 }
6423
6424 sub restore_proxmox_backup_archive {
6425     my ($archive, $vmid, $user, $options) = @_;
6426
6427     my $storecfg = PVE::Storage::config();
6428
6429     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6430     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6431
6432     my $fingerprint = $scfg->{fingerprint};
6433     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6434
6435     my $repo = PVE::PBSClient::get_repository($scfg);
6436
6437     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6438     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6439     local $ENV{PBS_PASSWORD} = $password;
6440     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6441
6442     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6443         PVE::Storage::parse_volname($storecfg, $archive);
6444
6445     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6446
6447     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6448
6449     my $tmpdir = "/var/tmp/vzdumptmp$$";
6450     rmtree $tmpdir;
6451     mkpath $tmpdir;
6452
6453     my $conffile = PVE::QemuConfig->config_file($vmid);
6454      # disable interrupts (always do cleanups)
6455     local $SIG{INT} =
6456         local $SIG{TERM} =
6457         local $SIG{QUIT} =
6458         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6459
6460     # Note: $oldconf is undef if VM does not exists
6461     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6462     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6463     my $new_conf_raw = '';
6464
6465     my $rpcenv = PVE::RPCEnvironment::get();
6466     my $devinfo = {};
6467
6468     eval {
6469         # enable interrupts
6470         local $SIG{INT} =
6471             local $SIG{TERM} =
6472             local $SIG{QUIT} =
6473             local $SIG{HUP} =
6474             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6475
6476         my $cfgfn = "$tmpdir/qemu-server.conf";
6477         my $firewall_config_fn = "$tmpdir/fw.conf";
6478         my $index_fn = "$tmpdir/index.json";
6479
6480         my $cmd = "restore";
6481
6482         my $param = [$pbs_backup_name, "index.json", $index_fn];
6483         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6484         my $index = PVE::Tools::file_get_contents($index_fn);
6485         $index = decode_json($index);
6486
6487         # print Dumper($index);
6488         foreach my $info (@{$index->{files}}) {
6489             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6490                 my $devname = $1;
6491                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6492                     $devinfo->{$devname}->{size} = $1;
6493                 } else {
6494                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6495                 }
6496             }
6497         }
6498
6499         my $is_qemu_server_backup = scalar(
6500             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6501         );
6502         if (!$is_qemu_server_backup) {
6503             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6504         }
6505         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6506
6507         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6508         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6509
6510         if ($has_firewall_config) {
6511             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6512             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6513
6514             my $pve_firewall_dir = '/etc/pve/firewall';
6515             mkdir $pve_firewall_dir; # make sure the dir exists
6516             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6517         }
6518
6519         my $fh = IO::File->new($cfgfn, "r") ||
6520             die "unable to read qemu-server.conf - $!\n";
6521
6522         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6523
6524         # fixme: rate limit?
6525
6526         # create empty/temp config
6527         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6528
6529         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6530
6531         # allocate volumes
6532         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6533
6534         foreach my $virtdev (sort keys %$virtdev_hash) {
6535             my $d = $virtdev_hash->{$virtdev};
6536             next if $d->{is_cloudinit}; # no need to restore cloudinit
6537
6538             # this fails if storage is unavailable
6539             my $volid = $d->{volid};
6540             my $path = PVE::Storage::path($storecfg, $volid);
6541
6542             # for live-restore we only want to preload the efidisk and TPM state
6543             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6544
6545             my $pbs_restore_cmd = [
6546                 '/usr/bin/pbs-restore',
6547                 '--repository', $repo,
6548                 $pbs_backup_name,
6549                 "$d->{devname}.img.fidx",
6550                 $path,
6551                 '--verbose',
6552                 ];
6553
6554             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6555             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6556
6557             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6558                 push @$pbs_restore_cmd, '--skip-zero';
6559             }
6560
6561             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6562             print "restore proxmox backup image: $dbg_cmdstring\n";
6563             run_command($pbs_restore_cmd);
6564         }
6565
6566         $fh->seek(0, 0) || die "seek failed - $!\n";
6567
6568         my $cookie = { netcount => 0 };
6569         while (defined(my $line = <$fh>)) {
6570             $new_conf_raw .= restore_update_config_line(
6571                 $cookie,
6572                 $map,
6573                 $line,
6574                 $options->{unique},
6575             );
6576         }
6577
6578         $fh->close();
6579     };
6580     my $err = $@;
6581
6582     if ($err || !$options->{live}) {
6583         $restore_deactivate_volumes->($storecfg, $devinfo);
6584     }
6585
6586     rmtree $tmpdir;
6587
6588     if ($err) {
6589         $restore_destroy_volumes->($storecfg, $devinfo);
6590         die $err;
6591     }
6592
6593     if ($options->{live}) {
6594         # keep lock during live-restore
6595         $new_conf_raw .= "\nlock: create";
6596     }
6597
6598     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6599
6600     PVE::Cluster::cfs_update(); # make sure we read new file
6601
6602     eval { rescan($vmid, 1); };
6603     warn $@ if $@;
6604
6605     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6606
6607     if ($options->{live}) {
6608         # enable interrupts
6609         local $SIG{INT} =
6610             local $SIG{TERM} =
6611             local $SIG{QUIT} =
6612             local $SIG{HUP} =
6613             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6614
6615         my $conf = PVE::QemuConfig->load_config($vmid);
6616         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6617
6618         # these special drives are already restored before start
6619         delete $devinfo->{'drive-efidisk0'};
6620         delete $devinfo->{'drive-tpmstate0-backup'};
6621         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6622
6623         PVE::QemuConfig->remove_lock($vmid, "create");
6624     }
6625 }
6626
6627 sub pbs_live_restore {
6628     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6629
6630     print "starting VM for live-restore\n";
6631     print "repository: '$repo', snapshot: '$snap'\n";
6632
6633     my $pbs_backing = {};
6634     for my $ds (keys %$restored_disks) {
6635         $ds =~ m/^drive-(.*)$/;
6636         my $confname = $1;
6637         $pbs_backing->{$confname} = {
6638             repository => $repo,
6639             snapshot => $snap,
6640             archive => "$ds.img.fidx",
6641         };
6642         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6643
6644         my $drive = parse_drive($confname, $conf->{$confname});
6645         print "restoring '$ds' to '$drive->{file}'\n";
6646     }
6647
6648     my $drives_streamed = 0;
6649     eval {
6650         # make sure HA doesn't interrupt our restore by stopping the VM
6651         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6652             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6653         }
6654
6655         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6656         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6657         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6658
6659         my $qmeventd_fd = register_qmeventd_handle($vmid);
6660
6661         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6662         # this will effectively collapse the backing image chain consisting of
6663         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6664         # removes itself once all backing images vanish with 'auto-remove=on')
6665         my $jobs = {};
6666         for my $ds (sort keys %$restored_disks) {
6667             my $job_id = "restore-$ds";
6668             mon_cmd($vmid, 'block-stream',
6669                 'job-id' => $job_id,
6670                 device => "$ds",
6671             );
6672             $jobs->{$job_id} = {};
6673         }
6674
6675         mon_cmd($vmid, 'cont');
6676         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6677
6678         print "restore-drive jobs finished successfully, removing all tracking block devices"
6679             ." to disconnect from Proxmox Backup Server\n";
6680
6681         for my $ds (sort keys %$restored_disks) {
6682             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6683         }
6684
6685         close($qmeventd_fd);
6686     };
6687
6688     my $err = $@;
6689
6690     if ($err) {
6691         warn "An error occured during live-restore: $err\n";
6692         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6693         die "live-restore failed\n";
6694     }
6695 }
6696
6697 sub restore_vma_archive {
6698     my ($archive, $vmid, $user, $opts, $comp) = @_;
6699
6700     my $readfrom = $archive;
6701
6702     my $cfg = PVE::Storage::config();
6703     my $commands = [];
6704     my $bwlimit = $opts->{bwlimit};
6705
6706     my $dbg_cmdstring = '';
6707     my $add_pipe = sub {
6708         my ($cmd) = @_;
6709         push @$commands, $cmd;
6710         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6711         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6712         $readfrom = '-';
6713     };
6714
6715     my $input = undef;
6716     if ($archive eq '-') {
6717         $input = '<&STDIN';
6718     } else {
6719         # If we use a backup from a PVE defined storage we also consider that
6720         # storage's rate limit:
6721         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6722         if (defined($volid)) {
6723             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6724             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6725             if ($readlimit) {
6726                 print STDERR "applying read rate limit: $readlimit\n";
6727                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6728                 $add_pipe->($cstream);
6729             }
6730         }
6731     }
6732
6733     if ($comp) {
6734         my $info = PVE::Storage::decompressor_info('vma', $comp);
6735         my $cmd = $info->{decompressor};
6736         push @$cmd, $readfrom;
6737         $add_pipe->($cmd);
6738     }
6739
6740     my $tmpdir = "/var/tmp/vzdumptmp$$";
6741     rmtree $tmpdir;
6742
6743     # disable interrupts (always do cleanups)
6744     local $SIG{INT} =
6745         local $SIG{TERM} =
6746         local $SIG{QUIT} =
6747         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6748
6749     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6750     POSIX::mkfifo($mapfifo, 0600);
6751     my $fifofh;
6752     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6753
6754     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6755
6756     my $oldtimeout;
6757     my $timeout = 5;
6758
6759     my $devinfo = {};
6760
6761     my $rpcenv = PVE::RPCEnvironment::get();
6762
6763     my $conffile = PVE::QemuConfig->config_file($vmid);
6764
6765     # Note: $oldconf is undef if VM does not exist
6766     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6767     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6768     my $new_conf_raw = '';
6769
6770     my %storage_limits;
6771
6772     my $print_devmap = sub {
6773         my $cfgfn = "$tmpdir/qemu-server.conf";
6774
6775         # we can read the config - that is already extracted
6776         my $fh = IO::File->new($cfgfn, "r") ||
6777             die "unable to read qemu-server.conf - $!\n";
6778
6779         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6780         if (-f $fwcfgfn) {
6781             my $pve_firewall_dir = '/etc/pve/firewall';
6782             mkdir $pve_firewall_dir; # make sure the dir exists
6783             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6784         }
6785
6786         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6787
6788         foreach my $info (values %{$virtdev_hash}) {
6789             my $storeid = $info->{storeid};
6790             next if defined($storage_limits{$storeid});
6791
6792             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6793             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6794             $storage_limits{$storeid} = $limit * 1024;
6795         }
6796
6797         foreach my $devname (keys %$devinfo) {
6798             die "found no device mapping information for device '$devname'\n"
6799                 if !$devinfo->{$devname}->{virtdev};
6800         }
6801
6802         # create empty/temp config
6803         if ($oldconf) {
6804             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6805             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6806         }
6807
6808         # allocate volumes
6809         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6810
6811         # print restore information to $fifofh
6812         foreach my $virtdev (sort keys %$virtdev_hash) {
6813             my $d = $virtdev_hash->{$virtdev};
6814             next if $d->{is_cloudinit}; # no need to restore cloudinit
6815
6816             my $storeid = $d->{storeid};
6817             my $volid = $d->{volid};
6818
6819             my $map_opts = '';
6820             if (my $limit = $storage_limits{$storeid}) {
6821                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6822             }
6823
6824             my $write_zeros = 1;
6825             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6826                 $write_zeros = 0;
6827             }
6828
6829             my $path = PVE::Storage::path($cfg, $volid);
6830
6831             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6832
6833             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6834         }
6835
6836         $fh->seek(0, 0) || die "seek failed - $!\n";
6837
6838         my $cookie = { netcount => 0 };
6839         while (defined(my $line = <$fh>)) {
6840             $new_conf_raw .= restore_update_config_line(
6841                 $cookie,
6842                 $map,
6843                 $line,
6844                 $opts->{unique},
6845             );
6846         }
6847
6848         $fh->close();
6849     };
6850
6851     eval {
6852         # enable interrupts
6853         local $SIG{INT} =
6854             local $SIG{TERM} =
6855             local $SIG{QUIT} =
6856             local $SIG{HUP} =
6857             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6858         local $SIG{ALRM} = sub { die "got timeout\n"; };
6859
6860         $oldtimeout = alarm($timeout);
6861
6862         my $parser = sub {
6863             my $line = shift;
6864
6865             print "$line\n";
6866
6867             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6868                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6869                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6870             } elsif ($line =~ m/^CTIME: /) {
6871                 # we correctly received the vma config, so we can disable
6872                 # the timeout now for disk allocation (set to 10 minutes, so
6873                 # that we always timeout if something goes wrong)
6874                 alarm(600);
6875                 &$print_devmap();
6876                 print $fifofh "done\n";
6877                 my $tmp = $oldtimeout || 0;
6878                 $oldtimeout = undef;
6879                 alarm($tmp);
6880                 close($fifofh);
6881                 $fifofh = undef;
6882             }
6883         };
6884
6885         print "restore vma archive: $dbg_cmdstring\n";
6886         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6887     };
6888     my $err = $@;
6889
6890     alarm($oldtimeout) if $oldtimeout;
6891
6892     $restore_deactivate_volumes->($cfg, $devinfo);
6893
6894     close($fifofh) if $fifofh;
6895     unlink $mapfifo;
6896     rmtree $tmpdir;
6897
6898     if ($err) {
6899         $restore_destroy_volumes->($cfg, $devinfo);
6900         die $err;
6901     }
6902
6903     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6904
6905     PVE::Cluster::cfs_update(); # make sure we read new file
6906
6907     eval { rescan($vmid, 1); };
6908     warn $@ if $@;
6909
6910     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6911 }
6912
6913 sub restore_tar_archive {
6914     my ($archive, $vmid, $user, $opts) = @_;
6915
6916     if ($archive ne '-') {
6917         my $firstfile = tar_archive_read_firstfile($archive);
6918         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6919             if $firstfile ne 'qemu-server.conf';
6920     }
6921
6922     my $storecfg = PVE::Storage::config();
6923
6924     # avoid zombie disks when restoring over an existing VM -> cleanup first
6925     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6926     # skiplock=1 because qmrestore has set the 'create' lock itself already
6927     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6928     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6929
6930     my $tocmd = "/usr/lib/qemu-server/qmextract";
6931
6932     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6933     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6934     $tocmd .= ' --prealloc' if $opts->{prealloc};
6935     $tocmd .= ' --info' if $opts->{info};
6936
6937     # tar option "xf" does not autodetect compression when read from STDIN,
6938     # so we pipe to zcat
6939     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6940         PVE::Tools::shellquote("--to-command=$tocmd");
6941
6942     my $tmpdir = "/var/tmp/vzdumptmp$$";
6943     mkpath $tmpdir;
6944
6945     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6946     local $ENV{VZDUMP_VMID} = $vmid;
6947     local $ENV{VZDUMP_USER} = $user;
6948
6949     my $conffile = PVE::QemuConfig->config_file($vmid);
6950     my $new_conf_raw = '';
6951
6952     # disable interrupts (always do cleanups)
6953     local $SIG{INT} =
6954         local $SIG{TERM} =
6955         local $SIG{QUIT} =
6956         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6957
6958     eval {
6959         # enable interrupts
6960         local $SIG{INT} =
6961             local $SIG{TERM} =
6962             local $SIG{QUIT} =
6963             local $SIG{HUP} =
6964             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6965
6966         if ($archive eq '-') {
6967             print "extracting archive from STDIN\n";
6968             run_command($cmd, input => "<&STDIN");
6969         } else {
6970             print "extracting archive '$archive'\n";
6971             run_command($cmd);
6972         }
6973
6974         return if $opts->{info};
6975
6976         # read new mapping
6977         my $map = {};
6978         my $statfile = "$tmpdir/qmrestore.stat";
6979         if (my $fd = IO::File->new($statfile, "r")) {
6980             while (defined (my $line = <$fd>)) {
6981                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6982                     $map->{$1} = $2 if $1;
6983                 } else {
6984                     print STDERR "unable to parse line in statfile - $line\n";
6985                 }
6986             }
6987             $fd->close();
6988         }
6989
6990         my $confsrc = "$tmpdir/qemu-server.conf";
6991
6992         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6993
6994         my $cookie = { netcount => 0 };
6995         while (defined (my $line = <$srcfd>)) {
6996             $new_conf_raw .= restore_update_config_line(
6997                 $cookie,
6998                 $map,
6999                 $line,
7000                 $opts->{unique},
7001             );
7002         }
7003
7004         $srcfd->close();
7005     };
7006     if (my $err = $@) {
7007         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7008         die $err;
7009     }
7010
7011     rmtree $tmpdir;
7012
7013     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7014
7015     PVE::Cluster::cfs_update(); # make sure we read new file
7016
7017     eval { rescan($vmid, 1); };
7018     warn $@ if $@;
7019 };
7020
7021 sub foreach_storage_used_by_vm {
7022     my ($conf, $func) = @_;
7023
7024     my $sidhash = {};
7025
7026     PVE::QemuConfig->foreach_volume($conf, sub {
7027         my ($ds, $drive) = @_;
7028         return if drive_is_cdrom($drive);
7029
7030         my $volid = $drive->{file};
7031
7032         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7033         $sidhash->{$sid} = $sid if $sid;
7034     });
7035
7036     foreach my $sid (sort keys %$sidhash) {
7037         &$func($sid);
7038     }
7039 }
7040
7041 my $qemu_snap_storage = {
7042     rbd => 1,
7043 };
7044 sub do_snapshots_with_qemu {
7045     my ($storecfg, $volid) = @_;
7046
7047     my $storage_name = PVE::Storage::parse_volume_id($volid);
7048     my $scfg = $storecfg->{ids}->{$storage_name};
7049     die "could not find storage '$storage_name'\n" if !defined($scfg);
7050
7051     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7052         return 1;
7053     }
7054
7055     if ($volid =~ m/\.(qcow2|qed)$/){
7056         return 1;
7057     }
7058
7059     return;
7060 }
7061
7062 sub qga_check_running {
7063     my ($vmid, $nowarn) = @_;
7064
7065     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7066     if ($@) {
7067         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7068         return 0;
7069     }
7070     return 1;
7071 }
7072
7073 sub template_create {
7074     my ($vmid, $conf, $disk) = @_;
7075
7076     my $storecfg = PVE::Storage::config();
7077
7078     PVE::QemuConfig->foreach_volume($conf, sub {
7079         my ($ds, $drive) = @_;
7080
7081         return if drive_is_cdrom($drive);
7082         return if $disk && $ds ne $disk;
7083
7084         my $volid = $drive->{file};
7085         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7086
7087         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7088         $drive->{file} = $voliddst;
7089         $conf->{$ds} = print_drive($drive);
7090         PVE::QemuConfig->write_config($vmid, $conf);
7091     });
7092 }
7093
7094 sub convert_iscsi_path {
7095     my ($path) = @_;
7096
7097     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7098         my $portal = $1;
7099         my $target = $2;
7100         my $lun = $3;
7101
7102         my $initiator_name = get_initiator_name();
7103
7104         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7105                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7106     }
7107
7108     die "cannot convert iscsi path '$path', unkown format\n";
7109 }
7110
7111 sub qemu_img_convert {
7112     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7113
7114     my $storecfg = PVE::Storage::config();
7115     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7116     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7117
7118     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7119
7120     my $cachemode;
7121     my $src_path;
7122     my $src_is_iscsi = 0;
7123     my $src_format;
7124
7125     if ($src_storeid) {
7126         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7127         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7128         $src_format = qemu_img_format($src_scfg, $src_volname);
7129         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7130         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7131         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7132     } elsif (-f $src_volid) {
7133         $src_path = $src_volid;
7134         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7135             $src_format = $1;
7136         }
7137     }
7138
7139     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7140
7141     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7142     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7143     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7144     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7145
7146     my $cmd = [];
7147     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7148     push @$cmd, '-l', "snapshot.name=$snapname"
7149         if $snapname && $src_format && $src_format eq "qcow2";
7150     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7151     push @$cmd, '-T', $cachemode if defined($cachemode);
7152
7153     if ($src_is_iscsi) {
7154         push @$cmd, '--image-opts';
7155         $src_path = convert_iscsi_path($src_path);
7156     } elsif ($src_format) {
7157         push @$cmd, '-f', $src_format;
7158     }
7159
7160     if ($dst_is_iscsi) {
7161         push @$cmd, '--target-image-opts';
7162         $dst_path = convert_iscsi_path($dst_path);
7163     } else {
7164         push @$cmd, '-O', $dst_format;
7165     }
7166
7167     push @$cmd, $src_path;
7168
7169     if (!$dst_is_iscsi && $is_zero_initialized) {
7170         push @$cmd, "zeroinit:$dst_path";
7171     } else {
7172         push @$cmd, $dst_path;
7173     }
7174
7175     my $parser = sub {
7176         my $line = shift;
7177         if($line =~ m/\((\S+)\/100\%\)/){
7178             my $percent = $1;
7179             my $transferred = int($size * $percent / 100);
7180             my $total_h = render_bytes($size, 1);
7181             my $transferred_h = render_bytes($transferred, 1);
7182
7183             print "transferred $transferred_h of $total_h ($percent%)\n";
7184         }
7185
7186     };
7187
7188     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7189     my $err = $@;
7190     die "copy failed: $err" if $err;
7191 }
7192
7193 sub qemu_img_format {
7194     my ($scfg, $volname) = @_;
7195
7196     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7197         return $1;
7198     } else {
7199         return "raw";
7200     }
7201 }
7202
7203 sub qemu_drive_mirror {
7204     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7205
7206     $jobs = {} if !$jobs;
7207
7208     my $qemu_target;
7209     my $format;
7210     $jobs->{"drive-$drive"} = {};
7211
7212     if ($dst_volid =~ /^nbd:/) {
7213         $qemu_target = $dst_volid;
7214         $format = "nbd";
7215     } else {
7216         my $storecfg = PVE::Storage::config();
7217         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7218
7219         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7220
7221         $format = qemu_img_format($dst_scfg, $dst_volname);
7222
7223         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7224
7225         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7226     }
7227
7228     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7229     $opts->{format} = $format if $format;
7230
7231     if (defined($src_bitmap)) {
7232         $opts->{sync} = 'incremental';
7233         $opts->{bitmap} = $src_bitmap;
7234         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7235     }
7236
7237     if (defined($bwlimit)) {
7238         $opts->{speed} = $bwlimit * 1024;
7239         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7240     } else {
7241         print "drive mirror is starting for drive-$drive\n";
7242     }
7243
7244     # if a job already runs for this device we get an error, catch it for cleanup
7245     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7246     if (my $err = $@) {
7247         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7248         warn "$@\n" if $@;
7249         die "mirroring error: $err\n";
7250     }
7251
7252     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7253 }
7254
7255 # $completion can be either
7256 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7257 # 'cancel': wait until all jobs are ready, block-job-cancel them
7258 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7259 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7260 sub qemu_drive_mirror_monitor {
7261     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7262
7263     $completion //= 'complete';
7264     $op //= "mirror";
7265
7266     eval {
7267         my $err_complete = 0;
7268
7269         my $starttime = time ();
7270         while (1) {
7271             die "block job ('$op') timed out\n" if $err_complete > 300;
7272
7273             my $stats = mon_cmd($vmid, "query-block-jobs");
7274             my $ctime = time();
7275
7276             my $running_jobs = {};
7277             for my $stat (@$stats) {
7278                 next if $stat->{type} ne $op;
7279                 $running_jobs->{$stat->{device}} = $stat;
7280             }
7281
7282             my $readycounter = 0;
7283
7284             for my $job_id (sort keys %$jobs) {
7285                 my $job = $running_jobs->{$job_id};
7286
7287                 my $vanished = !defined($job);
7288                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7289                 if($complete || ($vanished && $completion eq 'auto')) {
7290                     print "$job_id: $op-job finished\n";
7291                     delete $jobs->{$job_id};
7292                     next;
7293                 }
7294
7295                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7296
7297                 my $busy = $job->{busy};
7298                 my $ready = $job->{ready};
7299                 if (my $total = $job->{len}) {
7300                     my $transferred = $job->{offset} || 0;
7301                     my $remaining = $total - $transferred;
7302                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7303
7304                     my $duration = $ctime - $starttime;
7305                     my $total_h = render_bytes($total, 1);
7306                     my $transferred_h = render_bytes($transferred, 1);
7307
7308                     my $status = sprintf(
7309                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7310                         render_duration($duration),
7311                     );
7312
7313                     if ($ready) {
7314                         if ($busy) {
7315                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7316                         } else {
7317                             $status .= ", ready";
7318                         }
7319                     }
7320                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7321                     $jobs->{$job_id}->{ready} = $ready;
7322                 }
7323
7324                 $readycounter++ if $job->{ready};
7325             }
7326
7327             last if scalar(keys %$jobs) == 0;
7328
7329             if ($readycounter == scalar(keys %$jobs)) {
7330                 print "all '$op' jobs are ready\n";
7331
7332                 # do the complete later (or has already been done)
7333                 last if $completion eq 'skip' || $completion eq 'auto';
7334
7335                 if ($vmiddst && $vmiddst != $vmid) {
7336                     my $agent_running = $qga && qga_check_running($vmid);
7337                     if ($agent_running) {
7338                         print "freeze filesystem\n";
7339                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7340                     } else {
7341                         print "suspend vm\n";
7342                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7343                     }
7344
7345                     # if we clone a disk for a new target vm, we don't switch the disk
7346                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7347
7348                     if ($agent_running) {
7349                         print "unfreeze filesystem\n";
7350                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7351                     } else {
7352                         print "resume vm\n";
7353                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7354                     }
7355
7356                     last;
7357                 } else {
7358
7359                     for my $job_id (sort keys %$jobs) {
7360                         # try to switch the disk if source and destination are on the same guest
7361                         print "$job_id: Completing block job_id...\n";
7362
7363                         my $op;
7364                         if ($completion eq 'complete') {
7365                             $op = 'block-job-complete';
7366                         } elsif ($completion eq 'cancel') {
7367                             $op = 'block-job-cancel';
7368                         } else {
7369                             die "invalid completion value: $completion\n";
7370                         }
7371                         eval { mon_cmd($vmid, $op, device => $job_id) };
7372                         if ($@ =~ m/cannot be completed/) {
7373                             print "$job_id: block job cannot be completed, trying again.\n";
7374                             $err_complete++;
7375                         }else {
7376                             print "$job_id: Completed successfully.\n";
7377                             $jobs->{$job_id}->{complete} = 1;
7378                         }
7379                     }
7380                 }
7381             }
7382             sleep 1;
7383         }
7384     };
7385     my $err = $@;
7386
7387     if ($err) {
7388         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7389         die "block job ($op) error: $err";
7390     }
7391 }
7392
7393 sub qemu_blockjobs_cancel {
7394     my ($vmid, $jobs) = @_;
7395
7396     foreach my $job (keys %$jobs) {
7397         print "$job: Cancelling block job\n";
7398         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7399         $jobs->{$job}->{cancel} = 1;
7400     }
7401
7402     while (1) {
7403         my $stats = mon_cmd($vmid, "query-block-jobs");
7404
7405         my $running_jobs = {};
7406         foreach my $stat (@$stats) {
7407             $running_jobs->{$stat->{device}} = $stat;
7408         }
7409
7410         foreach my $job (keys %$jobs) {
7411
7412             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7413                 print "$job: Done.\n";
7414                 delete $jobs->{$job};
7415             }
7416         }
7417
7418         last if scalar(keys %$jobs) == 0;
7419
7420         sleep 1;
7421     }
7422 }
7423
7424 sub clone_disk {
7425     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7426         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7427
7428     my $newvolid;
7429
7430     if (!$full) {
7431         print "create linked clone of drive $drivename ($drive->{file})\n";
7432         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7433         push @$newvollist, $newvolid;
7434     } else {
7435
7436         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7437         $storeid = $storage if $storage;
7438
7439         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7440
7441         print "create full clone of drive $drivename ($drive->{file})\n";
7442         my $name = undef;
7443         my $size = undef;
7444         if (drive_is_cloudinit($drive)) {
7445             $name = "vm-$newvmid-cloudinit";
7446             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7447             if ($scfg->{path}) {
7448                 $name .= ".$dst_format";
7449             }
7450             $snapname = undef;
7451             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7452         } elsif ($drivename eq 'efidisk0') {
7453             $size = get_efivars_size($conf);
7454         } elsif ($drivename eq 'tpmstate0') {
7455             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7456         } else {
7457             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7458         }
7459         $newvolid = PVE::Storage::vdisk_alloc(
7460             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7461         );
7462         push @$newvollist, $newvolid;
7463
7464         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7465
7466         if (drive_is_cloudinit($drive)) {
7467             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7468             # if this is the case, we have to complete any block-jobs still there from
7469             # previous drive-mirrors
7470             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7471                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7472             }
7473             goto no_data_clone;
7474         }
7475
7476         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7477         if (!$running || $snapname) {
7478             # TODO: handle bwlimits
7479             if ($drivename eq 'efidisk0') {
7480                 # the relevant data on the efidisk may be smaller than the source
7481                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7482                 # that is given by the OVMF_VARS.fd
7483                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7484                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7485
7486                 # better for Ceph if block size is not too small, see bug #3324
7487                 my $bs = 1024*1024;
7488
7489                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7490                     "if=$src_path", "of=$dst_path"]);
7491             } else {
7492                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7493             }
7494         } else {
7495
7496             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7497
7498             my $kvmver = get_running_qemu_version ($vmid);
7499             if (!min_version($kvmver, 2, 7)) {
7500                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7501                     if $drive->{iothread};
7502             }
7503
7504             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7505                 $completion, $qga, $bwlimit);
7506         }
7507     }
7508
7509 no_data_clone:
7510     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7511
7512     my $disk = $drive;
7513     $disk->{format} = undef;
7514     $disk->{file} = $newvolid;
7515     $disk->{size} = $size if defined($size);
7516
7517     return $disk;
7518 }
7519
7520 sub get_running_qemu_version {
7521     my ($vmid) = @_;
7522     my $res = mon_cmd($vmid, "query-version");
7523     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7524 }
7525
7526 sub qemu_use_old_bios_files {
7527     my ($machine_type) = @_;
7528
7529     return if !$machine_type;
7530
7531     my $use_old_bios_files = undef;
7532
7533     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7534         $machine_type = $1;
7535         $use_old_bios_files = 1;
7536     } else {
7537         my $version = extract_version($machine_type, kvm_user_version());
7538         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7539         # load new efi bios files on migration. So this hack is required to allow
7540         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7541         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7542         $use_old_bios_files = !min_version($version, 2, 4);
7543     }
7544
7545     return ($use_old_bios_files, $machine_type);
7546 }
7547
7548 sub get_efivars_size {
7549     my ($conf) = @_;
7550     my $arch = get_vm_arch($conf);
7551     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7552     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk);
7553     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7554     return -s $ovmf_vars;
7555 }
7556
7557 sub update_efidisk_size {
7558     my ($conf) = @_;
7559
7560     return if !defined($conf->{efidisk0});
7561
7562     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7563     $disk->{size} = get_efivars_size($conf);
7564     $conf->{efidisk0} = print_drive($disk);
7565
7566     return;
7567 }
7568
7569 sub update_tpmstate_size {
7570     my ($conf) = @_;
7571
7572     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7573     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7574     $conf->{tpmstate0} = print_drive($disk);
7575 }
7576
7577 sub create_efidisk($$$$$$) {
7578     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk) = @_;
7579
7580     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk);
7581     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7582
7583     my $vars_size_b = -s $ovmf_vars;
7584     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7585     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7586     PVE::Storage::activate_volumes($storecfg, [$volid]);
7587
7588     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7589     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7590
7591     return ($volid, $size/1024);
7592 }
7593
7594 sub vm_iothreads_list {
7595     my ($vmid) = @_;
7596
7597     my $res = mon_cmd($vmid, 'query-iothreads');
7598
7599     my $iothreads = {};
7600     foreach my $iothread (@$res) {
7601         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7602     }
7603
7604     return $iothreads;
7605 }
7606
7607 sub scsihw_infos {
7608     my ($conf, $drive) = @_;
7609
7610     my $maxdev = 0;
7611
7612     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7613         $maxdev = 7;
7614     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7615         $maxdev = 1;
7616     } else {
7617         $maxdev = 256;
7618     }
7619
7620     my $controller = int($drive->{index} / $maxdev);
7621     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7622         ? "virtioscsi"
7623         : "scsihw";
7624
7625     return ($maxdev, $controller, $controller_prefix);
7626 }
7627
7628 sub windows_version {
7629     my ($ostype) = @_;
7630
7631     return 0 if !$ostype;
7632
7633     my $winversion = 0;
7634
7635     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7636         $winversion = 5;
7637     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7638         $winversion = 6;
7639     } elsif ($ostype =~ m/^win(\d+)$/) {
7640         $winversion = $1;
7641     }
7642
7643     return $winversion;
7644 }
7645
7646 sub resolve_dst_disk_format {
7647         my ($storecfg, $storeid, $src_volname, $format) = @_;
7648         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7649
7650         if (!$format) {
7651             # if no target format is specified, use the source disk format as hint
7652             if ($src_volname) {
7653                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7654                 $format = qemu_img_format($scfg, $src_volname);
7655             } else {
7656                 return $defFormat;
7657             }
7658         }
7659
7660         # test if requested format is supported - else use default
7661         my $supported = grep { $_ eq $format } @$validFormats;
7662         $format = $defFormat if !$supported;
7663         return $format;
7664 }
7665
7666 # NOTE: if this logic changes, please update docs & possibly gui logic
7667 sub find_vmstate_storage {
7668     my ($conf, $storecfg) = @_;
7669
7670     # first, return storage from conf if set
7671     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7672
7673     my ($target, $shared, $local);
7674
7675     foreach_storage_used_by_vm($conf, sub {
7676         my ($sid) = @_;
7677         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7678         my $dst = $scfg->{shared} ? \$shared : \$local;
7679         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7680     });
7681
7682     # second, use shared storage where VM has at least one disk
7683     # third, use local storage where VM has at least one disk
7684     # fall back to local storage
7685     $target = $shared // $local // 'local';
7686
7687     return $target;
7688 }
7689
7690 sub generate_uuid {
7691     my ($uuid, $uuid_str);
7692     UUID::generate($uuid);
7693     UUID::unparse($uuid, $uuid_str);
7694     return $uuid_str;
7695 }
7696
7697 sub generate_smbios1_uuid {
7698     return "uuid=".generate_uuid();
7699 }
7700
7701 sub nbd_stop {
7702     my ($vmid) = @_;
7703
7704     mon_cmd($vmid, 'nbd-server-stop');
7705 }
7706
7707 sub create_reboot_request {
7708     my ($vmid) = @_;
7709     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7710         or die "failed to create reboot trigger file: $!\n";
7711     close($fh);
7712 }
7713
7714 sub clear_reboot_request {
7715     my ($vmid) = @_;
7716     my $path = "/run/qemu-server/$vmid.reboot";
7717     my $res = 0;
7718
7719     $res = unlink($path);
7720     die "could not remove reboot request for $vmid: $!"
7721         if !$res && $! != POSIX::ENOENT;
7722
7723     return $res;
7724 }
7725
7726 sub bootorder_from_legacy {
7727     my ($conf, $bootcfg) = @_;
7728
7729     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7730     my $bootindex_hash = {};
7731     my $i = 1;
7732     foreach my $o (split(//, $boot)) {
7733         $bootindex_hash->{$o} = $i*100;
7734         $i++;
7735     }
7736
7737     my $bootorder = {};
7738
7739     PVE::QemuConfig->foreach_volume($conf, sub {
7740         my ($ds, $drive) = @_;
7741
7742         if (drive_is_cdrom ($drive, 1)) {
7743             if ($bootindex_hash->{d}) {
7744                 $bootorder->{$ds} = $bootindex_hash->{d};
7745                 $bootindex_hash->{d} += 1;
7746             }
7747         } elsif ($bootindex_hash->{c}) {
7748             $bootorder->{$ds} = $bootindex_hash->{c}
7749                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7750             $bootindex_hash->{c} += 1;
7751         }
7752     });
7753
7754     if ($bootindex_hash->{n}) {
7755         for (my $i = 0; $i < $MAX_NETS; $i++) {
7756             my $netname = "net$i";
7757             next if !$conf->{$netname};
7758             $bootorder->{$netname} = $bootindex_hash->{n};
7759             $bootindex_hash->{n} += 1;
7760         }
7761     }
7762
7763     return $bootorder;
7764 }
7765
7766 # Generate default device list for 'boot: order=' property. Matches legacy
7767 # default boot order, but with explicit device names. This is important, since
7768 # the fallback for when neither 'order' nor the old format is specified relies
7769 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7770 sub get_default_bootdevices {
7771     my ($conf) = @_;
7772
7773     my @ret = ();
7774
7775     # harddisk
7776     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7777     push @ret, $first if $first;
7778
7779     # cdrom
7780     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7781     push @ret, $first if $first;
7782
7783     # network
7784     for (my $i = 0; $i < $MAX_NETS; $i++) {
7785         my $netname = "net$i";
7786         next if !$conf->{$netname};
7787         push @ret, $netname;
7788         last;
7789     }
7790
7791     return \@ret;
7792 }
7793
7794 sub device_bootorder {
7795     my ($conf) = @_;
7796
7797     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7798
7799     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7800
7801     my $bootorder = {};
7802     if (!defined($boot) || $boot->{legacy}) {
7803         $bootorder = bootorder_from_legacy($conf, $boot);
7804     } elsif ($boot->{order}) {
7805         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7806         for my $dev (PVE::Tools::split_list($boot->{order})) {
7807             $bootorder->{$dev} = $i++;
7808         }
7809     }
7810
7811     return $bootorder;
7812 }
7813
7814 sub register_qmeventd_handle {
7815     my ($vmid) = @_;
7816
7817     my $fh;
7818     my $peer = "/var/run/qmeventd.sock";
7819     my $count = 0;
7820
7821     for (;;) {
7822         $count++;
7823         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7824         last if $fh;
7825         if ($! != EINTR && $! != EAGAIN) {
7826             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7827         }
7828         if ($count > 4) {
7829             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7830               . "after $count retries\n";
7831         }
7832         usleep(25000);
7833     }
7834
7835     # send handshake to mark VM as backing up
7836     print $fh to_json({vzdump => {vmid => "$vmid"}});
7837
7838     # return handle to be closed later when inhibit is no longer required
7839     return $fh;
7840 }
7841
7842 # bash completion helper
7843
7844 sub complete_backup_archives {
7845     my ($cmdname, $pname, $cvalue) = @_;
7846
7847     my $cfg = PVE::Storage::config();
7848
7849     my $storeid;
7850
7851     if ($cvalue =~ m/^([^:]+):/) {
7852         $storeid = $1;
7853     }
7854
7855     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7856
7857     my $res = [];
7858     foreach my $id (keys %$data) {
7859         foreach my $item (@{$data->{$id}}) {
7860             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7861             push @$res, $item->{volid} if defined($item->{volid});
7862         }
7863     }
7864
7865     return $res;
7866 }
7867
7868 my $complete_vmid_full = sub {
7869     my ($running) = @_;
7870
7871     my $idlist = vmstatus();
7872
7873     my $res = [];
7874
7875     foreach my $id (keys %$idlist) {
7876         my $d = $idlist->{$id};
7877         if (defined($running)) {
7878             next if $d->{template};
7879             next if $running && $d->{status} ne 'running';
7880             next if !$running && $d->{status} eq 'running';
7881         }
7882         push @$res, $id;
7883
7884     }
7885     return $res;
7886 };
7887
7888 sub complete_vmid {
7889     return &$complete_vmid_full();
7890 }
7891
7892 sub complete_vmid_stopped {
7893     return &$complete_vmid_full(0);
7894 }
7895
7896 sub complete_vmid_running {
7897     return &$complete_vmid_full(1);
7898 }
7899
7900 sub complete_storage {
7901
7902     my $cfg = PVE::Storage::config();
7903     my $ids = $cfg->{ids};
7904
7905     my $res = [];
7906     foreach my $sid (keys %$ids) {
7907         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7908         next if !$ids->{$sid}->{content}->{images};
7909         push @$res, $sid;
7910     }
7911
7912     return $res;
7913 }
7914
7915 sub complete_migration_storage {
7916     my ($cmd, $param, $current_value, $all_args) = @_;
7917
7918     my $targetnode = @$all_args[1];
7919
7920     my $cfg = PVE::Storage::config();
7921     my $ids = $cfg->{ids};
7922
7923     my $res = [];
7924     foreach my $sid (keys %$ids) {
7925         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7926         next if !$ids->{$sid}->{content}->{images};
7927         push @$res, $sid;
7928     }
7929
7930     return $res;
7931 }
7932
7933 sub vm_is_paused {
7934     my ($vmid) = @_;
7935     my $qmpstatus = eval {
7936         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7937         mon_cmd($vmid, "query-status");
7938     };
7939     warn "$@\n" if $@;
7940     return $qmpstatus && $qmpstatus->{status} eq "paused";
7941 }
7942
7943 sub check_volume_storage_type {
7944     my ($storecfg, $vol) = @_;
7945
7946     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7947     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7948     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7949
7950     die "storage '$storeid' does not support content-type '$vtype'\n"
7951         if !$scfg->{content}->{$vtype};
7952
7953     return 1;
7954 }
7955
7956 1;