PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         default => [
  76             "$EDK2_FW_BASE/OVMF_CODE.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS.fd",
  78         ],
  79     },
  80     aarch64 => {
  81         default => [
  82             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  83             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  84         ],
  85     },
  86 };
  87
  88 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  89
  90 # Note about locking: we use flock on the config file protect
  91 # against concurent actions.
  92 # Aditionaly, we have a 'lock' setting in the config file. This
  93 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  94 # allowed when such lock is set. But you can ignore this kind of
  95 # lock with the --skiplock flag.
  96
  97 cfs_register_file('/qemu-server/',
  98                   \&parse_vm_config,
  99                   \&write_vm_config);
 100
 101 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 102     description => "Some command save/restore state from this location.",
 103     type => 'string',
 104     maxLength => 128,
 105     optional => 1,
 106 });
 107
 108 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 109         description => "Specifies the Qemu machine type.",
 110         type => 'string',
 111         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 112         maxLength => 40,
 113         optional => 1,
 114 });
 115
 116
 117 sub map_storage {
 118     my ($map, $source) = @_;
 119
 120     return $source if !defined($map);
 121
 122     return $map->{entries}->{$source}
 123         if $map->{entries} && defined($map->{entries}->{$source});
 124
 125     return $map->{default} if $map->{default};
 126
 127     # identity (fallback)
 128     return $source;
 129 }
 130
 131 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 132     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 133     type => 'string',
 134     format => 'storagepair-list',
 135     optional => 1,
 136 });
 137
 138 #no warnings 'redefine';
 139
 140 my $nodename_cache;
 141 sub nodename {
 142     $nodename_cache //= PVE::INotify::nodename();
 143     return $nodename_cache;
 144 }
 145
 146 my $watchdog_fmt = {
 147     model => {
 148         default_key => 1,
 149         type => 'string',
 150         enum => [qw(i6300esb ib700)],
 151         description => "Watchdog type to emulate.",
 152         default => 'i6300esb',
 153         optional => 1,
 154     },
 155     action => {
 156         type => 'string',
 157         enum => [qw(reset shutdown poweroff pause debug none)],
 158         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 159         optional => 1,
 160     },
 161 };
 162 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 163
 164 my $agent_fmt = {
 165     enabled => {
 166         description => "Enable/disable Qemu GuestAgent.",
 167         type => 'boolean',
 168         default => 0,
 169         default_key => 1,
 170     },
 171     fstrim_cloned_disks => {
 172         description => "Run fstrim after moving a disk or migrating the VM.",
 173         type => 'boolean',
 174         optional => 1,
 175         default => 0
 176     },
 177     type => {
 178         description => "Select the agent type",
 179         type => 'string',
 180         default => 'virtio',
 181         optional => 1,
 182         enum => [qw(virtio isa)],
 183     },
 184 };
 185
 186 my $vga_fmt = {
 187     type => {
 188         description => "Select the VGA type.",
 189         type => 'string',
 190         default => 'std',
 191         optional => 1,
 192         default_key => 1,
 193         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 194     },
 195     memory => {
 196         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 197         type => 'integer',
 198         optional => 1,
 199         minimum => 4,
 200         maximum => 512,
 201     },
 202 };
 203
 204 my $ivshmem_fmt = {
 205     size => {
 206         type => 'integer',
 207         minimum => 1,
 208         description => "The size of the file in MB.",
 209     },
 210     name => {
 211         type => 'string',
 212         pattern => '[a-zA-Z0-9\-]+',
 213         optional => 1,
 214         format_description => 'string',
 215         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 216     },
 217 };
 218
 219 my $audio_fmt = {
 220     device => {
 221         type => 'string',
 222         enum => [qw(ich9-intel-hda intel-hda AC97)],
 223         description =>  "Configure an audio device."
 224     },
 225     driver =>  {
 226         type => 'string',
 227         enum => ['spice', 'none'],
 228         default => 'spice',
 229         optional => 1,
 230         description => "Driver backend for the audio device."
 231     },
 232 };
 233
 234 my $spice_enhancements_fmt = {
 235     foldersharing => {
 236         type => 'boolean',
 237         optional => 1,
 238         default => '0',
 239         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 240     },
 241     videostreaming =>  {
 242         type => 'string',
 243         enum => ['off', 'all', 'filter'],
 244         default => 'off',
 245         optional => 1,
 246         description => "Enable video streaming. Uses compression for detected video streams."
 247     },
 248 };
 249
 250 my $rng_fmt = {
 251     source => {
 252         type => 'string',
 253         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 254         default_key => 1,
 255         description => "The file on the host to gather entropy from. In most"
 256                      . " cases /dev/urandom should be preferred over /dev/random"
 257                      . " to avoid entropy-starvation issues on the host. Using"
 258                      . " urandom does *not* decrease security in any meaningful"
 259                      . " way, as it's still seeded from real entropy, and the"
 260                      . " bytes provided will most likely be mixed with real"
 261                      . " entropy on the guest as well. /dev/hwrng can be used"
 262                      . " to pass through a hardware RNG from the host.",
 263     },
 264     max_bytes => {
 265         type => 'integer',
 266         description => "Maximum bytes of entropy injected into the guest every"
 267                      . " 'period' milliseconds. Prefer a lower value when using"
 268                      . " /dev/random as source. Use 0 to disable limiting"
 269                      . " (potentially dangerous!).",
 270         optional => 1,
 271
 272         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 273         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 274         # of overwhelming the host, provided we're reading from /dev/urandom
 275         default => 1024,
 276     },
 277     period => {
 278         type => 'integer',
 279         description => "Every 'period' milliseconds the entropy-injection quota"
 280                      . " is reset, allowing the guest to retrieve another"
 281                      . " 'max_bytes' of entropy.",
 282         optional => 1,
 283         default => 1000,
 284     },
 285 };
 286
 287 my $confdesc = {
 288     onboot => {
 289         optional => 1,
 290         type => 'boolean',
 291         description => "Specifies whether a VM will be started during system bootup.",
 292         default => 0,
 293     },
 294     autostart => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Automatic restart after crash (currently ignored).",
 298         default => 0,
 299     },
 300     hotplug => {
 301         optional => 1,
 302         type => 'string', format => 'pve-hotplug-features',
 303         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 304         default => 'network,disk,usb',
 305     },
 306     reboot => {
 307         optional => 1,
 308         type => 'boolean',
 309         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 310         default => 1,
 311     },
 312     lock => {
 313         optional => 1,
 314         type => 'string',
 315         description => "Lock/unlock the VM.",
 316         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 317     },
 318     cpulimit => {
 319         optional => 1,
 320         type => 'number',
 321         description => "Limit of CPU usage.",
 322         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 323         minimum => 0,
 324         maximum => 128,
 325         default => 0,
 326     },
 327     cpuunits => {
 328         optional => 1,
 329         type => 'integer',
 330         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 331         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 332             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 333             ." weights of all the other running VMs.",
 334         minimum => 2,
 335         maximum => 262144,
 336         default => 'cgroup v1: 1024, cgroup v2: 100',
 337     },
 338     memory => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 342         minimum => 16,
 343         default => 512,
 344     },
 345     balloon => {
 346         optional => 1,
 347         type => 'integer',
 348         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 349         minimum => 0,
 350     },
 351     shares => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 355         minimum => 0,
 356         maximum => 50000,
 357         default => 1000,
 358     },
 359     keyboard => {
 360         optional => 1,
 361         type => 'string',
 362         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 363                        "It should not be necessary to set it.",
 364         enum => PVE::Tools::kvmkeymaplist(),
 365         default => undef,
 366     },
 367     name => {
 368         optional => 1,
 369         type => 'string', format => 'dns-name',
 370         description => "Set a name for the VM. Only used on the configuration web interface.",
 371     },
 372     scsihw => {
 373         optional => 1,
 374         type => 'string',
 375         description => "SCSI controller model",
 376         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 377         default => 'lsi',
 378     },
 379     description => {
 380         optional => 1,
 381         type => 'string',
 382         description => "Description for the VM. Shown in the web-interface VM's summary."
 383             ." This is saved as comment inside the configuration file.",
 384         maxLength => 1024 * 8,
 385     },
 386     ostype => {
 387         optional => 1,
 388         type => 'string',
 389         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 390         description => "Specify guest operating system.",
 391         verbose_description => <<EODESC,
 392 Specify guest operating system. This is used to enable special
 393 optimization/features for specific operating systems:
 394
 395 [horizontal]
 396 other;; unspecified OS
 397 wxp;; Microsoft Windows XP
 398 w2k;; Microsoft Windows 2000
 399 w2k3;; Microsoft Windows 2003
 400 w2k8;; Microsoft Windows 2008
 401 wvista;; Microsoft Windows Vista
 402 win7;; Microsoft Windows 7
 403 win8;; Microsoft Windows 8/2012/2012r2
 404 win10;; Microsoft Windows 10/2016/2019
 405 l24;; Linux 2.4 Kernel
 406 l26;; Linux 2.6 - 5.X Kernel
 407 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 408 EODESC
 409     },
 410     boot => {
 411         optional => 1,
 412         type => 'string', format => 'pve-qm-boot',
 413         description => "Specify guest boot order. Use with 'order=', usage with"
 414                      . " no key or 'legacy=' is deprecated.",
 415     },
 416     bootdisk => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-bootdisk',
 419         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 420         pattern => '(ide|sata|scsi|virtio)\d+',
 421     },
 422     smp => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of CPUs. Please use option -sockets instead.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     sockets => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPU sockets.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     cores => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of cores per socket.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     numa => {
 444         optional => 1,
 445         type => 'boolean',
 446         description => "Enable/disable NUMA.",
 447         default => 0,
 448     },
 449     hugepages => {
 450         optional => 1,
 451         type => 'string',
 452         description => "Enable/disable hugepages memory.",
 453         enum => [qw(any 2 1024)],
 454     },
 455     keephugepages => {
 456         optional => 1,
 457         type => 'boolean',
 458         default => 0,
 459         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 460             ." after VM shutdown and can be used for subsequent starts.",
 461     },
 462     vcpus => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "Number of hotplugged vcpus.",
 466         minimum => 1,
 467         default => 0,
 468     },
 469     acpi => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable ACPI.",
 473         default => 1,
 474     },
 475     agent => {
 476         optional => 1,
 477         description => "Enable/disable Qemu GuestAgent and its properties.",
 478         type => 'string',
 479         format => $agent_fmt,
 480     },
 481     kvm => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Enable/disable KVM hardware virtualization.",
 485         default => 1,
 486     },
 487     tdf => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Enable/disable time drift fix.",
 491         default => 0,
 492     },
 493     localtime => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Set the real time clock to local time. This is enabled by default if ostype"
 497             ." indicates a Microsoft OS.",
 498     },
 499     freeze => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 503     },
 504     vga => {
 505         optional => 1,
 506         type => 'string', format => $vga_fmt,
 507         description => "Configure the VGA hardware.",
 508         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 509             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 510             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 511             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 512             ." display server. For win* OS you can select how many independent displays you want,"
 513             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 514             ." using a serial device as terminal.",
 515     },
 516     watchdog => {
 517         optional => 1,
 518         type => 'string', format => 'pve-qm-watchdog',
 519         description => "Create a virtual hardware watchdog device.",
 520         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 521             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 522             ." else the watchdog will reset the guest (or execute the respective action specified)",
 523     },
 524     startdate => {
 525         optional => 1,
 526         type => 'string',
 527         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 528         description => "Set the initial date of the real time clock. Valid format for date are:"
 529             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 530         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 531         default => 'now',
 532     },
 533     startup =>  get_standard_option('pve-startup-order'),
 534     template => {
 535         optional => 1,
 536         type => 'boolean',
 537         description => "Enable/disable Template.",
 538         default => 0,
 539     },
 540     args => {
 541         optional => 1,
 542         type => 'string',
 543         description => "Arbitrary arguments passed to kvm.",
 544         verbose_description => <<EODESCR,
 545 Arbitrary arguments passed to kvm, for example:
 546
 547 args: -no-reboot -no-hpet
 548
 549 NOTE: this option is for experts only.
 550 EODESCR
 551     },
 552     tablet => {
 553         optional => 1,
 554         type => 'boolean',
 555         default => 1,
 556         description => "Enable/disable the USB tablet device.",
 557         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 558             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 559             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 560             ." may consider disabling this to save some context switches. This is turned off by"
 561             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 562     },
 563     migrate_speed => {
 564         optional => 1,
 565         type => 'integer',
 566         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 567         minimum => 0,
 568         default => 0,
 569     },
 570     migrate_downtime => {
 571         optional => 1,
 572         type => 'number',
 573         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 574         minimum => 0,
 575         default => 0.1,
 576     },
 577     cdrom => {
 578         optional => 1,
 579         type => 'string', format => 'pve-qm-ide',
 580         typetext => '<volume>',
 581         description => "This is an alias for option -ide2",
 582     },
 583     cpu => {
 584         optional => 1,
 585         description => "Emulated CPU type.",
 586         type => 'string',
 587         format => 'pve-vm-cpu-conf',
 588     },
 589     parent => get_standard_option('pve-snapshot-name', {
 590         optional => 1,
 591         description => "Parent snapshot name. This is used internally, and should not be modified.",
 592     }),
 593     snaptime => {
 594         optional => 1,
 595         description => "Timestamp for snapshots.",
 596         type => 'integer',
 597         minimum => 0,
 598     },
 599     vmstate => {
 600         optional => 1,
 601         type => 'string', format => 'pve-volume-id',
 602         description => "Reference to a volume which stores the VM state. This is used internally"
 603             ." for snapshots.",
 604     },
 605     vmstatestorage => get_standard_option('pve-storage-id', {
 606         description => "Default storage for VM state volumes/files.",
 607         optional => 1,
 608     }),
 609     runningmachine => get_standard_option('pve-qemu-machine', {
 610         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 611             ." for snapshots.",
 612     }),
 613     runningcpu => {
 614         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 615             ." internally for snapshots.",
 616         optional => 1,
 617         type => 'string',
 618         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 619         format_description => 'QEMU -cpu parameter'
 620     },
 621     machine => get_standard_option('pve-qemu-machine'),
 622     arch => {
 623         description => "Virtual processor architecture. Defaults to the host.",
 624         optional => 1,
 625         type => 'string',
 626         enum => [qw(x86_64 aarch64)],
 627     },
 628     smbios1 => {
 629         description => "Specify SMBIOS type 1 fields.",
 630         type => 'string', format => 'pve-qm-smbios1',
 631         maxLength => 512,
 632         optional => 1,
 633     },
 634     protection => {
 635         optional => 1,
 636         type => 'boolean',
 637         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 638             ." remove disk operations.",
 639         default => 0,
 640     },
 641     bios => {
 642         optional => 1,
 643         type => 'string',
 644         enum => [ qw(seabios ovmf) ],
 645         description => "Select BIOS implementation.",
 646         default => 'seabios',
 647     },
 648     vmgenid => {
 649         type => 'string',
 650         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 651         format_description => 'UUID',
 652         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 653             ." to disable explicitly.",
 654         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 655             ." value identifier to the guest OS. This allows to notify the guest operating system"
 656             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 657             ." execution or creation from a template). The guest operating system notices the"
 658             ." change, and is then able to react as appropriate by marking its copies of"
 659             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 660             ."Note that auto-creation only works when done through API/CLI create or update methods"
 661             .", but not when manually editing the config file.",
 662         default => "1 (autogenerated)",
 663         optional => 1,
 664     },
 665     hookscript => {
 666         type => 'string',
 667         format => 'pve-volume-id',
 668         optional => 1,
 669         description => "Script that will be executed during various steps in the vms lifetime.",
 670     },
 671     ivshmem => {
 672         type => 'string',
 673         format => $ivshmem_fmt,
 674         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 675             ." the host.",
 676         optional => 1,
 677     },
 678     audio0 => {
 679         type => 'string',
 680         format => $audio_fmt,
 681         description => "Configure a audio device, useful in combination with QXL/Spice.",
 682         optional => 1
 683     },
 684     spice_enhancements => {
 685         type => 'string',
 686         format => $spice_enhancements_fmt,
 687         description => "Configure additional enhancements for SPICE.",
 688         optional => 1
 689     },
 690     tags => {
 691         type => 'string', format => 'pve-tag-list',
 692         description => 'Tags of the VM. This is only meta information.',
 693         optional => 1,
 694     },
 695     rng0 => {
 696         type => 'string',
 697         format => $rng_fmt,
 698         description => "Configure a VirtIO-based Random Number Generator.",
 699         optional => 1,
 700     },
 701 };
 702
 703 my $cicustom_fmt = {
 704     meta => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all meta data passed to the VM via"
 708             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712     network => {
 713         type => 'string',
 714         optional => 1,
 715         description => 'Specify a custom file containing all network data passed to the VM via'
 716             .' cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via'
 724             .' cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728 };
 729 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 730
 731 my $confdesc_cloudinit = {
 732     citype => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'Specifies the cloud-init configuration format. The default depends on the'
 736             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 737             .' and `configdrive2` for windows.',
 738         enum => ['configdrive2', 'nocloud', 'opennebula'],
 739     },
 740     ciuser => {
 741         optional => 1,
 742         type => 'string',
 743         description => "cloud-init: User name to change ssh keys and password for instead of the"
 744             ." image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not'
 750             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 751             .' support hashed passwords.',
 752     },
 753     cicustom => {
 754         optional => 1,
 755         type => 'string',
 756         description => 'cloud-init: Specify custom files to replace the automatically generated'
 757             .' ones at start.',
 758         format => 'pve-qm-cicustom',
 759     },
 760     searchdomain => {
 761         optional => 1,
 762         type => 'string',
 763         description => "cloud-init: Sets DNS search domains for a container. Create will'
 764             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 765             .' are set.",
 766     },
 767     nameserver => {
 768         optional => 1,
 769         type => 'string', format => 'address-list',
 770         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 771             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 772             .' are set.",
 773     },
 774     sshkeys => {
 775         optional => 1,
 776         type => 'string',
 777         format => 'urlencoded',
 778         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 779     },
 780 };
 781
 782 # what about other qemu settings ?
 783 #cpu => 'string',
 784 #machine => 'string',
 785 #fda => 'file',
 786 #fdb => 'file',
 787 #mtdblock => 'file',
 788 #sd => 'file',
 789 #pflash => 'file',
 790 #snapshot => 'bool',
 791 #bootp => 'file',
 792 ##tftp => 'dir',
 793 ##smb => 'dir',
 794 #kernel => 'file',
 795 #append => 'string',
 796 #initrd => 'file',
 797 ##soundhw => 'string',
 798
 799 while (my ($k, $v) = each %$confdesc) {
 800     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 801 }
 802
 803 my $MAX_USB_DEVICES = 5;
 804 my $MAX_NETS = 32;
 805 my $MAX_SERIAL_PORTS = 4;
 806 my $MAX_PARALLEL_PORTS = 3;
 807 my $MAX_NUMA = 8;
 808
 809 my $numa_fmt = {
 810     cpus => {
 811         type => "string",
 812         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 813         description => "CPUs accessing this NUMA node.",
 814         format_description => "id[-id];...",
 815     },
 816     memory => {
 817         type => "number",
 818         description => "Amount of memory this NUMA node provides.",
 819         optional => 1,
 820     },
 821     hostnodes => {
 822         type => "string",
 823         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 824         description => "Host NUMA nodes to use.",
 825         format_description => "id[-id];...",
 826         optional => 1,
 827     },
 828     policy => {
 829         type => 'string',
 830         enum => [qw(preferred bind interleave)],
 831         description => "NUMA allocation policy.",
 832         optional => 1,
 833     },
 834 };
 835 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 836 my $numadesc = {
 837     optional => 1,
 838     type => 'string', format => $numa_fmt,
 839     description => "NUMA topology.",
 840 };
 841 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 842
 843 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 844     $confdesc->{"numa$i"} = $numadesc;
 845 }
 846
 847 my $nic_model_list = [
 848     'e1000',
 849     'e1000-82540em',
 850     'e1000-82544gc',
 851     'e1000-82545em',
 852     'e1000e',
 853     'i82551',
 854     'i82557b',
 855     'i82559er',
 856     'ne2k_isa',
 857     'ne2k_pci',
 858     'pcnet',
 859     'rtl8139',
 860     'virtio',
 861     'vmxnet3',
 862 ];
 863 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 864
 865 my $net_fmt_bridge_descr = <<__EOD__;
 866 Bridge to attach the network device to. The Proxmox VE standard bridge
 867 is called 'vmbr0'.
 868
 869 If you do not specify a bridge, we create a kvm user (NATed) network
 870 device, which provides DHCP and DNS services. The following addresses
 871 are used:
 872
 873  10.0.2.2   Gateway
 874  10.0.2.3   DNS Server
 875  10.0.2.4   SMB Server
 876
 877 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 878 __EOD__
 879
 880 my $net_fmt = {
 881     macaddr  => get_standard_option('mac-addr', {
 882         description => "MAC address. That address must be unique withing your network. This is"
 883             ." automatically generated if not specified.",
 884     }),
 885     model => {
 886         type => 'string',
 887         description => "Network Card Model. The 'virtio' model provides the best performance with"
 888             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 889             ." best to use 'e1000'.",
 890         enum => $nic_model_list,
 891         default_key => 1,
 892     },
 893     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 894     bridge => {
 895         type => 'string',
 896         description => $net_fmt_bridge_descr,
 897         format_description => 'bridge',
 898         pattern => '[-_.\w\d]+',
 899         optional => 1,
 900     },
 901     queues => {
 902         type => 'integer',
 903         minimum => 0, maximum => 16,
 904         description => 'Number of packet queues to be used on the device.',
 905         optional => 1,
 906     },
 907     rate => {
 908         type => 'number',
 909         minimum => 0,
 910         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 911         optional => 1,
 912     },
 913     tag => {
 914         type => 'integer',
 915         minimum => 1, maximum => 4094,
 916         description => 'VLAN tag to apply to packets on this interface.',
 917         optional => 1,
 918     },
 919     trunks => {
 920         type => 'string',
 921         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 922         description => 'VLAN trunks to pass through this interface.',
 923         format_description => 'vlanid[;vlanid...]',
 924         optional => 1,
 925     },
 926     firewall => {
 927         type => 'boolean',
 928         description => 'Whether this interface should be protected by the firewall.',
 929         optional => 1,
 930     },
 931     link_down => {
 932         type => 'boolean',
 933         description => 'Whether this interface should be disconnected (like pulling the plug).',
 934         optional => 1,
 935     },
 936     mtu => {
 937         type => 'integer',
 938         minimum => 1, maximum => 65520,
 939         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 940         optional => 1,
 941     },
 942 };
 943
 944 my $netdesc = {
 945     optional => 1,
 946     type => 'string', format => $net_fmt,
 947     description => "Specify network devices.",
 948 };
 949
 950 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 951
 952 my $ipconfig_fmt = {
 953     ip => {
 954         type => 'string',
 955         format => 'pve-ipv4-config',
 956         format_description => 'IPv4Format/CIDR',
 957         description => 'IPv4 address in CIDR format.',
 958         optional => 1,
 959         default => 'dhcp',
 960     },
 961     gw => {
 962         type => 'string',
 963         format => 'ipv4',
 964         format_description => 'GatewayIPv4',
 965         description => 'Default gateway for IPv4 traffic.',
 966         optional => 1,
 967         requires => 'ip',
 968     },
 969     ip6 => {
 970         type => 'string',
 971         format => 'pve-ipv6-config',
 972         format_description => 'IPv6Format/CIDR',
 973         description => 'IPv6 address in CIDR format.',
 974         optional => 1,
 975         default => 'dhcp',
 976     },
 977     gw6 => {
 978         type => 'string',
 979         format => 'ipv6',
 980         format_description => 'GatewayIPv6',
 981         description => 'Default gateway for IPv6 traffic.',
 982         optional => 1,
 983         requires => 'ip6',
 984     },
 985 };
 986 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 987 my $ipconfigdesc = {
 988     optional => 1,
 989     type => 'string', format => 'pve-qm-ipconfig',
 990     description => <<'EODESCR',
 991 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 992
 993 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 994
 995 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 996 gateway should be provided.
 997 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 998 cloud-init 19.4 or newer.
 999
1000 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1001 dhcp on IPv4.
1002 EODESCR
1003 };
1004 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1005
1006 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1007     $confdesc->{"net$i"} = $netdesc;
1008     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1009 }
1010
1011 foreach my $key (keys %$confdesc_cloudinit) {
1012     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1013 }
1014
1015 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1016 sub verify_volume_id_or_qm_path {
1017     my ($volid, $noerr) = @_;
1018
1019     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1020         return $volid;
1021     }
1022
1023     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1024     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1025     if ($@) {
1026         return if $noerr;
1027         die $@;
1028     }
1029     return $volid;
1030 }
1031
1032 my $usb_fmt = {
1033     host => {
1034         default_key => 1,
1035         type => 'string', format => 'pve-qm-usb-device',
1036         format_description => 'HOSTUSBDEVICE|spice',
1037         description => <<EODESCR,
1038 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1039
1040  'bus-port(.port)*' (decimal numbers) or
1041  'vendor_id:product_id' (hexadeciaml numbers) or
1042  'spice'
1043
1044 You can use the 'lsusb -t' command to list existing usb devices.
1045
1046 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1047 machines - use with special care.
1048
1049 The value 'spice' can be used to add a usb redirection devices for spice.
1050 EODESCR
1051     },
1052     usb3 => {
1053         optional => 1,
1054         type => 'boolean',
1055         description => "Specifies whether if given host option is a USB3 device or port.",
1056         default => 0,
1057     },
1058 };
1059
1060 my $usbdesc = {
1061     optional => 1,
1062     type => 'string', format => $usb_fmt,
1063     description => "Configure an USB device (n is 0 to 4).",
1064 };
1065 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1066
1067 my $serialdesc = {
1068         optional => 1,
1069         type => 'string',
1070         pattern => '(/dev/.+|socket)',
1071         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1072         verbose_description =>  <<EODESCR,
1073 Create a serial device inside the VM (n is 0 to 3), and pass through a
1074 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1075 host side (use 'qm terminal' to open a terminal connection).
1076
1077 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1078 use with special care.
1079
1080 CAUTION: Experimental! User reported problems with this option.
1081 EODESCR
1082 };
1083
1084 my $paralleldesc= {
1085         optional => 1,
1086         type => 'string',
1087         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1088         description =>  "Map host parallel devices (n is 0 to 2).",
1089         verbose_description =>  <<EODESCR,
1090 Map host parallel devices (n is 0 to 2).
1091
1092 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1093 machines - use with special care.
1094
1095 CAUTION: Experimental! User reported problems with this option.
1096 EODESCR
1097 };
1098
1099 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1100     $confdesc->{"parallel$i"} = $paralleldesc;
1101 }
1102
1103 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1104     $confdesc->{"serial$i"} = $serialdesc;
1105 }
1106
1107 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1108     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1109 }
1110
1111 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1112     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1113 }
1114
1115 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1116     $confdesc->{"usb$i"} = $usbdesc;
1117 }
1118
1119 my $boot_fmt = {
1120     legacy => {
1121         optional => 1,
1122         default_key => 1,
1123         type => 'string',
1124         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1125                      . " Deprecated, use 'order=' instead.",
1126         pattern => '[acdn]{1,4}',
1127         format_description => "[acdn]{1,4}",
1128
1129         # note: this is also the fallback if boot: is not given at all
1130         default => 'cdn',
1131     },
1132     order => {
1133         optional => 1,
1134         type => 'string',
1135         format => 'pve-qm-bootdev-list',
1136         format_description => "device[;device...]",
1137         description => <<EODESC,
1138 The guest will attempt to boot from devices in the order they appear here.
1139
1140 Disks, optical drives and passed-through storage USB devices will be directly
1141 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1142 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1143
1144 Note that only devices in this list will be marked as bootable and thus loaded
1145 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1146 (e.g. software-raid), you need to specify all of them here.
1147
1148 Overrides the deprecated 'legacy=[acdn]*' value when given.
1149 EODESC
1150     },
1151 };
1152 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1153
1154 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1155 sub verify_bootdev {
1156     my ($dev, $noerr) = @_;
1157
1158     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1159     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1160
1161     my $check = sub {
1162         my ($base) = @_;
1163         return 0 if $dev !~ m/^$base\d+$/;
1164         return 0 if !$confdesc->{$dev};
1165         return 1;
1166     };
1167
1168     return $dev if $check->("net");
1169     return $dev if $check->("usb");
1170     return $dev if $check->("hostpci");
1171
1172     return if $noerr;
1173     die "invalid boot device '$dev'\n";
1174 }
1175
1176 sub print_bootorder {
1177     my ($devs) = @_;
1178     return "" if !@$devs;
1179     my $data = { order => join(';', @$devs) };
1180     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1181 }
1182
1183 my $kvm_api_version = 0;
1184
1185 sub kvm_version {
1186     return $kvm_api_version if $kvm_api_version;
1187
1188     open my $fh, '<', '/dev/kvm' or return;
1189
1190     # 0xae00 => KVM_GET_API_VERSION
1191     $kvm_api_version = ioctl($fh, 0xae00, 0);
1192     close($fh);
1193
1194     return $kvm_api_version;
1195 }
1196
1197 my $kvm_user_version = {};
1198 my $kvm_mtime = {};
1199
1200 sub kvm_user_version {
1201     my ($binary) = @_;
1202
1203     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1204     my $st = stat($binary);
1205
1206     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1207     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1208         $cachedmtime == $st->mtime;
1209
1210     $kvm_user_version->{$binary} = 'unknown';
1211     $kvm_mtime->{$binary} = $st->mtime;
1212
1213     my $code = sub {
1214         my $line = shift;
1215         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1216             $kvm_user_version->{$binary} = $2;
1217         }
1218     };
1219
1220     eval { run_command([$binary, '--version'], outfunc => $code); };
1221     warn $@ if $@;
1222
1223     return $kvm_user_version->{$binary};
1224
1225 }
1226 my sub extract_version {
1227     my ($machine_type, $version) = @_;
1228     $version = kvm_user_version() if !defined($version);
1229     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1230 }
1231
1232 sub kernel_has_vhost_net {
1233     return -c '/dev/vhost-net';
1234 }
1235
1236 sub option_exists {
1237     my $key = shift;
1238     return defined($confdesc->{$key});
1239 }
1240
1241 my $cdrom_path;
1242 sub get_cdrom_path {
1243
1244     return  $cdrom_path if $cdrom_path;
1245
1246     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1247     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1248     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub scsi_inquiry {
1354     my($fh, $noerr) = @_;
1355
1356     my $SG_IO = 0x2285;
1357     my $SG_GET_VERSION_NUM = 0x2282;
1358
1359     my $versionbuf = "\x00" x 8;
1360     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1361     if (!$ret) {
1362         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1363         return;
1364     }
1365     my $version = unpack("I", $versionbuf);
1366     if ($version < 30000) {
1367         die "scsi generic interface too old\n"  if !$noerr;
1368         return;
1369     }
1370
1371     my $buf = "\x00" x 36;
1372     my $sensebuf = "\x00" x 8;
1373     my $cmd = pack("C x3 C x1", 0x12, 36);
1374
1375     # see /usr/include/scsi/sg.h
1376     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1377
1378     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1379                       length($sensebuf), 0, length($buf), $buf,
1380                       $cmd, $sensebuf, 6000);
1381
1382     $ret = ioctl($fh, $SG_IO, $packet);
1383     if (!$ret) {
1384         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1385         return;
1386     }
1387
1388     my @res = unpack($sg_io_hdr_t, $packet);
1389     if ($res[17] || $res[18]) {
1390         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1391         return;
1392     }
1393
1394     my $res = {};
1395     (my $byte0, my $byte1, $res->{vendor},
1396      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1397
1398     $res->{removable} = $byte1 & 128 ? 1 : 0;
1399     $res->{type} = $byte0 & 31;
1400
1401     return $res;
1402 }
1403
1404 sub path_is_scsi {
1405     my ($path) = @_;
1406
1407     my $fh = IO::File->new("+<$path") || return;
1408     my $res = scsi_inquiry($fh, 1);
1409     close($fh);
1410
1411     return $res;
1412 }
1413
1414 sub print_tabletdevice_full {
1415     my ($conf, $arch) = @_;
1416
1417     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1418
1419     # we use uhci for old VMs because tablet driver was buggy in older qemu
1420     my $usbbus;
1421     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1422         $usbbus = 'ehci';
1423     } else {
1424         $usbbus = 'uhci';
1425     }
1426
1427     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1428 }
1429
1430 sub print_keyboarddevice_full {
1431     my ($conf, $arch, $machine) = @_;
1432
1433     return if $arch ne 'aarch64';
1434
1435     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1436 }
1437
1438 my sub get_drive_id {
1439     my ($drive) = @_;
1440     return "$drive->{interface}$drive->{index}";
1441 }
1442
1443 sub print_drivedevice_full {
1444     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1445
1446     my $device = '';
1447     my $maxdev = 0;
1448
1449     my $drive_id = get_drive_id($drive);
1450     if ($drive->{interface} eq 'virtio') {
1451         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1452         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1453         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1454     } elsif ($drive->{interface} eq 'scsi') {
1455
1456         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1457         my $unit = $drive->{index} % $maxdev;
1458         my $devicetype = 'hd';
1459         my $path = '';
1460         if (drive_is_cdrom($drive)) {
1461             $devicetype = 'cd';
1462         } else {
1463             if ($drive->{file} =~ m|^/|) {
1464                 $path = $drive->{file};
1465                 if (my $info = path_is_scsi($path)) {
1466                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1467                         $devicetype = 'block';
1468                     } elsif ($info->{type} == 1) { # tape
1469                         $devicetype = 'generic';
1470                     }
1471                 }
1472             } else {
1473                  $path = PVE::Storage::path($storecfg, $drive->{file});
1474             }
1475
1476             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1477             my $version = extract_version($machine_type, kvm_user_version());
1478             if ($path =~ m/^iscsi\:\/\// &&
1479                !min_version($version, 4, 1)) {
1480                 $devicetype = 'generic';
1481             }
1482         }
1483
1484         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1485             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1486         } else {
1487             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1488                 .",lun=$drive->{index}";
1489         }
1490         $device .= ",drive=drive-$drive_id,id=$drive_id";
1491
1492         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1493             $device .= ",rotation_rate=1";
1494         }
1495         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1496
1497     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1498         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1499         my $controller = int($drive->{index} / $maxdev);
1500         my $unit = $drive->{index} % $maxdev;
1501         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1502
1503         $device = "ide-$devicetype";
1504         if ($drive->{interface} eq 'ide') {
1505             $device .= ",bus=ide.$controller,unit=$unit";
1506         } else {
1507             $device .= ",bus=ahci$controller.$unit";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($devicetype eq 'hd') {
1512             if (my $model = $drive->{model}) {
1513                 $model = URI::Escape::uri_unescape($model);
1514                 $device .= ",model=$model";
1515             }
1516             if ($drive->{ssd}) {
1517                 $device .= ",rotation_rate=1";
1518             }
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521     } elsif ($drive->{interface} eq 'usb') {
1522         die "implement me";
1523         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1524     } else {
1525         die "unsupported interface type";
1526     }
1527
1528     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1529
1530     if (my $serial = $drive->{serial}) {
1531         $serial = URI::Escape::uri_unescape($serial);
1532         $device .= ",serial=$serial";
1533     }
1534
1535
1536     return $device;
1537 }
1538
1539 sub get_initiator_name {
1540     my $initiator;
1541
1542     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1543     while (defined(my $line = <$fh>)) {
1544         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1545         $initiator = $1;
1546         last;
1547     }
1548     $fh->close();
1549
1550     return $initiator;
1551 }
1552
1553 sub print_drive_commandline_full {
1554     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1555
1556     my $path;
1557     my $volid = $drive->{file};
1558     my $format = $drive->{format};
1559     my $drive_id = get_drive_id($drive);
1560
1561     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1562     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1563
1564     if (drive_is_cdrom($drive)) {
1565         $path = get_iso_path($storecfg, $vmid, $volid);
1566         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1567     } else {
1568         if ($storeid) {
1569             $path = PVE::Storage::path($storecfg, $volid);
1570             $format //= qemu_img_format($scfg, $volname);
1571         } else {
1572             $path = $volid;
1573             $format //= "raw";
1574         }
1575    }
1576
1577    my $is_rbd = $path =~ m/^rbd:/;
1578
1579     my $opts = '';
1580     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1581     foreach my $o (@qemu_drive_options) {
1582         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1583     }
1584
1585     # snapshot only accepts on|off
1586     if (defined($drive->{snapshot})) {
1587         my $v = $drive->{snapshot} ? 'on' : 'off';
1588         $opts .= ",snapshot=$v";
1589     }
1590
1591     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1592         my ($dir, $qmpname) = @$type;
1593         if (my $v = $drive->{"mbps$dir"}) {
1594             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1595         }
1596         if (my $v = $drive->{"mbps${dir}_max"}) {
1597             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1598         }
1599         if (my $v = $drive->{"bps${dir}_max_length"}) {
1600             $opts .= ",throttling.bps$qmpname-max-length=$v";
1601         }
1602         if (my $v = $drive->{"iops${dir}"}) {
1603             $opts .= ",throttling.iops$qmpname=$v";
1604         }
1605         if (my $v = $drive->{"iops${dir}_max"}) {
1606             $opts .= ",throttling.iops$qmpname-max=$v";
1607         }
1608         if (my $v = $drive->{"iops${dir}_max_length"}) {
1609             $opts .= ",throttling.iops$qmpname-max-length=$v";
1610         }
1611     }
1612
1613     if ($pbs_name) {
1614         $format = "rbd" if $is_rbd;
1615         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1616             if !$format;
1617         $opts .= ",format=alloc-track,file.driver=$format";
1618     } elsif ($format) {
1619         $opts .= ",format=$format";
1620     }
1621
1622     my $cache_direct = 0;
1623
1624     if (my $cache = $drive->{cache}) {
1625         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1626     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1627         $opts .= ",cache=none";
1628         $cache_direct = 1;
1629     }
1630
1631     # io_uring with cache mode writeback or writethrough on krbd will hang...
1632     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1633
1634     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1635     # sometimes, just plain disable...
1636     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1637
1638     if (!$drive->{aio}) {
1639         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1640             # io_uring supports all cache modes
1641             $opts .= ",aio=io_uring";
1642         } else {
1643             # aio native works only with O_DIRECT
1644             if($cache_direct) {
1645                 $opts .= ",aio=native";
1646             } else {
1647                 $opts .= ",aio=threads";
1648             }
1649         }
1650     }
1651
1652     if (!drive_is_cdrom($drive)) {
1653         my $detectzeroes;
1654         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1655             $detectzeroes = 'off';
1656         } elsif ($drive->{discard}) {
1657             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1658         } else {
1659             # This used to be our default with discard not being specified:
1660             $detectzeroes = 'on';
1661         }
1662
1663         # note: 'detect-zeroes' works per blockdev and we want it to persist
1664         # after the alloc-track is removed, so put it on 'file' directly
1665         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1666         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1667     }
1668
1669     if ($pbs_name) {
1670         $opts .= ",backing=$pbs_name";
1671         $opts .= ",auto-remove=on";
1672     }
1673
1674     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1675     my $file_param = "file";
1676     if ($pbs_name) {
1677         # non-rbd drivers require the underlying file to be a seperate block
1678         # node, so add a second .file indirection
1679         $file_param .= ".file" if !$is_rbd;
1680         $file_param .= ".filename";
1681     }
1682     my $pathinfo = $path ? "$file_param=$path," : '';
1683
1684     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1685 }
1686
1687 sub print_pbs_blockdev {
1688     my ($pbs_conf, $pbs_name) = @_;
1689     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1690     $blockdev .= ",repository=$pbs_conf->{repository}";
1691     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1692     $blockdev .= ",archive=$pbs_conf->{archive}";
1693     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1694     return $blockdev;
1695 }
1696
1697 sub print_netdevice_full {
1698     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1699
1700     my $device = $net->{model};
1701     if ($net->{model} eq 'virtio') {
1702          $device = 'virtio-net-pci';
1703      };
1704
1705     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1706     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1707     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1708         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1709         # and out of each queue plus one config interrupt and control vector queue
1710         my $vectors = $net->{queues} * 2 + 2;
1711         $tmpstr .= ",vectors=$vectors,mq=on";
1712     }
1713     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1714
1715     if (my $mtu = $net->{mtu}) {
1716         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1717             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1718             if ($mtu == 1) {
1719                  $mtu = $bridge_mtu;
1720             } elsif ($mtu < 576) {
1721                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1722             } elsif ($mtu > $bridge_mtu) {
1723                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1724             }
1725             $tmpstr .= ",host_mtu=$mtu";
1726         } else {
1727             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1728         }
1729     }
1730
1731     if ($use_old_bios_files) {
1732         my $romfile;
1733         if ($device eq 'virtio-net-pci') {
1734             $romfile = 'pxe-virtio.rom';
1735         } elsif ($device eq 'e1000') {
1736             $romfile = 'pxe-e1000.rom';
1737         } elsif ($device eq 'e1000e') {
1738             $romfile = 'pxe-e1000e.rom';
1739         } elsif ($device eq 'ne2k') {
1740             $romfile = 'pxe-ne2k_pci.rom';
1741         } elsif ($device eq 'pcnet') {
1742             $romfile = 'pxe-pcnet.rom';
1743         } elsif ($device eq 'rtl8139') {
1744             $romfile = 'pxe-rtl8139.rom';
1745         }
1746         $tmpstr .= ",romfile=$romfile" if $romfile;
1747     }
1748
1749     return $tmpstr;
1750 }
1751
1752 sub print_netdev_full {
1753     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1754
1755     my $i = '';
1756     if ($netid =~ m/^net(\d+)$/) {
1757         $i = int($1);
1758     }
1759
1760     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1761
1762     my $ifname = "tap${vmid}i$i";
1763
1764     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1765     die "interface name '$ifname' is too long (max 15 character)\n"
1766         if length($ifname) >= 16;
1767
1768     my $vhostparam = '';
1769     if (is_native($arch)) {
1770         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1771     }
1772
1773     my $vmname = $conf->{name} || "vm$vmid";
1774
1775     my $netdev = "";
1776     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1777
1778     if ($net->{bridge}) {
1779         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1780             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1781     } else {
1782         $netdev = "type=user,id=$netid,hostname=$vmname";
1783     }
1784
1785     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1786
1787     return $netdev;
1788 }
1789
1790 my $vga_map = {
1791     'cirrus' => 'cirrus-vga',
1792     'std' => 'VGA',
1793     'vmware' => 'vmware-svga',
1794     'virtio' => 'virtio-vga',
1795 };
1796
1797 sub print_vga_device {
1798     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1799
1800     my $type = $vga_map->{$vga->{type}};
1801     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1802         $type = 'virtio-gpu';
1803     }
1804     my $vgamem_mb = $vga->{memory};
1805
1806     my $max_outputs = '';
1807     if ($qxlnum) {
1808         $type = $id ? 'qxl' : 'qxl-vga';
1809
1810         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1811             # set max outputs so linux can have up to 4 qxl displays with one device
1812             if (min_version($machine_version, 4, 1)) {
1813                 $max_outputs = ",max_outputs=4";
1814             }
1815         }
1816     }
1817
1818     die "no devicetype for $vga->{type}\n" if !$type;
1819
1820     my $memory = "";
1821     if ($vgamem_mb) {
1822         if ($vga->{type} eq 'virtio') {
1823             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1824             $memory = ",max_hostmem=$bytes";
1825         } elsif ($qxlnum) {
1826             # from https://www.spice-space.org/multiple-monitors.html
1827             $memory = ",vgamem_mb=$vga->{memory}";
1828             my $ram = $vgamem_mb * 4;
1829             my $vram = $vgamem_mb * 2;
1830             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1831         } else {
1832             $memory = ",vgamem_mb=$vga->{memory}";
1833         }
1834     } elsif ($qxlnum && $id) {
1835         $memory = ",ram_size=67108864,vram_size=33554432";
1836     }
1837
1838     my $edidoff = "";
1839     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1840         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1841     }
1842
1843     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1844     my $vgaid = "vga" . ($id // '');
1845     my $pciaddr;
1846
1847     if ($q35 && $vgaid eq 'vga') {
1848         # the first display uses pcie.0 bus on q35 machines
1849         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1850     } else {
1851         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1852     }
1853
1854     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1855 }
1856
1857 sub parse_number_sets {
1858     my ($set) = @_;
1859     my $res = [];
1860     foreach my $part (split(/;/, $set)) {
1861         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1862             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1863             push @$res, [ $1, $2 ];
1864         } else {
1865             die "invalid range: $part\n";
1866         }
1867     }
1868     return $res;
1869 }
1870
1871 sub parse_numa {
1872     my ($data) = @_;
1873
1874     my $res = parse_property_string($numa_fmt, $data);
1875     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1876     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1877     return $res;
1878 }
1879
1880 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1881 sub parse_net {
1882     my ($data) = @_;
1883
1884     my $res = eval { parse_property_string($net_fmt, $data) };
1885     if ($@) {
1886         warn $@;
1887         return;
1888     }
1889     if (!defined($res->{macaddr})) {
1890         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1891         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1892     }
1893     return $res;
1894 }
1895
1896 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1897 sub parse_ipconfig {
1898     my ($data) = @_;
1899
1900     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1901     if ($@) {
1902         warn $@;
1903         return;
1904     }
1905
1906     if ($res->{gw} && !$res->{ip}) {
1907         warn 'gateway specified without specifying an IP address';
1908         return;
1909     }
1910     if ($res->{gw6} && !$res->{ip6}) {
1911         warn 'IPv6 gateway specified without specifying an IPv6 address';
1912         return;
1913     }
1914     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1915         warn 'gateway specified together with DHCP';
1916         return;
1917     }
1918     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1919         # gw6 + auto/dhcp
1920         warn "IPv6 gateway specified together with $res->{ip6} address";
1921         return;
1922     }
1923
1924     if (!$res->{ip} && !$res->{ip6}) {
1925         return { ip => 'dhcp', ip6 => 'dhcp' };
1926     }
1927
1928     return $res;
1929 }
1930
1931 sub print_net {
1932     my $net = shift;
1933
1934     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1935 }
1936
1937 sub add_random_macs {
1938     my ($settings) = @_;
1939
1940     foreach my $opt (keys %$settings) {
1941         next if $opt !~ m/^net(\d+)$/;
1942         my $net = parse_net($settings->{$opt});
1943         next if !$net;
1944         $settings->{$opt} = print_net($net);
1945     }
1946 }
1947
1948 sub vm_is_volid_owner {
1949     my ($storecfg, $vmid, $volid) = @_;
1950
1951     if ($volid !~  m|^/|) {
1952         my ($path, $owner);
1953         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1954         if ($owner && ($owner == $vmid)) {
1955             return 1;
1956         }
1957     }
1958
1959     return;
1960 }
1961
1962 sub vmconfig_register_unused_drive {
1963     my ($storecfg, $vmid, $conf, $drive) = @_;
1964
1965     if (drive_is_cloudinit($drive)) {
1966         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1967         warn $@ if $@;
1968     } elsif (!drive_is_cdrom($drive)) {
1969         my $volid = $drive->{file};
1970         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1971             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1972         }
1973     }
1974 }
1975
1976 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1977 my $smbios1_fmt = {
1978     uuid => {
1979         type => 'string',
1980         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1981         format_description => 'UUID',
1982         description => "Set SMBIOS1 UUID.",
1983         optional => 1,
1984     },
1985     version => {
1986         type => 'string',
1987         pattern => '[A-Za-z0-9+\/]+={0,2}',
1988         format_description => 'Base64 encoded string',
1989         description => "Set SMBIOS1 version.",
1990         optional => 1,
1991     },
1992     serial => {
1993         type => 'string',
1994         pattern => '[A-Za-z0-9+\/]+={0,2}',
1995         format_description => 'Base64 encoded string',
1996         description => "Set SMBIOS1 serial number.",
1997         optional => 1,
1998     },
1999     manufacturer => {
2000         type => 'string',
2001         pattern => '[A-Za-z0-9+\/]+={0,2}',
2002         format_description => 'Base64 encoded string',
2003         description => "Set SMBIOS1 manufacturer.",
2004         optional => 1,
2005     },
2006     product => {
2007         type => 'string',
2008         pattern => '[A-Za-z0-9+\/]+={0,2}',
2009         format_description => 'Base64 encoded string',
2010         description => "Set SMBIOS1 product ID.",
2011         optional => 1,
2012     },
2013     sku => {
2014         type => 'string',
2015         pattern => '[A-Za-z0-9+\/]+={0,2}',
2016         format_description => 'Base64 encoded string',
2017         description => "Set SMBIOS1 SKU string.",
2018         optional => 1,
2019     },
2020     family => {
2021         type => 'string',
2022         pattern => '[A-Za-z0-9+\/]+={0,2}',
2023         format_description => 'Base64 encoded string',
2024         description => "Set SMBIOS1 family string.",
2025         optional => 1,
2026     },
2027     base64 => {
2028         type => 'boolean',
2029         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2030         optional => 1,
2031     },
2032 };
2033
2034 sub parse_smbios1 {
2035     my ($data) = @_;
2036
2037     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2038     warn $@ if $@;
2039     return $res;
2040 }
2041
2042 sub print_smbios1 {
2043     my ($smbios1) = @_;
2044     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2045 }
2046
2047 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2048
2049 sub parse_watchdog {
2050     my ($value) = @_;
2051
2052     return if !$value;
2053
2054     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2055     warn $@ if $@;
2056     return $res;
2057 }
2058
2059 sub parse_guest_agent {
2060     my ($conf) = @_;
2061
2062     return {} if !defined($conf->{agent});
2063
2064     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2065     warn $@ if $@;
2066
2067     # if the agent is disabled ignore the other potentially set properties
2068     return {} if !$res->{enabled};
2069     return $res;
2070 }
2071
2072 sub get_qga_key {
2073     my ($conf, $key) = @_;
2074     return undef if !defined($conf->{agent});
2075
2076     my $agent = parse_guest_agent($conf);
2077     return $agent->{$key};
2078 }
2079
2080 sub parse_vga {
2081     my ($value) = @_;
2082
2083     return {} if !$value;
2084     my $res = eval { parse_property_string($vga_fmt, $value) };
2085     warn $@ if $@;
2086     return $res;
2087 }
2088
2089 sub parse_rng {
2090     my ($value) = @_;
2091
2092     return if !$value;
2093
2094     my $res = eval { parse_property_string($rng_fmt, $value) };
2095     warn $@ if $@;
2096     return $res;
2097 }
2098
2099 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2100 sub verify_usb_device {
2101     my ($value, $noerr) = @_;
2102
2103     return $value if parse_usb_device($value);
2104
2105     return if $noerr;
2106
2107     die "unable to parse usb device\n";
2108 }
2109
2110 # add JSON properties for create and set function
2111 sub json_config_properties {
2112     my $prop = shift;
2113
2114     foreach my $opt (keys %$confdesc) {
2115         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2116             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2117         $prop->{$opt} = $confdesc->{$opt};
2118     }
2119
2120     return $prop;
2121 }
2122
2123 # return copy of $confdesc_cloudinit to generate documentation
2124 sub cloudinit_config_properties {
2125
2126     return dclone($confdesc_cloudinit);
2127 }
2128
2129 sub check_type {
2130     my ($key, $value) = @_;
2131
2132     die "unknown setting '$key'\n" if !$confdesc->{$key};
2133
2134     my $type = $confdesc->{$key}->{type};
2135
2136     if (!defined($value)) {
2137         die "got undefined value\n";
2138     }
2139
2140     if ($value =~ m/[\n\r]/) {
2141         die "property contains a line feed\n";
2142     }
2143
2144     if ($type eq 'boolean') {
2145         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2146         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2147         die "type check ('boolean') failed - got '$value'\n";
2148     } elsif ($type eq 'integer') {
2149         return int($1) if $value =~ m/^(\d+)$/;
2150         die "type check ('integer') failed - got '$value'\n";
2151     } elsif ($type eq 'number') {
2152         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2153         die "type check ('number') failed - got '$value'\n";
2154     } elsif ($type eq 'string') {
2155         if (my $fmt = $confdesc->{$key}->{format}) {
2156             PVE::JSONSchema::check_format($fmt, $value);
2157             return $value;
2158         }
2159         $value =~ s/^\"(.*)\"$/$1/;
2160         return $value;
2161     } else {
2162         die "internal error"
2163     }
2164 }
2165
2166 sub destroy_vm {
2167     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2168
2169     my $conf = PVE::QemuConfig->load_config($vmid);
2170
2171     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2172
2173     if ($conf->{template}) {
2174         # check if any base image is still used by a linked clone
2175         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2176                 my ($ds, $drive) = @_;
2177                 return if drive_is_cdrom($drive);
2178
2179                 my $volid = $drive->{file};
2180                 return if !$volid || $volid =~ m|^/|;
2181
2182                 die "base volume '$volid' is still in use by linked cloned\n"
2183                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2184
2185         });
2186     }
2187
2188     my $volids = {};
2189     my $remove_owned_drive = sub {
2190         my ($ds, $drive) = @_;
2191         return if drive_is_cdrom($drive, 1);
2192
2193         my $volid = $drive->{file};
2194         return if !$volid || $volid =~ m|^/|;
2195         return if $volids->{$volid};
2196
2197         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2198         return if !$path || !$owner || ($owner != $vmid);
2199
2200         $volids->{$volid} = 1;
2201         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2202         warn "Could not remove disk '$volid', check manually: $@" if $@;
2203     };
2204
2205     # only remove disks owned by this VM (referenced in the config)
2206     my $include_opts = {
2207         include_unused => 1,
2208         extra_keys => ['vmstate'],
2209     };
2210     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2211
2212     for my $snap (values %{$conf->{snapshots}}) {
2213         next if !defined($snap->{vmstate});
2214         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2215         next if !defined($drive);
2216         $remove_owned_drive->('vmstate', $drive);
2217     }
2218
2219     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2220
2221     if ($purge_unreferenced) { # also remove unreferenced disk
2222         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2223         PVE::Storage::foreach_volid($vmdisks, sub {
2224             my ($volid, $sid, $volname, $d) = @_;
2225             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2226             warn $@ if $@;
2227         });
2228     }
2229
2230     if (defined $replacement_conf) {
2231         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2232     } else {
2233         PVE::QemuConfig->destroy_config($vmid);
2234     }
2235 }
2236
2237 sub parse_vm_config {
2238     my ($filename, $raw) = @_;
2239
2240     return if !defined($raw);
2241
2242     my $res = {
2243         digest => Digest::SHA::sha1_hex($raw),
2244         snapshots => {},
2245         pending => {},
2246     };
2247
2248     $filename =~ m|/qemu-server/(\d+)\.conf$|
2249         || die "got strange filename '$filename'";
2250
2251     my $vmid = $1;
2252
2253     my $conf = $res;
2254     my $descr;
2255     my $section = '';
2256
2257     my @lines = split(/\n/, $raw);
2258     foreach my $line (@lines) {
2259         next if $line =~ m/^\s*$/;
2260
2261         if ($line =~ m/^\[PENDING\]\s*$/i) {
2262             $section = 'pending';
2263             if (defined($descr)) {
2264                 $descr =~ s/\s+$//;
2265                 $conf->{description} = $descr;
2266             }
2267             $descr = undef;
2268             $conf = $res->{$section} = {};
2269             next;
2270
2271         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2272             $section = $1;
2273             if (defined($descr)) {
2274                 $descr =~ s/\s+$//;
2275                 $conf->{description} = $descr;
2276             }
2277             $descr = undef;
2278             $conf = $res->{snapshots}->{$section} = {};
2279             next;
2280         }
2281
2282         if ($line =~ m/^\#(.*)\s*$/) {
2283             $descr = '' if !defined($descr);
2284             $descr .= PVE::Tools::decode_text($1) . "\n";
2285             next;
2286         }
2287
2288         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2289             $descr = '' if !defined($descr);
2290             $descr .= PVE::Tools::decode_text($2);
2291         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2292             $conf->{snapstate} = $1;
2293         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2294             my $key = $1;
2295             my $value = $2;
2296             $conf->{$key} = $value;
2297         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2298             my $value = $1;
2299             if ($section eq 'pending') {
2300                 $conf->{delete} = $value; # we parse this later
2301             } else {
2302                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2303             }
2304         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2305             my $key = $1;
2306             my $value = $2;
2307             eval { $value = check_type($key, $value); };
2308             if ($@) {
2309                 warn "vm $vmid - unable to parse value of '$key' - $@";
2310             } else {
2311                 $key = 'ide2' if $key eq 'cdrom';
2312                 my $fmt = $confdesc->{$key}->{format};
2313                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2314                     my $v = parse_drive($key, $value);
2315                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2316                         $v->{file} = $volid;
2317                         $value = print_drive($v);
2318                     } else {
2319                         warn "vm $vmid - unable to parse value of '$key'\n";
2320                         next;
2321                     }
2322                 }
2323
2324                 $conf->{$key} = $value;
2325             }
2326         } else {
2327             warn "vm $vmid - unable to parse config: $line\n";
2328         }
2329     }
2330
2331     if (defined($descr)) {
2332         $descr =~ s/\s+$//;
2333         $conf->{description} = $descr;
2334     }
2335     delete $res->{snapstate}; # just to be sure
2336
2337     return $res;
2338 }
2339
2340 sub write_vm_config {
2341     my ($filename, $conf) = @_;
2342
2343     delete $conf->{snapstate}; # just to be sure
2344
2345     if ($conf->{cdrom}) {
2346         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2347         $conf->{ide2} = $conf->{cdrom};
2348         delete $conf->{cdrom};
2349     }
2350
2351     # we do not use 'smp' any longer
2352     if ($conf->{sockets}) {
2353         delete $conf->{smp};
2354     } elsif ($conf->{smp}) {
2355         $conf->{sockets} = $conf->{smp};
2356         delete $conf->{cores};
2357         delete $conf->{smp};
2358     }
2359
2360     my $used_volids = {};
2361
2362     my $cleanup_config = sub {
2363         my ($cref, $pending, $snapname) = @_;
2364
2365         foreach my $key (keys %$cref) {
2366             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2367                 $key eq 'snapstate' || $key eq 'pending';
2368             my $value = $cref->{$key};
2369             if ($key eq 'delete') {
2370                 die "propertry 'delete' is only allowed in [PENDING]\n"
2371                     if !$pending;
2372                 # fixme: check syntax?
2373                 next;
2374             }
2375             eval { $value = check_type($key, $value); };
2376             die "unable to parse value of '$key' - $@" if $@;
2377
2378             $cref->{$key} = $value;
2379
2380             if (!$snapname && is_valid_drivename($key)) {
2381                 my $drive = parse_drive($key, $value);
2382                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2383             }
2384         }
2385     };
2386
2387     &$cleanup_config($conf);
2388
2389     &$cleanup_config($conf->{pending}, 1);
2390
2391     foreach my $snapname (keys %{$conf->{snapshots}}) {
2392         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2393         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2394     }
2395
2396     # remove 'unusedX' settings if we re-add a volume
2397     foreach my $key (keys %$conf) {
2398         my $value = $conf->{$key};
2399         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2400             delete $conf->{$key};
2401         }
2402     }
2403
2404     my $generate_raw_config = sub {
2405         my ($conf, $pending) = @_;
2406
2407         my $raw = '';
2408
2409         # add description as comment to top of file
2410         if (defined(my $descr = $conf->{description})) {
2411             if ($descr) {
2412                 foreach my $cl (split(/\n/, $descr)) {
2413                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2414                 }
2415             } else {
2416                 $raw .= "#\n" if $pending;
2417             }
2418         }
2419
2420         foreach my $key (sort keys %$conf) {
2421             next if $key =~ /^(digest|description|pending|snapshots)$/;
2422             $raw .= "$key: $conf->{$key}\n";
2423         }
2424         return $raw;
2425     };
2426
2427     my $raw = &$generate_raw_config($conf);
2428
2429     if (scalar(keys %{$conf->{pending}})){
2430         $raw .= "\n[PENDING]\n";
2431         $raw .= &$generate_raw_config($conf->{pending}, 1);
2432     }
2433
2434     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2435         $raw .= "\n[$snapname]\n";
2436         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2437     }
2438
2439     return $raw;
2440 }
2441
2442 sub load_defaults {
2443
2444     my $res = {};
2445
2446     # we use static defaults from our JSON schema configuration
2447     foreach my $key (keys %$confdesc) {
2448         if (defined(my $default = $confdesc->{$key}->{default})) {
2449             $res->{$key} = $default;
2450         }
2451     }
2452
2453     return $res;
2454 }
2455
2456 sub config_list {
2457     my $vmlist = PVE::Cluster::get_vmlist();
2458     my $res = {};
2459     return $res if !$vmlist || !$vmlist->{ids};
2460     my $ids = $vmlist->{ids};
2461     my $nodename = nodename();
2462
2463     foreach my $vmid (keys %$ids) {
2464         my $d = $ids->{$vmid};
2465         next if !$d->{node} || $d->{node} ne $nodename;
2466         next if !$d->{type} || $d->{type} ne 'qemu';
2467         $res->{$vmid}->{exists} = 1;
2468     }
2469     return $res;
2470 }
2471
2472 # test if VM uses local resources (to prevent migration)
2473 sub check_local_resources {
2474     my ($conf, $noerr) = @_;
2475
2476     my @loc_res = ();
2477
2478     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2479     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2480
2481     push @loc_res, "ivshmem" if $conf->{ivshmem};
2482
2483     foreach my $k (keys %$conf) {
2484         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2485         # sockets are safe: they will recreated be on the target side post-migrate
2486         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2487         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2488     }
2489
2490     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2491
2492     return \@loc_res;
2493 }
2494
2495 # check if used storages are available on all nodes (use by migrate)
2496 sub check_storage_availability {
2497     my ($storecfg, $conf, $node) = @_;
2498
2499     PVE::QemuConfig->foreach_volume($conf, sub {
2500         my ($ds, $drive) = @_;
2501
2502         my $volid = $drive->{file};
2503         return if !$volid;
2504
2505         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2506         return if !$sid;
2507
2508         # check if storage is available on both nodes
2509         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2510         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2511
2512         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2513
2514         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2515             if !$scfg->{content}->{$vtype};
2516    });
2517 }
2518
2519 # list nodes where all VM images are available (used by has_feature API)
2520 sub shared_nodes {
2521     my ($conf, $storecfg) = @_;
2522
2523     my $nodelist = PVE::Cluster::get_nodelist();
2524     my $nodehash = { map { $_ => 1 } @$nodelist };
2525     my $nodename = nodename();
2526
2527     PVE::QemuConfig->foreach_volume($conf, sub {
2528         my ($ds, $drive) = @_;
2529
2530         my $volid = $drive->{file};
2531         return if !$volid;
2532
2533         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2534         if ($storeid) {
2535             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2536             if ($scfg->{disable}) {
2537                 $nodehash = {};
2538             } elsif (my $avail = $scfg->{nodes}) {
2539                 foreach my $node (keys %$nodehash) {
2540                     delete $nodehash->{$node} if !$avail->{$node};
2541                 }
2542             } elsif (!$scfg->{shared}) {
2543                 foreach my $node (keys %$nodehash) {
2544                     delete $nodehash->{$node} if $node ne $nodename
2545                 }
2546             }
2547         }
2548     });
2549
2550     return $nodehash
2551 }
2552
2553 sub check_local_storage_availability {
2554     my ($conf, $storecfg) = @_;
2555
2556     my $nodelist = PVE::Cluster::get_nodelist();
2557     my $nodehash = { map { $_ => {} } @$nodelist };
2558
2559     PVE::QemuConfig->foreach_volume($conf, sub {
2560         my ($ds, $drive) = @_;
2561
2562         my $volid = $drive->{file};
2563         return if !$volid;
2564
2565         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2566         if ($storeid) {
2567             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2568
2569             if ($scfg->{disable}) {
2570                 foreach my $node (keys %$nodehash) {
2571                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2572                 }
2573             } elsif (my $avail = $scfg->{nodes}) {
2574                 foreach my $node (keys %$nodehash) {
2575                     if (!$avail->{$node}) {
2576                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2577                     }
2578                 }
2579             }
2580         }
2581     });
2582
2583     foreach my $node (values %$nodehash) {
2584         if (my $unavail = $node->{unavailable_storages}) {
2585             $node->{unavailable_storages} = [ sort keys %$unavail ];
2586         }
2587     }
2588
2589     return $nodehash
2590 }
2591
2592 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2593 sub check_running {
2594     my ($vmid, $nocheck, $node) = @_;
2595
2596     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2597     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2598 }
2599
2600 sub vzlist {
2601
2602     my $vzlist = config_list();
2603
2604     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2605
2606     while (defined(my $de = $fd->read)) {
2607         next if $de !~ m/^(\d+)\.pid$/;
2608         my $vmid = $1;
2609         next if !defined($vzlist->{$vmid});
2610         if (my $pid = check_running($vmid)) {
2611             $vzlist->{$vmid}->{pid} = $pid;
2612         }
2613     }
2614
2615     return $vzlist;
2616 }
2617
2618 our $vmstatus_return_properties = {
2619     vmid => get_standard_option('pve-vmid'),
2620     status => {
2621         description => "Qemu process status.",
2622         type => 'string',
2623         enum => ['stopped', 'running'],
2624     },
2625     maxmem => {
2626         description => "Maximum memory in bytes.",
2627         type => 'integer',
2628         optional => 1,
2629         renderer => 'bytes',
2630     },
2631     maxdisk => {
2632         description => "Root disk size in bytes.",
2633         type => 'integer',
2634         optional => 1,
2635         renderer => 'bytes',
2636     },
2637     name => {
2638         description => "VM name.",
2639         type => 'string',
2640         optional => 1,
2641     },
2642     qmpstatus => {
2643         description => "Qemu QMP agent status.",
2644         type => 'string',
2645         optional => 1,
2646     },
2647     pid => {
2648         description => "PID of running qemu process.",
2649         type => 'integer',
2650         optional => 1,
2651     },
2652     uptime => {
2653         description => "Uptime.",
2654         type => 'integer',
2655         optional => 1,
2656         renderer => 'duration',
2657     },
2658     cpus => {
2659         description => "Maximum usable CPUs.",
2660         type => 'number',
2661         optional => 1,
2662     },
2663     lock => {
2664         description => "The current config lock, if any.",
2665         type => 'string',
2666         optional => 1,
2667     },
2668     tags => {
2669         description  => "The current configured tags, if any",
2670         type => 'string',
2671         optional => 1,
2672     },
2673     'running-machine' => {
2674         description => "The currently running machine type (if running).",
2675         type => 'string',
2676         optional => 1,
2677     },
2678     'running-qemu' => {
2679         description => "The currently running QEMU version (if running).",
2680         type => 'string',
2681         optional => 1,
2682     },
2683 };
2684
2685 my $last_proc_pid_stat;
2686
2687 # get VM status information
2688 # This must be fast and should not block ($full == false)
2689 # We only query KVM using QMP if $full == true (this can be slow)
2690 sub vmstatus {
2691     my ($opt_vmid, $full) = @_;
2692
2693     my $res = {};
2694
2695     my $storecfg = PVE::Storage::config();
2696
2697     my $list = vzlist();
2698     my $defaults = load_defaults();
2699
2700     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2701
2702     my $cpucount = $cpuinfo->{cpus} || 1;
2703
2704     foreach my $vmid (keys %$list) {
2705         next if $opt_vmid && ($vmid ne $opt_vmid);
2706
2707         my $conf = PVE::QemuConfig->load_config($vmid);
2708
2709         my $d = { vmid => int($vmid) };
2710         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2711
2712         # fixme: better status?
2713         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2714
2715         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2716         if (defined($size)) {
2717             $d->{disk} = 0; # no info available
2718             $d->{maxdisk} = $size;
2719         } else {
2720             $d->{disk} = 0;
2721             $d->{maxdisk} = 0;
2722         }
2723
2724         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2725             * ($conf->{cores} || $defaults->{cores});
2726         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2727         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2728
2729         $d->{name} = $conf->{name} || "VM $vmid";
2730         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2731             : $defaults->{memory}*(1024*1024);
2732
2733         if ($conf->{balloon}) {
2734             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2735             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2736                 : $defaults->{shares};
2737         }
2738
2739         $d->{uptime} = 0;
2740         $d->{cpu} = 0;
2741         $d->{mem} = 0;
2742
2743         $d->{netout} = 0;
2744         $d->{netin} = 0;
2745
2746         $d->{diskread} = 0;
2747         $d->{diskwrite} = 0;
2748
2749         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2750
2751         $d->{serial} = 1 if conf_has_serial($conf);
2752         $d->{lock} = $conf->{lock} if $conf->{lock};
2753         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2754
2755         $res->{$vmid} = $d;
2756     }
2757
2758     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2759     foreach my $dev (keys %$netdev) {
2760         next if $dev !~ m/^tap([1-9]\d*)i/;
2761         my $vmid = $1;
2762         my $d = $res->{$vmid};
2763         next if !$d;
2764
2765         $d->{netout} += $netdev->{$dev}->{receive};
2766         $d->{netin} += $netdev->{$dev}->{transmit};
2767
2768         if ($full) {
2769             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2770             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2771         }
2772
2773     }
2774
2775     my $ctime = gettimeofday;
2776
2777     foreach my $vmid (keys %$list) {
2778
2779         my $d = $res->{$vmid};
2780         my $pid = $d->{pid};
2781         next if !$pid;
2782
2783         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2784         next if !$pstat; # not running
2785
2786         my $used = $pstat->{utime} + $pstat->{stime};
2787
2788         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2789
2790         if ($pstat->{vsize}) {
2791             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2792         }
2793
2794         my $old = $last_proc_pid_stat->{$pid};
2795         if (!$old) {
2796             $last_proc_pid_stat->{$pid} = {
2797                 time => $ctime,
2798                 used => $used,
2799                 cpu => 0,
2800             };
2801             next;
2802         }
2803
2804         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2805
2806         if ($dtime > 1000) {
2807             my $dutime = $used -  $old->{used};
2808
2809             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2810             $last_proc_pid_stat->{$pid} = {
2811                 time => $ctime,
2812                 used => $used,
2813                 cpu => $d->{cpu},
2814             };
2815         } else {
2816             $d->{cpu} = $old->{cpu};
2817         }
2818     }
2819
2820     return $res if !$full;
2821
2822     my $qmpclient = PVE::QMPClient->new();
2823
2824     my $ballooncb = sub {
2825         my ($vmid, $resp) = @_;
2826
2827         my $info = $resp->{'return'};
2828         return if !$info->{max_mem};
2829
2830         my $d = $res->{$vmid};
2831
2832         # use memory assigned to VM
2833         $d->{maxmem} = $info->{max_mem};
2834         $d->{balloon} = $info->{actual};
2835
2836         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2837             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2838             $d->{freemem} = $info->{free_mem};
2839         }
2840
2841         $d->{ballooninfo} = $info;
2842     };
2843
2844     my $blockstatscb = sub {
2845         my ($vmid, $resp) = @_;
2846         my $data = $resp->{'return'} || [];
2847         my $totalrdbytes = 0;
2848         my $totalwrbytes = 0;
2849
2850         for my $blockstat (@$data) {
2851             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2852             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2853
2854             $blockstat->{device} =~ s/drive-//;
2855             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2856         }
2857         $res->{$vmid}->{diskread} = $totalrdbytes;
2858         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2859     };
2860
2861     my $machinecb = sub {
2862         my ($vmid, $resp) = @_;
2863         my $data = $resp->{'return'} || [];
2864
2865         $res->{$vmid}->{'running-machine'} =
2866             PVE::QemuServer::Machine::current_from_query_machines($data);
2867     };
2868
2869     my $versioncb = sub {
2870         my ($vmid, $resp) = @_;
2871         my $data = $resp->{'return'} // {};
2872         my $version = 'unknown';
2873
2874         if (my $v = $data->{qemu}) {
2875             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2876         }
2877
2878         $res->{$vmid}->{'running-qemu'} = $version;
2879     };
2880
2881     my $statuscb = sub {
2882         my ($vmid, $resp) = @_;
2883
2884         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2885         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2886         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2887         # this fails if ballon driver is not loaded, so this must be
2888         # the last commnand (following command are aborted if this fails).
2889         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2890
2891         my $status = 'unknown';
2892         if (!defined($status = $resp->{'return'}->{status})) {
2893             warn "unable to get VM status\n";
2894             return;
2895         }
2896
2897         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2898     };
2899
2900     foreach my $vmid (keys %$list) {
2901         next if $opt_vmid && ($vmid ne $opt_vmid);
2902         next if !$res->{$vmid}->{pid}; # not running
2903         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2904     }
2905
2906     $qmpclient->queue_execute(undef, 2);
2907
2908     foreach my $vmid (keys %$list) {
2909         next if $opt_vmid && ($vmid ne $opt_vmid);
2910         next if !$res->{$vmid}->{pid}; #not running
2911
2912         # we can't use the $qmpclient since it might have already aborted on
2913         # 'query-balloon', but this might also fail for older versions...
2914         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2915         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2916     }
2917
2918     foreach my $vmid (keys %$list) {
2919         next if $opt_vmid && ($vmid ne $opt_vmid);
2920         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2921     }
2922
2923     return $res;
2924 }
2925
2926 sub conf_has_serial {
2927     my ($conf) = @_;
2928
2929     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2930         if ($conf->{"serial$i"}) {
2931             return 1;
2932         }
2933     }
2934
2935     return 0;
2936 }
2937
2938 sub conf_has_audio {
2939     my ($conf, $id) = @_;
2940
2941     $id //= 0;
2942     my $audio = $conf->{"audio$id"};
2943     return if !defined($audio);
2944
2945     my $audioproperties = parse_property_string($audio_fmt, $audio);
2946     my $audiodriver = $audioproperties->{driver} // 'spice';
2947
2948     return {
2949         dev => $audioproperties->{device},
2950         dev_id => "audiodev$id",
2951         backend => $audiodriver,
2952         backend_id => "$audiodriver-backend${id}",
2953     };
2954 }
2955
2956 sub audio_devs {
2957     my ($audio, $audiopciaddr, $machine_version) = @_;
2958
2959     my $devs = [];
2960
2961     my $id = $audio->{dev_id};
2962     my $audiodev = "";
2963     if (min_version($machine_version, 4, 2)) {
2964         $audiodev = ",audiodev=$audio->{backend_id}";
2965     }
2966
2967     if ($audio->{dev} eq 'AC97') {
2968         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2969     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2970         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2971         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2972         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2973     } else {
2974         die "unkown audio device '$audio->{dev}', implement me!";
2975     }
2976
2977     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2978
2979     return $devs;
2980 }
2981
2982 sub get_tpm_paths {
2983     my ($vmid) = @_;
2984     return {
2985         socket => "/var/run/qemu-server/$vmid.swtpm",
2986         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
2987     };
2988 }
2989
2990 sub add_tpm_device {
2991     my ($vmid, $devices, $conf) = @_;
2992
2993     return if !$conf->{tpmstate0};
2994
2995     my $paths = get_tpm_paths($vmid);
2996
2997     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
2998     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
2999     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3000 }
3001
3002 sub start_swtpm {
3003     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3004
3005     return if !$tpmdrive;
3006
3007     my $state;
3008     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3009     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3010     if ($storeid) {
3011         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3012     } else {
3013         $state = $tpm->{file};
3014     }
3015
3016     my $paths = get_tpm_paths($vmid);
3017
3018     # during migration, we will get state from remote
3019     #
3020     if (!$migration) {
3021         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3022         my $setup_cmd = [
3023             "swtpm_setup",
3024             "--tpmstate",
3025             "file://$state",
3026             "--createek",
3027             "--create-ek-cert",
3028             "--create-platform-cert",
3029             "--lock-nvram",
3030             "--config",
3031             "/etc/swtpm_setup.conf", # do not use XDG configs
3032             "--runas",
3033             "0", # force creation as root, error if not possible
3034             "--not-overwrite", # ignore existing state, do not modify
3035         ];
3036
3037         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3038         # TPM 2.0 supports ECC crypto, use if possible
3039         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3040
3041         run_command($setup_cmd, outfunc => sub {
3042             print "swtpm_setup: $1\n";
3043         });
3044     }
3045
3046     my $emulator_cmd = [
3047         "swtpm",
3048         "socket",
3049         "--tpmstate",
3050         "backend-uri=file://$state,mode=0600",
3051         "--ctrl",
3052         "type=unixio,path=$paths->{socket},mode=0600",
3053         "--pid",
3054         "file=$paths->{pid}",
3055         "--terminate", # terminate on QEMU disconnect
3056         "--daemon",
3057     ];
3058     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3059     run_command($emulator_cmd, outfunc => sub { print $1; });
3060
3061     my $tries = 100; # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3062     while (! -e $paths->{pid}) {
3063         usleep(50_000);
3064         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n"
3065             if --$tries == 0;
3066     }
3067
3068     # return untainted PID of swtpm daemon so it can be killed on error
3069     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3070     return $1;
3071 }
3072
3073 sub vga_conf_has_spice {
3074     my ($vga) = @_;
3075
3076     my $vgaconf = parse_vga($vga);
3077     my $vgatype = $vgaconf->{type};
3078     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3079
3080     return $1 || 1;
3081 }
3082
3083 sub is_native($) {
3084     my ($arch) = @_;
3085     return get_host_arch() eq $arch;
3086 }
3087
3088 sub get_vm_arch {
3089     my ($conf) = @_;
3090     return $conf->{arch} // get_host_arch();
3091 }
3092
3093 my $default_machines = {
3094     x86_64 => 'pc',
3095     aarch64 => 'virt',
3096 };
3097
3098 sub get_installed_machine_version {
3099     my ($kvmversion) = @_;
3100     $kvmversion = kvm_user_version() if !defined($kvmversion);
3101     $kvmversion =~ m/^(\d+\.\d+)/;
3102     return $1;
3103 }
3104
3105 sub windows_get_pinned_machine_version {
3106     my ($machine, $base_version, $kvmversion) = @_;
3107
3108     my $pin_version = $base_version;
3109     if (!defined($base_version) ||
3110         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3111     ) {
3112         $pin_version = get_installed_machine_version($kvmversion);
3113     }
3114     if (!$machine || $machine eq 'pc') {
3115         $machine = "pc-i440fx-$pin_version";
3116     } elsif ($machine eq 'q35') {
3117         $machine = "pc-q35-$pin_version";
3118     } elsif ($machine eq 'virt') {
3119         $machine = "virt-$pin_version";
3120     } else {
3121         warn "unknown machine type '$machine', not touching that!\n";
3122     }
3123
3124     return $machine;
3125 }
3126
3127 sub get_vm_machine {
3128     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3129
3130     my $machine = $forcemachine || $conf->{machine};
3131
3132     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3133         $kvmversion //= kvm_user_version();
3134         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3135         # layout which confuses windows quite a bit and may result in various regressions..
3136         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3137         if (windows_version($conf->{ostype})) {
3138             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3139         }
3140         $arch //= 'x86_64';
3141         $machine ||= $default_machines->{$arch};
3142         if ($add_pve_version) {
3143             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3144             $machine .= "+pve$pvever";
3145         }
3146     }
3147
3148     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3149         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3150         $machine = $1 if $is_pxe;
3151
3152         # for version-pinned machines that do not include a pve-version (e.g.
3153         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3154         $machine .= '+pve0';
3155
3156         $machine .= '.pxe' if $is_pxe;
3157     }
3158
3159     return $machine;
3160 }
3161
3162 sub get_ovmf_files($$) {
3163     my ($arch, $efidisk) = @_;
3164
3165     my $types = $OVMF->{$arch}
3166         or die "no OVMF images known for architecture '$arch'\n";
3167
3168     my $type = 'default';
3169     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3170         $type = $efidisk->{'pre-enrolled-keys'} ? "4m-ms" : "4m";
3171     }
3172
3173     return $types->{$type}->@*;
3174 }
3175
3176 my $Arch2Qemu = {
3177     aarch64 => '/usr/bin/qemu-system-aarch64',
3178     x86_64 => '/usr/bin/qemu-system-x86_64',
3179 };
3180 sub get_command_for_arch($) {
3181     my ($arch) = @_;
3182     return '/usr/bin/kvm' if is_native($arch);
3183
3184     my $cmd = $Arch2Qemu->{$arch}
3185         or die "don't know how to emulate architecture '$arch'\n";
3186     return $cmd;
3187 }
3188
3189 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3190 # to use in a QEMU command line (-cpu element), first array_intersect the result
3191 # of query_supported_ with query_understood_. This is necessary because:
3192 #
3193 # a) query_understood_ returns flags the host cannot use and
3194 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3195 #    flags, but CPU settings - with most of them being flags. Those settings
3196 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3197 #
3198 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3199 # expensive. If you need the value returned from this, you can get it much
3200 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3201 # $accel being 'kvm' or 'tcg'.
3202 #
3203 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3204 # changes, automatically populating pmxcfs.
3205 #
3206 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3207 # since kvm and tcg machines support different flags
3208 #
3209 sub query_supported_cpu_flags {
3210     my ($arch) = @_;
3211
3212     $arch //= get_host_arch();
3213     my $default_machine = $default_machines->{$arch};
3214
3215     my $flags = {};
3216
3217     # FIXME: Once this is merged, the code below should work for ARM as well:
3218     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3219     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3220         $arch eq "aarch64";
3221
3222     my $kvm_supported = defined(kvm_version());
3223     my $qemu_cmd = get_command_for_arch($arch);
3224     my $fakevmid = -1;
3225     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3226
3227     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3228     my $query_supported_run_qemu = sub {
3229         my ($kvm) = @_;
3230
3231         my $flags = {};
3232         my $cmd = [
3233             $qemu_cmd,
3234             '-machine', $default_machine,
3235             '-display', 'none',
3236             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3237             '-mon', 'chardev=qmp,mode=control',
3238             '-pidfile', $pidfile,
3239             '-S', '-daemonize'
3240         ];
3241
3242         if (!$kvm) {
3243             push @$cmd, '-accel', 'tcg';
3244         }
3245
3246         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3247         die "QEMU flag querying VM exited with code " . $rc if $rc;
3248
3249         eval {
3250             my $cmd_result = mon_cmd(
3251                 $fakevmid,
3252                 'query-cpu-model-expansion',
3253                 type => 'full',
3254                 model => { name => 'host' }
3255             );
3256
3257             my $props = $cmd_result->{model}->{props};
3258             foreach my $prop (keys %$props) {
3259                 next if $props->{$prop} ne '1';
3260                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3261                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3262                 # We only keep those with underscores, to match /proc/cpuinfo
3263                 $prop =~ s/\.|-/_/g;
3264                 $flags->{$prop} = 1;
3265             }
3266         };
3267         my $err = $@;
3268
3269         # force stop with 10 sec timeout and 'nocheck', always stop, even if QMP failed
3270         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3271
3272         die $err if $err;
3273
3274         return [ sort keys %$flags ];
3275     };
3276
3277     # We need to query QEMU twice, since KVM and TCG have different supported flags
3278     PVE::QemuConfig->lock_config($fakevmid, sub {
3279         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3280         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3281
3282         if ($kvm_supported) {
3283             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3284             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3285         }
3286     });
3287
3288     return $flags;
3289 }
3290
3291 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3292 my $understood_cpu_flag_dir = "/usr/share/kvm";
3293 sub query_understood_cpu_flags {
3294     my $arch = get_host_arch();
3295     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3296
3297     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3298         if ! -e $filepath;
3299
3300     my $raw = file_get_contents($filepath);
3301     $raw =~ s/^\s+|\s+$//g;
3302     my @flags = split(/\s+/, $raw);
3303
3304     return \@flags;
3305 }
3306
3307 my sub get_cpuunits {
3308     my ($conf) = @_;
3309     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3310 }
3311 sub config_to_command {
3312     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3313         $pbs_backing) = @_;
3314
3315     my $cmd = [];
3316     my $globalFlags = [];
3317     my $machineFlags = [];
3318     my $rtcFlags = [];
3319     my $devices = [];
3320     my $pciaddr = '';
3321     my $bridges = {};
3322     my $ostype = $conf->{ostype};
3323     my $winversion = windows_version($ostype);
3324     my $kvm = $conf->{kvm};
3325     my $nodename = nodename();
3326
3327     my $arch = get_vm_arch($conf);
3328     my $kvm_binary = get_command_for_arch($arch);
3329     my $kvmver = kvm_user_version($kvm_binary);
3330
3331     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3332         $kvmver //= "undefined";
3333         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3334     }
3335
3336     my $add_pve_version = min_version($kvmver, 4, 1);
3337
3338     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3339     my $machine_version = extract_version($machine_type, $kvmver);
3340     $kvm //= 1 if is_native($arch);
3341
3342     $machine_version =~ m/(\d+)\.(\d+)/;
3343     my ($machine_major, $machine_minor) = ($1, $2);
3344
3345     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3346         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3347     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3348         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3349             ." please upgrade node '$nodename'\n"
3350     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3351         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3352         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3353             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3354             ." node '$nodename'\n";
3355     }
3356
3357     # if a specific +pve version is required for a feature, use $version_guard
3358     # instead of min_version to allow machines to be run with the minimum
3359     # required version
3360     my $required_pve_version = 0;
3361     my $version_guard = sub {
3362         my ($major, $minor, $pve) = @_;
3363         return 0 if !min_version($machine_version, $major, $minor, $pve);
3364         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3365         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3366         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3367         return 1;
3368     };
3369
3370     if ($kvm && !defined kvm_version()) {
3371         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3372             ." or enable in BIOS.\n";
3373     }
3374
3375     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3376     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3377     my $use_old_bios_files = undef;
3378     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3379
3380     my $cpuunits = get_cpuunits($conf);
3381
3382     push @$cmd, $kvm_binary;
3383
3384     push @$cmd, '-id', $vmid;
3385
3386     my $vmname = $conf->{name} || "vm$vmid";
3387
3388     push @$cmd, '-name', $vmname;
3389
3390     push @$cmd, '-no-shutdown';
3391
3392     my $use_virtio = 0;
3393
3394     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3395     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3396     push @$cmd, '-mon', "chardev=qmp,mode=control";
3397
3398     if (min_version($machine_version, 2, 12)) {
3399         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3400         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3401     }
3402
3403     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3404
3405     push @$cmd, '-daemonize';
3406
3407     if ($conf->{smbios1}) {
3408         my $smbios_conf = parse_smbios1($conf->{smbios1});
3409         if ($smbios_conf->{base64}) {
3410             # Do not pass base64 flag to qemu
3411             delete $smbios_conf->{base64};
3412             my $smbios_string = "";
3413             foreach my $key (keys %$smbios_conf) {
3414                 my $value;
3415                 if ($key eq "uuid") {
3416                     $value = $smbios_conf->{uuid}
3417                 } else {
3418                     $value = decode_base64($smbios_conf->{$key});
3419                 }
3420                 # qemu accepts any binary data, only commas need escaping by double comma
3421                 $value =~ s/,/,,/g;
3422                 $smbios_string .= "," . $key . "=" . $value if $value;
3423             }
3424             push @$cmd, '-smbios', "type=1" . $smbios_string;
3425         } else {
3426             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3427         }
3428     }
3429
3430     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3431         my $d;
3432         if (my $efidisk = $conf->{efidisk0}) {
3433             $d = parse_drive('efidisk0', $efidisk);
3434         }
3435
3436         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d);
3437         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3438
3439         my ($path, $format);
3440         my $read_only_str = '';
3441         if ($d) {
3442             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3443             $format = $d->{format};
3444             if ($storeid) {
3445                 $path = PVE::Storage::path($storecfg, $d->{file});
3446                 if (!defined($format)) {
3447                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3448                     $format = qemu_img_format($scfg, $volname);
3449                 }
3450             } else {
3451                 $path = $d->{file};
3452                 die "efidisk format must be specified\n"
3453                     if !defined($format);
3454             }
3455
3456             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3457         } else {
3458             warn "no efidisk configured! Using temporary efivars disk.\n";
3459             $path = "/tmp/$vmid-ovmf.fd";
3460             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3461             $format = 'raw';
3462         }
3463
3464         my $size_str = "";
3465
3466         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3467             $size_str = ",size=" . (-s $ovmf_vars);
3468         }
3469
3470         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3471         my $cache = "";
3472         if ($path =~ m/^rbd:/) {
3473                 $cache = ',cache=writeback';
3474                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3475         }
3476
3477         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3478         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3479     }
3480
3481     # load q35 config
3482     if ($q35) {
3483         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3484         if (min_version($machine_version, 4, 0)) {
3485             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3486         } else {
3487             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3488         }
3489     }
3490
3491     if ($conf->{vmgenid}) {
3492         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3493     }
3494
3495     # add usb controllers
3496     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3497         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3498     push @$devices, @usbcontrollers if @usbcontrollers;
3499     my $vga = parse_vga($conf->{vga});
3500
3501     my $qxlnum = vga_conf_has_spice($conf->{vga});
3502     $vga->{type} = 'qxl' if $qxlnum;
3503
3504     if (!$vga->{type}) {
3505         if ($arch eq 'aarch64') {
3506             $vga->{type} = 'virtio';
3507         } elsif (min_version($machine_version, 2, 9)) {
3508             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3509         } else {
3510             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3511         }
3512     }
3513
3514     # enable absolute mouse coordinates (needed by vnc)
3515     my $tablet;
3516     if (defined($conf->{tablet})) {
3517         $tablet = $conf->{tablet};
3518     } else {
3519         $tablet = $defaults->{tablet};
3520         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3521         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3522     }
3523
3524     if ($tablet) {
3525         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3526         my $kbd = print_keyboarddevice_full($conf, $arch);
3527         push @$devices, '-device', $kbd if defined($kbd);
3528     }
3529
3530     my $bootorder = device_bootorder($conf);
3531
3532     # host pci device passthrough
3533     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3534         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3535
3536     # usb devices
3537     my $usb_dev_features = {};
3538     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3539
3540     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3541         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3542     push @$devices, @usbdevices if @usbdevices;
3543
3544     # serial devices
3545     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3546         if (my $path = $conf->{"serial$i"}) {
3547             if ($path eq 'socket') {
3548                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3549                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3550                 # On aarch64, serial0 is the UART device. Qemu only allows
3551                 # connecting UART devices via the '-serial' command line, as
3552                 # the device has a fixed slot on the hardware...
3553                 if ($arch eq 'aarch64' && $i == 0) {
3554                     push @$devices, '-serial', "chardev:serial$i";
3555                 } else {
3556                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3557                 }
3558             } else {
3559                 die "no such serial device\n" if ! -c $path;
3560                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3561                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3562             }
3563         }
3564     }
3565
3566     # parallel devices
3567     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3568         if (my $path = $conf->{"parallel$i"}) {
3569             die "no such parallel device\n" if ! -c $path;
3570             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3571             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3572             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3573         }
3574     }
3575
3576     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3577         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3578         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3579         push @$devices, @$audio_devs;
3580     }
3581
3582     add_tpm_device($vmid, $devices, $conf);
3583
3584     my $sockets = 1;
3585     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3586     $sockets = $conf->{sockets} if  $conf->{sockets};
3587
3588     my $cores = $conf->{cores} || 1;
3589
3590     my $maxcpus = $sockets * $cores;
3591
3592     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3593
3594     my $allowed_vcpus = $cpuinfo->{cpus};
3595
3596     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3597         if ($allowed_vcpus < $maxcpus);
3598
3599     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3600
3601         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3602         for (my $i = 2; $i <= $vcpus; $i++)  {
3603             my $cpustr = print_cpu_device($conf,$i);
3604             push @$cmd, '-device', $cpustr;
3605         }
3606
3607     } else {
3608
3609         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3610     }
3611     push @$cmd, '-nodefaults';
3612
3613     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3614
3615     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3616
3617     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3618
3619     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3620         push @$devices, '-device', print_vga_device(
3621             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3622         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3623         push @$cmd,  '-vnc', "unix:$socket,password=on";
3624     } else {
3625         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3626         push @$cmd, '-nographic';
3627     }
3628
3629     # time drift fix
3630     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3631     my $useLocaltime = $conf->{localtime};
3632
3633     if ($winversion >= 5) { # windows
3634         $useLocaltime = 1 if !defined($conf->{localtime});
3635
3636         # use time drift fix when acpi is enabled
3637         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3638             $tdf = 1 if !defined($conf->{tdf});
3639         }
3640     }
3641
3642     if ($winversion >= 6) {
3643         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3644         push @$cmd, '-no-hpet';
3645     }
3646
3647     push @$rtcFlags, 'driftfix=slew' if $tdf;
3648
3649     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3650         push @$rtcFlags, "base=$conf->{startdate}";
3651     } elsif ($useLocaltime) {
3652         push @$rtcFlags, 'base=localtime';
3653     }
3654
3655     if ($forcecpu) {
3656         push @$cmd, '-cpu', $forcecpu;
3657     } else {
3658         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3659     }
3660
3661     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3662
3663     push @$cmd, '-S' if $conf->{freeze};
3664
3665     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3666
3667     my $guest_agent = parse_guest_agent($conf);
3668
3669     if ($guest_agent->{enabled}) {
3670         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3671         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3672
3673         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3674             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3675             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3676             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3677         } elsif ($guest_agent->{type} eq 'isa') {
3678             push @$devices, '-device', "isa-serial,chardev=qga0";
3679         }
3680     }
3681
3682     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3683     if ($rng && $version_guard->(4, 1, 2)) {
3684         check_rng_source($rng->{source});
3685
3686         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3687         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3688         my $limiter_str = "";
3689         if ($max_bytes) {
3690             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3691         }
3692
3693         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3694         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3695         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3696     }
3697
3698     my $spice_port;
3699
3700     if ($qxlnum) {
3701         if ($qxlnum > 1) {
3702             if ($winversion){
3703                 for (my $i = 1; $i < $qxlnum; $i++){
3704                     push @$devices, '-device', print_vga_device(
3705                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3706                 }
3707             } else {
3708                 # assume other OS works like Linux
3709                 my ($ram, $vram) = ("134217728", "67108864");
3710                 if ($vga->{memory}) {
3711                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3712                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3713                 }
3714                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3715                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3716             }
3717         }
3718
3719         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3720
3721         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3722         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3723         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3724
3725         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3726         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3727         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3728
3729         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3730         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3731
3732         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3733         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3734         if ($spice_enhancement->{foldersharing}) {
3735             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3736             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3737         }
3738
3739         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3740         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3741             if $spice_enhancement->{videostreaming};
3742
3743         push @$devices, '-spice', "$spice_opts";
3744     }
3745
3746     # enable balloon by default, unless explicitly disabled
3747     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3748         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3749         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3750     }
3751
3752     if ($conf->{watchdog}) {
3753         my $wdopts = parse_watchdog($conf->{watchdog});
3754         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3755         my $watchdog = $wdopts->{model} || 'i6300esb';
3756         push @$devices, '-device', "$watchdog$pciaddr";
3757         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3758     }
3759
3760     my $vollist = [];
3761     my $scsicontroller = {};
3762     my $ahcicontroller = {};
3763     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3764
3765     # Add iscsi initiator name if available
3766     if (my $initiator = get_initiator_name()) {
3767         push @$devices, '-iscsi', "initiator-name=$initiator";
3768     }
3769
3770     PVE::QemuConfig->foreach_volume($conf, sub {
3771         my ($ds, $drive) = @_;
3772
3773         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3774             check_volume_storage_type($storecfg, $drive->{file});
3775             push @$vollist, $drive->{file};
3776         }
3777
3778         # ignore efidisk here, already added in bios/fw handling code above
3779         return if $drive->{interface} eq 'efidisk';
3780         # similar for TPM
3781         return if $drive->{interface} eq 'tpmstate';
3782
3783         $use_virtio = 1 if $ds =~ m/^virtio/;
3784
3785         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3786
3787         if ($drive->{interface} eq 'virtio'){
3788            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3789         }
3790
3791         if ($drive->{interface} eq 'scsi') {
3792
3793             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3794
3795             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3796                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3797
3798             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3799             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3800
3801             my $iothread = '';
3802             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3803                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3804                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3805             } elsif ($drive->{iothread}) {
3806                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3807             }
3808
3809             my $queues = '';
3810             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3811                 $queues = ",num_queues=$drive->{queues}";
3812             }
3813
3814             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3815                 if !$scsicontroller->{$controller};
3816             $scsicontroller->{$controller}=1;
3817         }
3818
3819         if ($drive->{interface} eq 'sata') {
3820             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3821             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3822             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3823                 if !$ahcicontroller->{$controller};
3824             $ahcicontroller->{$controller}=1;
3825         }
3826
3827         my $pbs_conf = $pbs_backing->{$ds};
3828         my $pbs_name = undef;
3829         if ($pbs_conf) {
3830             $pbs_name = "drive-$ds-pbs";
3831             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3832         }
3833
3834         my $drive_cmd = print_drive_commandline_full(
3835             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3836
3837         # extra protection for templates, but SATA and IDE don't support it..
3838         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3839
3840         push @$devices, '-drive',$drive_cmd;
3841         push @$devices, '-device', print_drivedevice_full(
3842             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3843     });
3844
3845     for (my $i = 0; $i < $MAX_NETS; $i++) {
3846         my $netname = "net$i";
3847
3848         next if !$conf->{$netname};
3849         my $d = parse_net($conf->{$netname});
3850         next if !$d;
3851
3852         $use_virtio = 1 if $d->{model} eq 'virtio';
3853
3854         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3855
3856         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3857         push @$devices, '-netdev', $netdevfull;
3858
3859         my $netdevicefull = print_netdevice_full(
3860             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3861
3862         push @$devices, '-device', $netdevicefull;
3863     }
3864
3865     if ($conf->{ivshmem}) {
3866         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3867
3868         my $bus;
3869         if ($q35) {
3870             $bus = print_pcie_addr("ivshmem");
3871         } else {
3872             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3873         }
3874
3875         my $ivshmem_name = $ivshmem->{name} // $vmid;
3876         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3877
3878         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3879         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3880             .",size=$ivshmem->{size}M";
3881     }
3882
3883     # pci.4 is nested in pci.1
3884     $bridges->{1} = 1 if $bridges->{4};
3885
3886     if (!$q35) {
3887         # add pci bridges
3888         if (min_version($machine_version, 2, 3)) {
3889            $bridges->{1} = 1;
3890            $bridges->{2} = 1;
3891         }
3892
3893         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3894
3895     }
3896
3897     for my $k (sort {$b cmp $a} keys %$bridges) {
3898         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3899
3900         my $k_name = $k;
3901         if ($k == 2 && $legacy_igd) {
3902             $k_name = "$k-igd";
3903         }
3904         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3905
3906         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3907         if ($q35) {
3908             # add after -readconfig pve-q35.cfg
3909             splice @$devices, 2, 0, '-device', $devstr;
3910         } else {
3911             unshift @$devices, '-device', $devstr if $k > 0;
3912         }
3913     }
3914
3915     if (!$kvm) {
3916         push @$machineFlags, 'accel=tcg';
3917     }
3918
3919     my $machine_type_min = $machine_type;
3920     if ($add_pve_version) {
3921         $machine_type_min =~ s/\+pve\d+$//;
3922         $machine_type_min .= "+pve$required_pve_version";
3923     }
3924     push @$machineFlags, "type=${machine_type_min}";
3925
3926     push @$cmd, @$devices;
3927     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3928     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3929     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3930
3931     if (my $vmstate = $conf->{vmstate}) {
3932         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3933         push @$vollist, $vmstate;
3934         push @$cmd, '-loadstate', $statepath;
3935         print "activating and using '$vmstate' as vmstate\n";
3936     }
3937
3938     if (PVE::QemuConfig->is_template($conf)) {
3939         # needed to workaround base volumes being read-only
3940         push @$cmd, '-snapshot';
3941     }
3942
3943     # add custom args
3944     if ($conf->{args}) {
3945         my $aa = PVE::Tools::split_args($conf->{args});
3946         push @$cmd, @$aa;
3947     }
3948
3949     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3950 }
3951
3952 sub check_rng_source {
3953     my ($source) = @_;
3954
3955     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3956     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3957         if ! -e $source;
3958
3959     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3960     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3961         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3962         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3963         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3964             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3965             ." to the host.\n";
3966     }
3967 }
3968
3969 sub spice_port {
3970     my ($vmid) = @_;
3971
3972     my $res = mon_cmd($vmid, 'query-spice');
3973
3974     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3975 }
3976
3977 sub vm_devices_list {
3978     my ($vmid) = @_;
3979
3980     my $res = mon_cmd($vmid, 'query-pci');
3981     my $devices_to_check = [];
3982     my $devices = {};
3983     foreach my $pcibus (@$res) {
3984         push @$devices_to_check, @{$pcibus->{devices}},
3985     }
3986
3987     while (@$devices_to_check) {
3988         my $to_check = [];
3989         for my $d (@$devices_to_check) {
3990             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3991             next if !$d->{'pci_bridge'};
3992
3993             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3994             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3995         }
3996         $devices_to_check = $to_check;
3997     }
3998
3999     my $resblock = mon_cmd($vmid, 'query-block');
4000     foreach my $block (@$resblock) {
4001         if($block->{device} =~ m/^drive-(\S+)/){
4002                 $devices->{$1} = 1;
4003         }
4004     }
4005
4006     my $resmice = mon_cmd($vmid, 'query-mice');
4007     foreach my $mice (@$resmice) {
4008         if ($mice->{name} eq 'QEMU HID Tablet') {
4009             $devices->{tablet} = 1;
4010             last;
4011         }
4012     }
4013
4014     # for usb devices there is no query-usb
4015     # but we can iterate over the entries in
4016     # qom-list path=/machine/peripheral
4017     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4018     foreach my $per (@$resperipheral) {
4019         if ($per->{name} =~ m/^usb\d+$/) {
4020             $devices->{$per->{name}} = 1;
4021         }
4022     }
4023
4024     return $devices;
4025 }
4026
4027 sub vm_deviceplug {
4028     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4029
4030     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4031
4032     my $devices_list = vm_devices_list($vmid);
4033     return 1 if defined($devices_list->{$deviceid});
4034
4035     # add PCI bridge if we need it for the device
4036     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4037
4038     if ($deviceid eq 'tablet') {
4039
4040         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4041
4042     } elsif ($deviceid eq 'keyboard') {
4043
4044         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4045
4046     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4047
4048         die "usb hotplug currently not reliable\n";
4049         # since we can't reliably hot unplug all added usb devices and usb
4050         # passthrough breaks live migration we disable usb hotplugging for now
4051         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4052
4053     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4054
4055         qemu_iothread_add($vmid, $deviceid, $device);
4056
4057         qemu_driveadd($storecfg, $vmid, $device);
4058         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4059
4060         qemu_deviceadd($vmid, $devicefull);
4061         eval { qemu_deviceaddverify($vmid, $deviceid); };
4062         if (my $err = $@) {
4063             eval { qemu_drivedel($vmid, $deviceid); };
4064             warn $@ if $@;
4065             die $err;
4066         }
4067
4068     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4069
4070
4071         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4072         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4073         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4074
4075         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4076
4077         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4078             qemu_iothread_add($vmid, $deviceid, $device);
4079             $devicefull .= ",iothread=iothread-$deviceid";
4080         }
4081
4082         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4083             $devicefull .= ",num_queues=$device->{queues}";
4084         }
4085
4086         qemu_deviceadd($vmid, $devicefull);
4087         qemu_deviceaddverify($vmid, $deviceid);
4088
4089     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4090
4091         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4092         qemu_driveadd($storecfg, $vmid, $device);
4093
4094         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4095         eval { qemu_deviceadd($vmid, $devicefull); };
4096         if (my $err = $@) {
4097             eval { qemu_drivedel($vmid, $deviceid); };
4098             warn $@ if $@;
4099             die $err;
4100         }
4101
4102     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4103
4104         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4105
4106         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4107         my $use_old_bios_files = undef;
4108         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4109
4110         my $netdevicefull = print_netdevice_full(
4111             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4112         qemu_deviceadd($vmid, $netdevicefull);
4113         eval {
4114             qemu_deviceaddverify($vmid, $deviceid);
4115             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4116         };
4117         if (my $err = $@) {
4118             eval { qemu_netdevdel($vmid, $deviceid); };
4119             warn $@ if $@;
4120             die $err;
4121         }
4122
4123     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4124
4125         my $bridgeid = $2;
4126         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4127         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4128
4129         qemu_deviceadd($vmid, $devicefull);
4130         qemu_deviceaddverify($vmid, $deviceid);
4131
4132     } else {
4133         die "can't hotplug device '$deviceid'\n";
4134     }
4135
4136     return 1;
4137 }
4138
4139 # fixme: this should raise exceptions on error!
4140 sub vm_deviceunplug {
4141     my ($vmid, $conf, $deviceid) = @_;
4142
4143     my $devices_list = vm_devices_list($vmid);
4144     return 1 if !defined($devices_list->{$deviceid});
4145
4146     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4147     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4148
4149     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4150         qemu_devicedel($vmid, $deviceid);
4151     } elsif ($deviceid =~ m/^usb\d+$/) {
4152         die "usb hotplug currently not reliable\n";
4153         # when unplugging usb devices this way, there may be remaining usb
4154         # controllers/hubs so we disable it for now
4155         #qemu_devicedel($vmid, $deviceid);
4156         #qemu_devicedelverify($vmid, $deviceid);
4157     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4158         my $device = parse_drive($deviceid, $conf->{$deviceid});
4159
4160         qemu_devicedel($vmid, $deviceid);
4161         qemu_devicedelverify($vmid, $deviceid);
4162         qemu_drivedel($vmid, $deviceid);
4163         qemu_iothread_del($vmid, $deviceid, $device);
4164     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4165         qemu_devicedel($vmid, $deviceid);
4166         qemu_devicedelverify($vmid, $deviceid);
4167     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4168         my $device = parse_drive($deviceid, $conf->{$deviceid});
4169
4170         qemu_devicedel($vmid, $deviceid);
4171         qemu_drivedel($vmid, $deviceid);
4172         qemu_deletescsihw($conf, $vmid, $deviceid);
4173
4174         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4175             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4176     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4177         qemu_devicedel($vmid, $deviceid);
4178         qemu_devicedelverify($vmid, $deviceid);
4179         qemu_netdevdel($vmid, $deviceid);
4180     } else {
4181         die "can't unplug device '$deviceid'\n";
4182     }
4183
4184     return 1;
4185 }
4186
4187 sub qemu_deviceadd {
4188     my ($vmid, $devicefull) = @_;
4189
4190     $devicefull = "driver=".$devicefull;
4191     my %options =  split(/[=,]/, $devicefull);
4192
4193     mon_cmd($vmid, "device_add" , %options);
4194 }
4195
4196 sub qemu_devicedel {
4197     my ($vmid, $deviceid) = @_;
4198
4199     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4200 }
4201
4202 sub qemu_iothread_add {
4203     my ($vmid, $deviceid, $device) = @_;
4204
4205     if ($device->{iothread}) {
4206         my $iothreads = vm_iothreads_list($vmid);
4207         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4208     }
4209 }
4210
4211 sub qemu_iothread_del {
4212     my ($vmid, $deviceid, $device) = @_;
4213
4214     if ($device->{iothread}) {
4215         my $iothreads = vm_iothreads_list($vmid);
4216         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4217     }
4218 }
4219
4220 sub qemu_objectadd {
4221     my ($vmid, $objectid, $qomtype) = @_;
4222
4223     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4224
4225     return 1;
4226 }
4227
4228 sub qemu_objectdel {
4229     my ($vmid, $objectid) = @_;
4230
4231     mon_cmd($vmid, "object-del", id => $objectid);
4232
4233     return 1;
4234 }
4235
4236 sub qemu_driveadd {
4237     my ($storecfg, $vmid, $device) = @_;
4238
4239     my $kvmver = get_running_qemu_version($vmid);
4240     my $io_uring = min_version($kvmver, 6, 0);
4241     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4242     $drive =~ s/\\/\\\\/g;
4243     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4244
4245     # If the command succeeds qemu prints: "OK"
4246     return 1 if $ret =~ m/OK/s;
4247
4248     die "adding drive failed: $ret\n";
4249 }
4250
4251 sub qemu_drivedel {
4252     my ($vmid, $deviceid) = @_;
4253
4254     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4255     $ret =~ s/^\s+//;
4256
4257     return 1 if $ret eq "";
4258
4259     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4260     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4261
4262     die "deleting drive $deviceid failed : $ret\n";
4263 }
4264
4265 sub qemu_deviceaddverify {
4266     my ($vmid, $deviceid) = @_;
4267
4268     for (my $i = 0; $i <= 5; $i++) {
4269          my $devices_list = vm_devices_list($vmid);
4270          return 1 if defined($devices_list->{$deviceid});
4271          sleep 1;
4272     }
4273
4274     die "error on hotplug device '$deviceid'\n";
4275 }
4276
4277
4278 sub qemu_devicedelverify {
4279     my ($vmid, $deviceid) = @_;
4280
4281     # need to verify that the device is correctly removed as device_del
4282     # is async and empty return is not reliable
4283
4284     for (my $i = 0; $i <= 5; $i++) {
4285          my $devices_list = vm_devices_list($vmid);
4286          return 1 if !defined($devices_list->{$deviceid});
4287          sleep 1;
4288     }
4289
4290     die "error on hot-unplugging device '$deviceid'\n";
4291 }
4292
4293 sub qemu_findorcreatescsihw {
4294     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4295
4296     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4297
4298     my $scsihwid="$controller_prefix$controller";
4299     my $devices_list = vm_devices_list($vmid);
4300
4301     if (!defined($devices_list->{$scsihwid})) {
4302         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4303     }
4304
4305     return 1;
4306 }
4307
4308 sub qemu_deletescsihw {
4309     my ($conf, $vmid, $opt) = @_;
4310
4311     my $device = parse_drive($opt, $conf->{$opt});
4312
4313     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4314         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4315         return 1;
4316     }
4317
4318     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4319
4320     my $devices_list = vm_devices_list($vmid);
4321     foreach my $opt (keys %{$devices_list}) {
4322         if (is_valid_drivename($opt)) {
4323             my $drive = parse_drive($opt, $conf->{$opt});
4324             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4325                 return 1;
4326             }
4327         }
4328     }
4329
4330     my $scsihwid="scsihw$controller";
4331
4332     vm_deviceunplug($vmid, $conf, $scsihwid);
4333
4334     return 1;
4335 }
4336
4337 sub qemu_add_pci_bridge {
4338     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4339
4340     my $bridges = {};
4341
4342     my $bridgeid;
4343
4344     print_pci_addr($device, $bridges, $arch, $machine_type);
4345
4346     while (my ($k, $v) = each %$bridges) {
4347         $bridgeid = $k;
4348     }
4349     return 1 if !defined($bridgeid) || $bridgeid < 1;
4350
4351     my $bridge = "pci.$bridgeid";
4352     my $devices_list = vm_devices_list($vmid);
4353
4354     if (!defined($devices_list->{$bridge})) {
4355         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4356     }
4357
4358     return 1;
4359 }
4360
4361 sub qemu_set_link_status {
4362     my ($vmid, $device, $up) = @_;
4363
4364     mon_cmd($vmid, "set_link", name => $device,
4365                up => $up ? JSON::true : JSON::false);
4366 }
4367
4368 sub qemu_netdevadd {
4369     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4370
4371     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4372     my %options =  split(/[=,]/, $netdev);
4373
4374     if (defined(my $vhost = $options{vhost})) {
4375         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4376     }
4377
4378     if (defined(my $queues = $options{queues})) {
4379         $options{queues} = $queues + 0;
4380     }
4381
4382     mon_cmd($vmid, "netdev_add",  %options);
4383     return 1;
4384 }
4385
4386 sub qemu_netdevdel {
4387     my ($vmid, $deviceid) = @_;
4388
4389     mon_cmd($vmid, "netdev_del", id => $deviceid);
4390 }
4391
4392 sub qemu_usb_hotplug {
4393     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4394
4395     return if !$device;
4396
4397     # remove the old one first
4398     vm_deviceunplug($vmid, $conf, $deviceid);
4399
4400     # check if xhci controller is necessary and available
4401     if ($device->{usb3}) {
4402
4403         my $devicelist = vm_devices_list($vmid);
4404
4405         if (!$devicelist->{xhci}) {
4406             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4407             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4408         }
4409     }
4410     my $d = parse_usb_device($device->{host});
4411     $d->{usb3} = $device->{usb3};
4412
4413     # add the new one
4414     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4415 }
4416
4417 sub qemu_cpu_hotplug {
4418     my ($vmid, $conf, $vcpus) = @_;
4419
4420     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4421
4422     my $sockets = 1;
4423     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4424     $sockets = $conf->{sockets} if  $conf->{sockets};
4425     my $cores = $conf->{cores} || 1;
4426     my $maxcpus = $sockets * $cores;
4427
4428     $vcpus = $maxcpus if !$vcpus;
4429
4430     die "you can't add more vcpus than maxcpus\n"
4431         if $vcpus > $maxcpus;
4432
4433     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4434
4435     if ($vcpus < $currentvcpus) {
4436
4437         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4438
4439             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4440                 qemu_devicedel($vmid, "cpu$i");
4441                 my $retry = 0;
4442                 my $currentrunningvcpus = undef;
4443                 while (1) {
4444                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4445                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4446                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4447                     $retry++;
4448                     sleep 1;
4449                 }
4450                 #update conf after each succesfull cpu unplug
4451                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4452                 PVE::QemuConfig->write_config($vmid, $conf);
4453             }
4454         } else {
4455             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4456         }
4457
4458         return;
4459     }
4460
4461     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4462     die "vcpus in running vm does not match its configuration\n"
4463         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4464
4465     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4466
4467         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4468             my $cpustr = print_cpu_device($conf, $i);
4469             qemu_deviceadd($vmid, $cpustr);
4470
4471             my $retry = 0;
4472             my $currentrunningvcpus = undef;
4473             while (1) {
4474                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4475                 last if scalar(@{$currentrunningvcpus}) == $i;
4476                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4477                 sleep 1;
4478                 $retry++;
4479             }
4480             #update conf after each succesfull cpu hotplug
4481             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4482             PVE::QemuConfig->write_config($vmid, $conf);
4483         }
4484     } else {
4485
4486         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4487             mon_cmd($vmid, "cpu-add", id => int($i));
4488         }
4489     }
4490 }
4491
4492 sub qemu_block_set_io_throttle {
4493     my ($vmid, $deviceid,
4494         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4495         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4496         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4497         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4498
4499     return if !check_running($vmid) ;
4500
4501     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4502         bps => int($bps),
4503         bps_rd => int($bps_rd),
4504         bps_wr => int($bps_wr),
4505         iops => int($iops),
4506         iops_rd => int($iops_rd),
4507         iops_wr => int($iops_wr),
4508         bps_max => int($bps_max),
4509         bps_rd_max => int($bps_rd_max),
4510         bps_wr_max => int($bps_wr_max),
4511         iops_max => int($iops_max),
4512         iops_rd_max => int($iops_rd_max),
4513         iops_wr_max => int($iops_wr_max),
4514         bps_max_length => int($bps_max_length),
4515         bps_rd_max_length => int($bps_rd_max_length),
4516         bps_wr_max_length => int($bps_wr_max_length),
4517         iops_max_length => int($iops_max_length),
4518         iops_rd_max_length => int($iops_rd_max_length),
4519         iops_wr_max_length => int($iops_wr_max_length),
4520     );
4521
4522 }
4523
4524 sub qemu_block_resize {
4525     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4526
4527     my $running = check_running($vmid);
4528
4529     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4530
4531     return if !$running;
4532
4533     my $padding = (1024 - $size % 1024) % 1024;
4534     $size = $size + $padding;
4535
4536     mon_cmd(
4537         $vmid,
4538         "block_resize",
4539         device => $deviceid,
4540         size => int($size),
4541         timeout => 60,
4542     );
4543 }
4544
4545 sub qemu_volume_snapshot {
4546     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4547
4548     my $running = check_running($vmid);
4549
4550     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4551         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4552     } else {
4553         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4554     }
4555 }
4556
4557 sub qemu_volume_snapshot_delete {
4558     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4559
4560     my $running = check_running($vmid);
4561
4562     if($running) {
4563
4564         $running = undef;
4565         my $conf = PVE::QemuConfig->load_config($vmid);
4566         PVE::QemuConfig->foreach_volume($conf, sub {
4567             my ($ds, $drive) = @_;
4568             $running = 1 if $drive->{file} eq $volid;
4569         });
4570     }
4571
4572     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4573         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4574     } else {
4575         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4576     }
4577 }
4578
4579 sub set_migration_caps {
4580     my ($vmid, $savevm) = @_;
4581
4582     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4583
4584     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4585     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4586
4587     my $cap_ref = [];
4588
4589     my $enabled_cap = {
4590         "auto-converge" => 1,
4591         "xbzrle" => 1,
4592         "x-rdma-pin-all" => 0,
4593         "zero-blocks" => 0,
4594         "compress" => 0,
4595         "dirty-bitmaps" => $dirty_bitmaps,
4596     };
4597
4598     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4599
4600     for my $supported_capability (@$supported_capabilities) {
4601         push @$cap_ref, {
4602             capability => $supported_capability->{capability},
4603             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4604         };
4605     }
4606
4607     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4608 }
4609
4610 sub foreach_volid {
4611     my ($conf, $func, @param) = @_;
4612
4613     my $volhash = {};
4614
4615     my $test_volid = sub {
4616         my ($key, $drive, $snapname) = @_;
4617
4618         my $volid = $drive->{file};
4619         return if !$volid;
4620
4621         $volhash->{$volid}->{cdrom} //= 1;
4622         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4623
4624         my $replicate = $drive->{replicate} // 1;
4625         $volhash->{$volid}->{replicate} //= 0;
4626         $volhash->{$volid}->{replicate} = 1 if $replicate;
4627
4628         $volhash->{$volid}->{shared} //= 0;
4629         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4630
4631         $volhash->{$volid}->{referenced_in_config} //= 0;
4632         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4633
4634         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4635             if defined($snapname);
4636
4637         my $size = $drive->{size};
4638         $volhash->{$volid}->{size} //= $size if $size;
4639
4640         $volhash->{$volid}->{is_vmstate} //= 0;
4641         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4642
4643         $volhash->{$volid}->{is_tpmstate} //= 0;
4644         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4645
4646         $volhash->{$volid}->{is_unused} //= 0;
4647         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4648
4649         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4650     };
4651
4652     my $include_opts = {
4653         extra_keys => ['vmstate'],
4654         include_unused => 1,
4655     };
4656
4657     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4658     foreach my $snapname (keys %{$conf->{snapshots}}) {
4659         my $snap = $conf->{snapshots}->{$snapname};
4660         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4661     }
4662
4663     foreach my $volid (keys %$volhash) {
4664         &$func($volid, $volhash->{$volid}, @param);
4665     }
4666 }
4667
4668 my $fast_plug_option = {
4669     'lock' => 1,
4670     'name' => 1,
4671     'onboot' => 1,
4672     'shares' => 1,
4673     'startup' => 1,
4674     'description' => 1,
4675     'protection' => 1,
4676     'vmstatestorage' => 1,
4677     'hookscript' => 1,
4678     'tags' => 1,
4679 };
4680
4681 # hotplug changes in [PENDING]
4682 # $selection hash can be used to only apply specified options, for
4683 # example: { cores => 1 } (only apply changed 'cores')
4684 # $errors ref is used to return error messages
4685 sub vmconfig_hotplug_pending {
4686     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4687
4688     my $defaults = load_defaults();
4689     my $arch = get_vm_arch($conf);
4690     my $machine_type = get_vm_machine($conf, undef, $arch);
4691
4692     # commit values which do not have any impact on running VM first
4693     # Note: those option cannot raise errors, we we do not care about
4694     # $selection and always apply them.
4695
4696     my $add_error = sub {
4697         my ($opt, $msg) = @_;
4698         $errors->{$opt} = "hotplug problem - $msg";
4699     };
4700
4701     my $changes = 0;
4702     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4703         if ($fast_plug_option->{$opt}) {
4704             $conf->{$opt} = $conf->{pending}->{$opt};
4705             delete $conf->{pending}->{$opt};
4706             $changes = 1;
4707         }
4708     }
4709
4710     if ($changes) {
4711         PVE::QemuConfig->write_config($vmid, $conf);
4712     }
4713
4714     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4715
4716     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4717     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4718     foreach my $opt (sort keys %$pending_delete_hash) {
4719         next if $selection && !$selection->{$opt};
4720         my $force = $pending_delete_hash->{$opt}->{force};
4721         eval {
4722             if ($opt eq 'hotplug') {
4723                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4724             } elsif ($opt eq 'tablet') {
4725                 die "skip\n" if !$hotplug_features->{usb};
4726                 if ($defaults->{tablet}) {
4727                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4728                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4729                         if $arch eq 'aarch64';
4730                 } else {
4731                     vm_deviceunplug($vmid, $conf, 'tablet');
4732                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4733                 }
4734             } elsif ($opt =~ m/^usb\d+/) {
4735                 die "skip\n";
4736                 # since we cannot reliably hot unplug usb devices we are disabling it
4737                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4738                 #vm_deviceunplug($vmid, $conf, $opt);
4739             } elsif ($opt eq 'vcpus') {
4740                 die "skip\n" if !$hotplug_features->{cpu};
4741                 qemu_cpu_hotplug($vmid, $conf, undef);
4742             } elsif ($opt eq 'balloon') {
4743                 # enable balloon device is not hotpluggable
4744                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4745                 # here we reset the ballooning value to memory
4746                 my $balloon = $conf->{memory} || $defaults->{memory};
4747                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4748             } elsif ($fast_plug_option->{$opt}) {
4749                 # do nothing
4750             } elsif ($opt =~ m/^net(\d+)$/) {
4751                 die "skip\n" if !$hotplug_features->{network};
4752                 vm_deviceunplug($vmid, $conf, $opt);
4753             } elsif (is_valid_drivename($opt)) {
4754                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4755                 vm_deviceunplug($vmid, $conf, $opt);
4756                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4757             } elsif ($opt =~ m/^memory$/) {
4758                 die "skip\n" if !$hotplug_features->{memory};
4759                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4760             } elsif ($opt eq 'cpuunits') {
4761                 $cgroup->change_cpu_shares(undef, 1024);
4762             } elsif ($opt eq 'cpulimit') {
4763                 $cgroup->change_cpu_quota(-1, 100000);
4764             } else {
4765                 die "skip\n";
4766             }
4767         };
4768         if (my $err = $@) {
4769             &$add_error($opt, $err) if $err ne "skip\n";
4770         } else {
4771             delete $conf->{$opt};
4772             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4773         }
4774     }
4775
4776     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4777     $apply_pending_cloudinit = sub {
4778         return if $apply_pending_cloudinit_done; # once is enough
4779         $apply_pending_cloudinit_done = 1; # once is enough
4780
4781         my ($key, $value) = @_;
4782
4783         my @cloudinit_opts = keys %$confdesc_cloudinit;
4784         foreach my $opt (keys %{$conf->{pending}}) {
4785             next if !grep { $_ eq $opt } @cloudinit_opts;
4786             $conf->{$opt} = delete $conf->{pending}->{$opt};
4787         }
4788
4789         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4790         foreach my $opt (sort keys %$pending_delete_hash) {
4791             next if !grep { $_ eq $opt } @cloudinit_opts;
4792             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4793             delete $conf->{$opt};
4794         }
4795
4796         my $new_conf = { %$conf };
4797         $new_conf->{$key} = $value;
4798         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4799     };
4800
4801     foreach my $opt (keys %{$conf->{pending}}) {
4802         next if $selection && !$selection->{$opt};
4803         my $value = $conf->{pending}->{$opt};
4804         eval {
4805             if ($opt eq 'hotplug') {
4806                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4807             } elsif ($opt eq 'tablet') {
4808                 die "skip\n" if !$hotplug_features->{usb};
4809                 if ($value == 1) {
4810                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4811                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4812                         if $arch eq 'aarch64';
4813                 } elsif ($value == 0) {
4814                     vm_deviceunplug($vmid, $conf, 'tablet');
4815                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4816                 }
4817             } elsif ($opt =~ m/^usb\d+$/) {
4818                 die "skip\n";
4819                 # since we cannot reliably hot unplug usb devices we disable it for now
4820                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4821                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4822                 #die "skip\n" if !$d;
4823                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4824             } elsif ($opt eq 'vcpus') {
4825                 die "skip\n" if !$hotplug_features->{cpu};
4826                 qemu_cpu_hotplug($vmid, $conf, $value);
4827             } elsif ($opt eq 'balloon') {
4828                 # enable/disable balloning device is not hotpluggable
4829                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4830                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4831                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4832
4833                 # allow manual ballooning if shares is set to zero
4834                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4835                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4836                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4837                 }
4838             } elsif ($opt =~ m/^net(\d+)$/) {
4839                 # some changes can be done without hotplug
4840                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4841                                     $vmid, $opt, $value, $arch, $machine_type);
4842             } elsif (is_valid_drivename($opt)) {
4843                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4844                 # some changes can be done without hotplug
4845                 my $drive = parse_drive($opt, $value);
4846                 if (drive_is_cloudinit($drive)) {
4847                     &$apply_pending_cloudinit($opt, $value);
4848                 }
4849                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4850                                      $vmid, $opt, $value, $arch, $machine_type);
4851             } elsif ($opt =~ m/^memory$/) { #dimms
4852                 die "skip\n" if !$hotplug_features->{memory};
4853                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4854             } elsif ($opt eq 'cpuunits') {
4855                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4856             } elsif ($opt eq 'cpulimit') {
4857                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4858                 $cgroup->change_cpu_quota($cpulimit, 100000);
4859             } else {
4860                 die "skip\n";  # skip non-hot-pluggable options
4861             }
4862         };
4863         if (my $err = $@) {
4864             &$add_error($opt, $err) if $err ne "skip\n";
4865         } else {
4866             $conf->{$opt} = $value;
4867             delete $conf->{pending}->{$opt};
4868         }
4869     }
4870
4871     PVE::QemuConfig->write_config($vmid, $conf);
4872 }
4873
4874 sub try_deallocate_drive {
4875     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4876
4877     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4878         my $volid = $drive->{file};
4879         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4880             my $sid = PVE::Storage::parse_volume_id($volid);
4881             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4882
4883             # check if the disk is really unused
4884             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4885                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4886             PVE::Storage::vdisk_free($storecfg, $volid);
4887             return 1;
4888         } else {
4889             # If vm is not owner of this disk remove from config
4890             return 1;
4891         }
4892     }
4893
4894     return;
4895 }
4896
4897 sub vmconfig_delete_or_detach_drive {
4898     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4899
4900     my $drive = parse_drive($opt, $conf->{$opt});
4901
4902     my $rpcenv = PVE::RPCEnvironment::get();
4903     my $authuser = $rpcenv->get_user();
4904
4905     if ($force) {
4906         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4907         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4908     } else {
4909         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4910     }
4911 }
4912
4913
4914
4915 sub vmconfig_apply_pending {
4916     my ($vmid, $conf, $storecfg, $errors) = @_;
4917
4918     my $add_apply_error = sub {
4919         my ($opt, $msg) = @_;
4920         my $err_msg = "unable to apply pending change $opt : $msg";
4921         $errors->{$opt} = $err_msg;
4922         warn $err_msg;
4923     };
4924
4925     # cold plug
4926
4927     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4928     foreach my $opt (sort keys %$pending_delete_hash) {
4929         my $force = $pending_delete_hash->{$opt}->{force};
4930         eval {
4931             if ($opt =~ m/^unused/) {
4932                 die "internal error";
4933             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4934                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4935             }
4936         };
4937         if (my $err = $@) {
4938             $add_apply_error->($opt, $err);
4939         } else {
4940             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4941             delete $conf->{$opt};
4942         }
4943     }
4944
4945     PVE::QemuConfig->cleanup_pending($conf);
4946
4947     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4948         next if $opt eq 'delete'; # just to be sure
4949         eval {
4950             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4951                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4952             }
4953         };
4954         if (my $err = $@) {
4955             $add_apply_error->($opt, $err);
4956         } else {
4957             $conf->{$opt} = delete $conf->{pending}->{$opt};
4958         }
4959     }
4960
4961     # write all changes at once to avoid unnecessary i/o
4962     PVE::QemuConfig->write_config($vmid, $conf);
4963 }
4964
4965 sub vmconfig_update_net {
4966     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4967
4968     my $newnet = parse_net($value);
4969
4970     if ($conf->{$opt}) {
4971         my $oldnet = parse_net($conf->{$opt});
4972
4973         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4974             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4975             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4976             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4977
4978             # for non online change, we try to hot-unplug
4979             die "skip\n" if !$hotplug;
4980             vm_deviceunplug($vmid, $conf, $opt);
4981         } else {
4982
4983             die "internal error" if $opt !~ m/net(\d+)/;
4984             my $iface = "tap${vmid}i$1";
4985
4986             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4987                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4988                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4989                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4990                 PVE::Network::tap_unplug($iface);
4991
4992                 if ($have_sdn) {
4993                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4994                 } else {
4995                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4996                 }
4997             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4998                 # Rate can be applied on its own but any change above needs to
4999                 # include the rate in tap_plug since OVS resets everything.
5000                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5001             }
5002
5003             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5004                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5005             }
5006
5007             return 1;
5008         }
5009     }
5010
5011     if ($hotplug) {
5012         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5013     } else {
5014         die "skip\n";
5015     }
5016 }
5017
5018 sub vmconfig_update_disk {
5019     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5020
5021     my $drive = parse_drive($opt, $value);
5022
5023     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5024         my $media = $drive->{media} || 'disk';
5025         my $oldmedia = $old_drive->{media} || 'disk';
5026         die "unable to change media type\n" if $media ne $oldmedia;
5027
5028         if (!drive_is_cdrom($old_drive)) {
5029
5030             if ($drive->{file} ne $old_drive->{file}) {
5031
5032                 die "skip\n" if !$hotplug;
5033
5034                 # unplug and register as unused
5035                 vm_deviceunplug($vmid, $conf, $opt);
5036                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5037
5038             } else {
5039                 # update existing disk
5040
5041                 # skip non hotpluggable value
5042                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5043                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5044                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5045                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5046                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5047                     die "skip\n";
5048                 }
5049
5050                 # apply throttle
5051                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5052                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5053                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5054                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5055                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5056                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5057                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5058                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5059                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5060                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5061                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5062                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5063                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5064                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5065                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5066                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5067                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5068                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5069
5070                     qemu_block_set_io_throttle(
5071                         $vmid,"drive-$opt",
5072                         ($drive->{mbps} || 0)*1024*1024,
5073                         ($drive->{mbps_rd} || 0)*1024*1024,
5074                         ($drive->{mbps_wr} || 0)*1024*1024,
5075                          $drive->{iops} || 0,
5076                          $drive->{iops_rd} || 0,
5077                          $drive->{iops_wr} || 0,
5078                         ($drive->{mbps_max} || 0)*1024*1024,
5079                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5080                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5081                          $drive->{iops_max} || 0,
5082                          $drive->{iops_rd_max} || 0,
5083                          $drive->{iops_wr_max} || 0,
5084                          $drive->{bps_max_length} || 1,
5085                          $drive->{bps_rd_max_length} || 1,
5086                          $drive->{bps_wr_max_length} || 1,
5087                          $drive->{iops_max_length} || 1,
5088                          $drive->{iops_rd_max_length} || 1,
5089                          $drive->{iops_wr_max_length} || 1,
5090                     );
5091
5092                 }
5093
5094                 return 1;
5095             }
5096
5097         } else { # cdrom
5098
5099             if ($drive->{file} eq 'none') {
5100                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5101                 if (drive_is_cloudinit($old_drive)) {
5102                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5103                 }
5104             } else {
5105                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5106
5107                 # force eject if locked
5108                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5109
5110                 if ($path) {
5111                     mon_cmd($vmid, "blockdev-change-medium",
5112                         id => "$opt", filename => "$path");
5113                 }
5114             }
5115
5116             return 1;
5117         }
5118     }
5119
5120     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5121     # hotplug new disks
5122     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5123     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5124 }
5125
5126 # called in locked context by incoming migration
5127 sub vm_migrate_get_nbd_disks {
5128     my ($storecfg, $conf, $replicated_volumes) = @_;
5129
5130     my $local_volumes = {};
5131     PVE::QemuConfig->foreach_volume($conf, sub {
5132         my ($ds, $drive) = @_;
5133
5134         return if drive_is_cdrom($drive);
5135
5136         my $volid = $drive->{file};
5137
5138         return if !$volid;
5139
5140         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5141
5142         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5143         return if $scfg->{shared};
5144
5145         # replicated disks re-use existing state via bitmap
5146         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5147         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5148     });
5149     return $local_volumes;
5150 }
5151
5152 # called in locked context by incoming migration
5153 sub vm_migrate_alloc_nbd_disks {
5154     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5155
5156     my $format = undef;
5157
5158     my $nbd = {};
5159     foreach my $opt (sort keys %$source_volumes) {
5160         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5161
5162         if ($use_existing) {
5163             $nbd->{$opt}->{drivestr} = print_drive($drive);
5164             $nbd->{$opt}->{volid} = $volid;
5165             $nbd->{$opt}->{replicated} = 1;
5166             next;
5167         }
5168
5169         # If a remote storage is specified and the format of the original
5170         # volume is not available there, fall back to the default format.
5171         # Otherwise use the same format as the original.
5172         if (!$storagemap->{identity}) {
5173             $storeid = map_storage($storagemap, $storeid);
5174             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5176             my $fileFormat = qemu_img_format($scfg, $volname);
5177             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5178         } else {
5179             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5180             $format = qemu_img_format($scfg, $volname);
5181         }
5182
5183         my $size = $drive->{size} / 1024;
5184         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5185         my $newdrive = $drive;
5186         $newdrive->{format} = $format;
5187         $newdrive->{file} = $newvolid;
5188         my $drivestr = print_drive($newdrive);
5189         $nbd->{$opt}->{drivestr} = $drivestr;
5190         $nbd->{$opt}->{volid} = $newvolid;
5191     }
5192
5193     return $nbd;
5194 }
5195
5196 # see vm_start_nolock for parameters, additionally:
5197 # migrate_opts:
5198 #   storagemap = parsed storage map for allocating NBD disks
5199 sub vm_start {
5200     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5201
5202     return PVE::QemuConfig->lock_config($vmid, sub {
5203         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5204
5205         die "you can't start a vm if it's a template\n"
5206             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5207
5208         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5209         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5210
5211         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5212
5213         if ($has_backup_lock && $running) {
5214             # a backup is currently running, attempt to start the guest in the
5215             # existing QEMU instance
5216             return vm_resume($vmid);
5217         }
5218
5219         PVE::QemuConfig->check_lock($conf)
5220             if !($params->{skiplock} || $has_suspended_lock);
5221
5222         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5223
5224         die "VM $vmid already running\n" if $running;
5225
5226         if (my $storagemap = $migrate_opts->{storagemap}) {
5227             my $replicated = $migrate_opts->{replicated_volumes};
5228             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5229             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5230
5231             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5232                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5233             }
5234         }
5235
5236         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5237     });
5238 }
5239
5240
5241 # params:
5242 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5243 #   skiplock => 0/1, skip checking for config lock
5244 #   skiptemplate => 0/1, skip checking whether VM is template
5245 #   forcemachine => to force Qemu machine (rollback/migration)
5246 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5247 #   timeout => in seconds
5248 #   paused => start VM in paused state (backup)
5249 #   resume => resume from hibernation
5250 #   pbs-backing => {
5251 #      sata0 => {
5252 #         repository
5253 #         snapshot
5254 #         keyfile
5255 #         archive
5256 #      },
5257 #      virtio2 => ...
5258 #   }
5259 # migrate_opts:
5260 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5261 #   migratedfrom => source node
5262 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5263 #   network => CIDR of migration network
5264 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5265 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5266 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5267 sub vm_start_nolock {
5268     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5269
5270     my $statefile = $params->{statefile};
5271     my $resume = $params->{resume};
5272
5273     my $migratedfrom = $migrate_opts->{migratedfrom};
5274     my $migration_type = $migrate_opts->{type};
5275
5276     my $res = {};
5277
5278     # clean up leftover reboot request files
5279     eval { clear_reboot_request($vmid); };
5280     warn $@ if $@;
5281
5282     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5283         vmconfig_apply_pending($vmid, $conf, $storecfg);
5284         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5285     }
5286
5287     # don't regenerate the ISO if the VM is started as part of a live migration
5288     # this way we can reuse the old ISO with the correct config
5289     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5290
5291     my $defaults = load_defaults();
5292
5293     # set environment variable useful inside network script
5294     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5295
5296     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5297
5298     my $forcemachine = $params->{forcemachine};
5299     my $forcecpu = $params->{forcecpu};
5300     if ($resume) {
5301         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5302         $forcemachine = $conf->{runningmachine};
5303         $forcecpu = $conf->{runningcpu};
5304         print "Resuming suspended VM\n";
5305     }
5306
5307     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5308         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5309
5310     my $migration_ip;
5311     my $get_migration_ip = sub {
5312         my ($nodename) = @_;
5313
5314         return $migration_ip if defined($migration_ip);
5315
5316         my $cidr = $migrate_opts->{network};
5317
5318         if (!defined($cidr)) {
5319             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5320             $cidr = $dc_conf->{migration}->{network};
5321         }
5322
5323         if (defined($cidr)) {
5324             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5325
5326             die "could not get IP: no address configured on local " .
5327                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5328
5329             die "could not get IP: multiple addresses configured on local " .
5330                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5331
5332             $migration_ip = @$ips[0];
5333         }
5334
5335         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5336             if !defined($migration_ip);
5337
5338         return $migration_ip;
5339     };
5340
5341     my $migrate_uri;
5342     if ($statefile) {
5343         if ($statefile eq 'tcp') {
5344             my $localip = "localhost";
5345             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5346             my $nodename = nodename();
5347
5348             if (!defined($migration_type)) {
5349                 if (defined($datacenterconf->{migration}->{type})) {
5350                     $migration_type = $datacenterconf->{migration}->{type};
5351                 } else {
5352                     $migration_type = 'secure';
5353                 }
5354             }
5355
5356             if ($migration_type eq 'insecure') {
5357                 $localip = $get_migration_ip->($nodename);
5358                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5359             }
5360
5361             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5362             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5363             $migrate_uri = "tcp:${localip}:${migrate_port}";
5364             push @$cmd, '-incoming', $migrate_uri;
5365             push @$cmd, '-S';
5366
5367         } elsif ($statefile eq 'unix') {
5368             # should be default for secure migrations as a ssh TCP forward
5369             # tunnel is not deterministic reliable ready and fails regurarly
5370             # to set up in time, so use UNIX socket forwards
5371             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5372             unlink $socket_addr;
5373
5374             $migrate_uri = "unix:$socket_addr";
5375
5376             push @$cmd, '-incoming', $migrate_uri;
5377             push @$cmd, '-S';
5378
5379         } elsif (-e $statefile) {
5380             push @$cmd, '-loadstate', $statefile;
5381         } else {
5382             my $statepath = PVE::Storage::path($storecfg, $statefile);
5383             push @$vollist, $statefile;
5384             push @$cmd, '-loadstate', $statepath;
5385         }
5386     } elsif ($params->{paused}) {
5387         push @$cmd, '-S';
5388     }
5389
5390     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5391
5392     my $pci_devices = {}; # host pci devices
5393     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5394         my $dev = $conf->{"hostpci$i"} or next;
5395         $pci_devices->{$i} = parse_hostpci($dev);
5396     }
5397
5398     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } values $pci_devices->%* ];
5399     # reserve all PCI IDs before actually doing anything with them
5400     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5401
5402     eval {
5403         for my $id (sort keys %$pci_devices) {
5404             my $d = $pci_devices->{$id};
5405             for my $dev ($d->{pciid}->@*) {
5406                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5407             }
5408         }
5409     };
5410     if (my $err = $@) {
5411         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5412         warn $@ if $@;
5413         die $err;
5414     }
5415
5416     PVE::Storage::activate_volumes($storecfg, $vollist);
5417
5418     eval {
5419         run_command(['/bin/systemctl', 'stop', "$vmid.scope"], outfunc => sub{}, errfunc => sub{});
5420     };
5421     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5422     # timeout should be more than enough here...
5423     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5424
5425     my $cpuunits = get_cpuunits($conf);
5426
5427     my %run_params = (
5428         timeout => $statefile ? undef : $start_timeout,
5429         umask => 0077,
5430         noerr => 1,
5431     );
5432
5433     # when migrating, prefix QEMU output so other side can pick up any
5434     # errors that might occur and show the user
5435     if ($migratedfrom) {
5436         $run_params{quiet} = 1;
5437         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5438     }
5439
5440     my %systemd_properties = (
5441         Slice => 'qemu.slice',
5442         KillMode => 'process',
5443         SendSIGKILL => 0,
5444         TimeoutStopUSec => ULONG_MAX, # infinity
5445     );
5446
5447     if (PVE::CGroup::cgroup_mode() == 2) {
5448         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5449         $systemd_properties{CPUWeight} = $cpuunits;
5450     } else {
5451         $systemd_properties{CPUShares} = $cpuunits;
5452     }
5453
5454     if (my $cpulimit = $conf->{cpulimit}) {
5455         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5456     }
5457     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5458
5459     my $run_qemu = sub {
5460         PVE::Tools::run_fork sub {
5461             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5462
5463             my $tpmpid;
5464             if (my $tpm = $conf->{tpmstate0}) {
5465                 # start the TPM emulator so QEMU can connect on start
5466                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5467             }
5468
5469             my $exitcode = run_command($cmd, %run_params);
5470             if ($exitcode) {
5471                 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5472                 kill 'TERM', $tpmpid if $tpmpid;
5473                 die "QEMU exited with code $exitcode\n";
5474             }
5475         };
5476     };
5477
5478     if ($conf->{hugepages}) {
5479
5480         my $code = sub {
5481             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5482             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5483
5484             PVE::QemuServer::Memory::hugepages_mount();
5485             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5486
5487             eval { $run_qemu->() };
5488             if (my $err = $@) {
5489                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5490                     if !$conf->{keephugepages};
5491                 die $err;
5492             }
5493
5494             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5495                 if !$conf->{keephugepages};
5496         };
5497         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5498
5499     } else {
5500         eval { $run_qemu->() };
5501     }
5502
5503     if (my $err = $@) {
5504         # deactivate volumes if start fails
5505         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5506         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5507
5508         die "start failed: $err";
5509     }
5510
5511     # re-reserve all PCI IDs now that we can know the actual VM PID
5512     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5513     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5514     warn $@ if $@;
5515
5516     print "migration listens on $migrate_uri\n" if $migrate_uri;
5517     $res->{migrate_uri} = $migrate_uri;
5518
5519     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5520         eval { mon_cmd($vmid, "cont"); };
5521         warn $@ if $@;
5522     }
5523
5524     #start nbd server for storage migration
5525     if (my $nbd = $migrate_opts->{nbd}) {
5526         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5527
5528         my $migrate_storage_uri;
5529         # nbd_protocol_version > 0 for unix socket support
5530         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5531             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5532             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5533             $migrate_storage_uri = "nbd:unix:$socket_path";
5534         } else {
5535             my $nodename = nodename();
5536             my $localip = $get_migration_ip->($nodename);
5537             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5538             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5539
5540             mon_cmd($vmid, "nbd-server-start", addr => {
5541                 type => 'inet',
5542                 data => {
5543                     host => "${localip}",
5544                     port => "${storage_migrate_port}",
5545                 },
5546             });
5547             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5548             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5549         }
5550
5551         $res->{migrate_storage_uri} = $migrate_storage_uri;
5552
5553         foreach my $opt (sort keys %$nbd) {
5554             my $drivestr = $nbd->{$opt}->{drivestr};
5555             my $volid = $nbd->{$opt}->{volid};
5556             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5557             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5558             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5559             print "re-using replicated volume: $opt - $volid\n"
5560                 if $nbd->{$opt}->{replicated};
5561
5562             $res->{drives}->{$opt} = $nbd->{$opt};
5563             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5564         }
5565     }
5566
5567     if ($migratedfrom) {
5568         eval {
5569             set_migration_caps($vmid);
5570         };
5571         warn $@ if $@;
5572
5573         if ($spice_port) {
5574             print "spice listens on port $spice_port\n";
5575             $res->{spice_port} = $spice_port;
5576             if ($migrate_opts->{spice_ticket}) {
5577                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5578                     $migrate_opts->{spice_ticket});
5579                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5580             }
5581         }
5582
5583     } else {
5584         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5585             if !$statefile && $conf->{balloon};
5586
5587         foreach my $opt (keys %$conf) {
5588             next if $opt !~  m/^net\d+$/;
5589             my $nicconf = parse_net($conf->{$opt});
5590             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5591         }
5592     }
5593
5594     mon_cmd($vmid, 'qom-set',
5595                 path => "machine/peripheral/balloon0",
5596                 property => "guest-stats-polling-interval",
5597                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5598
5599     if ($resume) {
5600         print "Resumed VM, removing state\n";
5601         if (my $vmstate = $conf->{vmstate}) {
5602             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5603             PVE::Storage::vdisk_free($storecfg, $vmstate);
5604         }
5605         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5606         PVE::QemuConfig->write_config($vmid, $conf);
5607     }
5608
5609     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5610
5611     return $res;
5612 }
5613
5614 sub vm_commandline {
5615     my ($storecfg, $vmid, $snapname) = @_;
5616
5617     my $conf = PVE::QemuConfig->load_config($vmid);
5618     my $forcemachine;
5619     my $forcecpu;
5620
5621     if ($snapname) {
5622         my $snapshot = $conf->{snapshots}->{$snapname};
5623         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5624
5625         # check for machine or CPU overrides in snapshot
5626         $forcemachine = $snapshot->{runningmachine};
5627         $forcecpu = $snapshot->{runningcpu};
5628
5629         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5630
5631         $conf = $snapshot;
5632     }
5633
5634     my $defaults = load_defaults();
5635
5636     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5637         $forcemachine, $forcecpu);
5638
5639     return PVE::Tools::cmd2string($cmd);
5640 }
5641
5642 sub vm_reset {
5643     my ($vmid, $skiplock) = @_;
5644
5645     PVE::QemuConfig->lock_config($vmid, sub {
5646
5647         my $conf = PVE::QemuConfig->load_config($vmid);
5648
5649         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5650
5651         mon_cmd($vmid, "system_reset");
5652     });
5653 }
5654
5655 sub get_vm_volumes {
5656     my ($conf) = @_;
5657
5658     my $vollist = [];
5659     foreach_volid($conf, sub {
5660         my ($volid, $attr) = @_;
5661
5662         return if $volid =~ m|^/|;
5663
5664         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5665         return if !$sid;
5666
5667         push @$vollist, $volid;
5668     });
5669
5670     return $vollist;
5671 }
5672
5673 sub vm_stop_cleanup {
5674     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5675
5676     eval {
5677
5678         if (!$keepActive) {
5679             my $vollist = get_vm_volumes($conf);
5680             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5681
5682             if (my $tpmdrive = $conf->{tpmstate0}) {
5683                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5684                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5685                 if ($storeid) {
5686                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5687                 }
5688             }
5689         }
5690
5691         foreach my $ext (qw(mon qmp pid vnc qga)) {
5692             unlink "/var/run/qemu-server/${vmid}.$ext";
5693         }
5694
5695         if ($conf->{ivshmem}) {
5696             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5697             # just delete it for now, VMs which have this already open do not
5698             # are affected, but new VMs will get a separated one. If this
5699             # becomes an issue we either add some sort of ref-counting or just
5700             # add a "don't delete on stop" flag to the ivshmem format.
5701             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5702         }
5703
5704         my $ids = [];
5705         foreach my $key (keys %$conf) {
5706             next if $key !~ m/^hostpci(\d+)$/;
5707             my $hostpciindex = $1;
5708             my $d = parse_hostpci($conf->{$key});
5709             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5710
5711             foreach my $pci (@{$d->{pciid}}) {
5712                 my $pciid = $pci->{id};
5713                 push @$ids, $pci->{id};
5714                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5715             }
5716         }
5717         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5718
5719         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5720     };
5721     warn $@ if $@; # avoid errors - just warn
5722 }
5723
5724 # call only in locked context
5725 sub _do_vm_stop {
5726     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5727
5728     my $pid = check_running($vmid, $nocheck);
5729     return if !$pid;
5730
5731     my $conf;
5732     if (!$nocheck) {
5733         $conf = PVE::QemuConfig->load_config($vmid);
5734         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5735         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5736             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5737             $timeout = $opts->{down} if $opts->{down};
5738         }
5739         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5740     }
5741
5742     eval {
5743         if ($shutdown) {
5744             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5745                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5746             } else {
5747                 mon_cmd($vmid, "system_powerdown");
5748             }
5749         } else {
5750             mon_cmd($vmid, "quit");
5751         }
5752     };
5753     my $err = $@;
5754
5755     if (!$err) {
5756         $timeout = 60 if !defined($timeout);
5757
5758         my $count = 0;
5759         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5760             $count++;
5761             sleep 1;
5762         }
5763
5764         if ($count >= $timeout) {
5765             if ($force) {
5766                 warn "VM still running - terminating now with SIGTERM\n";
5767                 kill 15, $pid;
5768             } else {
5769                 die "VM quit/powerdown failed - got timeout\n";
5770             }
5771         } else {
5772             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5773             return;
5774         }
5775     } else {
5776        if (!check_running($vmid, $nocheck)) {
5777             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5778             return;
5779        }
5780        if ($force) {
5781             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5782             kill 15, $pid;
5783         } else {
5784             die "VM quit/powerdown failed\n";
5785         }
5786     }
5787
5788     # wait again
5789     $timeout = 10;
5790
5791     my $count = 0;
5792     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5793         $count++;
5794         sleep 1;
5795     }
5796
5797     if ($count >= $timeout) {
5798         warn "VM still running - terminating now with SIGKILL\n";
5799         kill 9, $pid;
5800         sleep 1;
5801     }
5802
5803     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5804 }
5805
5806 # Note: use $nocheck to skip tests if VM configuration file exists.
5807 # We need that when migration VMs to other nodes (files already moved)
5808 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5809 sub vm_stop {
5810     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5811
5812     $force = 1 if !defined($force) && !$shutdown;
5813
5814     if ($migratedfrom){
5815         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5816         kill 15, $pid if $pid;
5817         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5818         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5819         return;
5820     }
5821
5822     PVE::QemuConfig->lock_config($vmid, sub {
5823         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5824    });
5825 }
5826
5827 sub vm_reboot {
5828     my ($vmid, $timeout) = @_;
5829
5830     PVE::QemuConfig->lock_config($vmid, sub {
5831         eval {
5832
5833             # only reboot if running, as qmeventd starts it again on a stop event
5834             return if !check_running($vmid);
5835
5836             create_reboot_request($vmid);
5837
5838             my $storecfg = PVE::Storage::config();
5839             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5840
5841         };
5842         if (my $err = $@) {
5843             # avoid that the next normal shutdown will be confused for a reboot
5844             clear_reboot_request($vmid);
5845             die $err;
5846         }
5847    });
5848 }
5849
5850 # note: if using the statestorage parameter, the caller has to check privileges
5851 sub vm_suspend {
5852     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5853
5854     my $conf;
5855     my $path;
5856     my $storecfg;
5857     my $vmstate;
5858
5859     PVE::QemuConfig->lock_config($vmid, sub {
5860
5861         $conf = PVE::QemuConfig->load_config($vmid);
5862
5863         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5864         PVE::QemuConfig->check_lock($conf)
5865             if !($skiplock || $is_backing_up);
5866
5867         die "cannot suspend to disk during backup\n"
5868             if $is_backing_up && $includestate;
5869
5870         if ($includestate) {
5871             $conf->{lock} = 'suspending';
5872             my $date = strftime("%Y-%m-%d", localtime(time()));
5873             $storecfg = PVE::Storage::config();
5874             if (!$statestorage) {
5875                 $statestorage = find_vmstate_storage($conf, $storecfg);
5876                 # check permissions for the storage
5877                 my $rpcenv = PVE::RPCEnvironment::get();
5878                 if ($rpcenv->{type} ne 'cli') {
5879                     my $authuser = $rpcenv->get_user();
5880                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5881                 }
5882             }
5883
5884
5885             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5886                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5887             $path = PVE::Storage::path($storecfg, $vmstate);
5888             PVE::QemuConfig->write_config($vmid, $conf);
5889         } else {
5890             mon_cmd($vmid, "stop");
5891         }
5892     });
5893
5894     if ($includestate) {
5895         # save vm state
5896         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5897
5898         eval {
5899             set_migration_caps($vmid, 1);
5900             mon_cmd($vmid, "savevm-start", statefile => $path);
5901             for(;;) {
5902                 my $state = mon_cmd($vmid, "query-savevm");
5903                 if (!$state->{status}) {
5904                     die "savevm not active\n";
5905                 } elsif ($state->{status} eq 'active') {
5906                     sleep(1);
5907                     next;
5908                 } elsif ($state->{status} eq 'completed') {
5909                     print "State saved, quitting\n";
5910                     last;
5911                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5912                     die "query-savevm failed with error '$state->{error}'\n"
5913                 } else {
5914                     die "query-savevm returned status '$state->{status}'\n";
5915                 }
5916             }
5917         };
5918         my $err = $@;
5919
5920         PVE::QemuConfig->lock_config($vmid, sub {
5921             $conf = PVE::QemuConfig->load_config($vmid);
5922             if ($err) {
5923                 # cleanup, but leave suspending lock, to indicate something went wrong
5924                 eval {
5925                     mon_cmd($vmid, "savevm-end");
5926                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5927                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5928                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5929                     PVE::QemuConfig->write_config($vmid, $conf);
5930                 };
5931                 warn $@ if $@;
5932                 die $err;
5933             }
5934
5935             die "lock changed unexpectedly\n"
5936                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5937
5938             mon_cmd($vmid, "quit");
5939             $conf->{lock} = 'suspended';
5940             PVE::QemuConfig->write_config($vmid, $conf);
5941         });
5942     }
5943 }
5944
5945 sub vm_resume {
5946     my ($vmid, $skiplock, $nocheck) = @_;
5947
5948     PVE::QemuConfig->lock_config($vmid, sub {
5949         my $res = mon_cmd($vmid, 'query-status');
5950         my $resume_cmd = 'cont';
5951         my $reset = 0;
5952
5953         if ($res->{status}) {
5954             return if $res->{status} eq 'running'; # job done, go home
5955             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5956             $reset = 1 if $res->{status} eq 'shutdown';
5957         }
5958
5959         if (!$nocheck) {
5960
5961             my $conf = PVE::QemuConfig->load_config($vmid);
5962
5963             PVE::QemuConfig->check_lock($conf)
5964                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5965         }
5966
5967         if ($reset) {
5968             # required if a VM shuts down during a backup and we get a resume
5969             # request before the backup finishes for example
5970             mon_cmd($vmid, "system_reset");
5971         }
5972         mon_cmd($vmid, $resume_cmd);
5973     });
5974 }
5975
5976 sub vm_sendkey {
5977     my ($vmid, $skiplock, $key) = @_;
5978
5979     PVE::QemuConfig->lock_config($vmid, sub {
5980
5981         my $conf = PVE::QemuConfig->load_config($vmid);
5982
5983         # there is no qmp command, so we use the human monitor command
5984         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5985         die $res if $res ne '';
5986     });
5987 }
5988
5989 # vzdump restore implementaion
5990
5991 sub tar_archive_read_firstfile {
5992     my $archive = shift;
5993
5994     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5995
5996     # try to detect archive type first
5997     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5998         die "unable to open file '$archive'\n";
5999     my $firstfile = <$fh>;
6000     kill 15, $pid;
6001     close $fh;
6002
6003     die "ERROR: archive contaions no data\n" if !$firstfile;
6004     chomp $firstfile;
6005
6006     return $firstfile;
6007 }
6008
6009 sub tar_restore_cleanup {
6010     my ($storecfg, $statfile) = @_;
6011
6012     print STDERR "starting cleanup\n";
6013
6014     if (my $fd = IO::File->new($statfile, "r")) {
6015         while (defined(my $line = <$fd>)) {
6016             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6017                 my $volid = $2;
6018                 eval {
6019                     if ($volid =~ m|^/|) {
6020                         unlink $volid || die 'unlink failed\n';
6021                     } else {
6022                         PVE::Storage::vdisk_free($storecfg, $volid);
6023                     }
6024                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6025                 };
6026                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6027             } else {
6028                 print STDERR "unable to parse line in statfile - $line";
6029             }
6030         }
6031         $fd->close();
6032     }
6033 }
6034
6035 sub restore_file_archive {
6036     my ($archive, $vmid, $user, $opts) = @_;
6037
6038     return restore_vma_archive($archive, $vmid, $user, $opts)
6039         if $archive eq '-';
6040
6041     my $info = PVE::Storage::archive_info($archive);
6042     my $format = $opts->{format} // $info->{format};
6043     my $comp = $info->{compression};
6044
6045     # try to detect archive format
6046     if ($format eq 'tar') {
6047         return restore_tar_archive($archive, $vmid, $user, $opts);
6048     } else {
6049         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6050     }
6051 }
6052
6053 # hepler to remove disks that will not be used after restore
6054 my $restore_cleanup_oldconf = sub {
6055     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6056
6057     PVE::QemuConfig->foreach_volume($oldconf, sub {
6058         my ($ds, $drive) = @_;
6059
6060         return if drive_is_cdrom($drive, 1);
6061
6062         my $volid = $drive->{file};
6063         return if !$volid || $volid =~ m|^/|;
6064
6065         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6066         return if !$path || !$owner || ($owner != $vmid);
6067
6068         # Note: only delete disk we want to restore
6069         # other volumes will become unused
6070         if ($virtdev_hash->{$ds}) {
6071             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6072             if (my $err = $@) {
6073                 warn $err;
6074             }
6075         }
6076     });
6077
6078     # delete vmstate files, after the restore we have no snapshots anymore
6079     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6080         my $snap = $oldconf->{snapshots}->{$snapname};
6081         if ($snap->{vmstate}) {
6082             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6083             if (my $err = $@) {
6084                 warn $err;
6085             }
6086         }
6087     }
6088 };
6089
6090 # Helper to parse vzdump backup device hints
6091 #
6092 # $rpcenv: Environment, used to ckeck storage permissions
6093 # $user: User ID, to check storage permissions
6094 # $storecfg: Storage configuration
6095 # $fh: the file handle for reading the configuration
6096 # $devinfo: should contain device sizes for all backu-up'ed devices
6097 # $options: backup options (pool, default storage)
6098 #
6099 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6100 my $parse_backup_hints = sub {
6101     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6102
6103     my $virtdev_hash = {};
6104
6105     while (defined(my $line = <$fh>)) {
6106         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6107             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6108             die "archive does not contain data for drive '$virtdev'\n"
6109                 if !$devinfo->{$devname};
6110
6111             if (defined($options->{storage})) {
6112                 $storeid = $options->{storage} || 'local';
6113             } elsif (!$storeid) {
6114                 $storeid = 'local';
6115             }
6116             $format = 'raw' if !$format;
6117             $devinfo->{$devname}->{devname} = $devname;
6118             $devinfo->{$devname}->{virtdev} = $virtdev;
6119             $devinfo->{$devname}->{format} = $format;
6120             $devinfo->{$devname}->{storeid} = $storeid;
6121
6122             # check permission on storage
6123             my $pool = $options->{pool}; # todo: do we need that?
6124             if ($user ne 'root@pam') {
6125                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6126             }
6127
6128             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6129         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6130             my $virtdev = $1;
6131             my $drive = parse_drive($virtdev, $2);
6132             if (drive_is_cloudinit($drive)) {
6133                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6134                 $storeid = $options->{storage} if defined ($options->{storage});
6135                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6136                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6137
6138                 $virtdev_hash->{$virtdev} = {
6139                     format => $format,
6140                     storeid => $storeid,
6141                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6142                     is_cloudinit => 1,
6143                 };
6144             }
6145         }
6146     }
6147
6148     return $virtdev_hash;
6149 };
6150
6151 # Helper to allocate and activate all volumes required for a restore
6152 #
6153 # $storecfg: Storage configuration
6154 # $virtdev_hash: as returned by parse_backup_hints()
6155 #
6156 # Returns: { $virtdev => $volid }
6157 my $restore_allocate_devices = sub {
6158     my ($storecfg, $virtdev_hash, $vmid) = @_;
6159
6160     my $map = {};
6161     foreach my $virtdev (sort keys %$virtdev_hash) {
6162         my $d = $virtdev_hash->{$virtdev};
6163         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6164         my $storeid = $d->{storeid};
6165         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6166
6167         # test if requested format is supported
6168         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6169         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6170         $d->{format} = $defFormat if !$supported;
6171
6172         my $name;
6173         if ($d->{is_cloudinit}) {
6174             $name = "vm-$vmid-cloudinit";
6175             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6176             if ($scfg->{path}) {
6177                 $name .= ".$d->{format}";
6178             }
6179         }
6180
6181         my $volid = PVE::Storage::vdisk_alloc(
6182             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6183
6184         print STDERR "new volume ID is '$volid'\n";
6185         $d->{volid} = $volid;
6186
6187         PVE::Storage::activate_volumes($storecfg, [$volid]);
6188
6189         $map->{$virtdev} = $volid;
6190     }
6191
6192     return $map;
6193 };
6194
6195 sub restore_update_config_line {
6196     my ($cookie, $map, $line, $unique) = @_;
6197
6198     return '' if $line =~ m/^\#qmdump\#/;
6199     return '' if $line =~ m/^\#vzdump\#/;
6200     return '' if $line =~ m/^lock:/;
6201     return '' if $line =~ m/^unused\d+:/;
6202     return '' if $line =~ m/^parent:/;
6203
6204     my $res = '';
6205
6206     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6207     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6208         # try to convert old 1.X settings
6209         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6210         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6211             my ($model, $macaddr) = split(/\=/, $devconfig);
6212             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6213             my $net = {
6214                 model => $model,
6215                 bridge => "vmbr$ind",
6216                 macaddr => $macaddr,
6217             };
6218             my $netstr = print_net($net);
6219
6220             $res .= "net$cookie->{netcount}: $netstr\n";
6221             $cookie->{netcount}++;
6222         }
6223     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6224         my ($id, $netstr) = ($1, $2);
6225         my $net = parse_net($netstr);
6226         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6227         $netstr = print_net($net);
6228         $res .= "$id: $netstr\n";
6229     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6230         my $virtdev = $1;
6231         my $value = $3;
6232         my $di = parse_drive($virtdev, $value);
6233         if (defined($di->{backup}) && !$di->{backup}) {
6234             $res .= "#$line";
6235         } elsif ($map->{$virtdev}) {
6236             delete $di->{format}; # format can change on restore
6237             $di->{file} = $map->{$virtdev};
6238             $value = print_drive($di);
6239             $res .= "$virtdev: $value\n";
6240         } else {
6241             $res .= $line;
6242         }
6243     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6244         my $vmgenid = $1;
6245         if ($vmgenid ne '0') {
6246             # always generate a new vmgenid if there was a valid one setup
6247             $vmgenid = generate_uuid();
6248         }
6249         $res .= "vmgenid: $vmgenid\n";
6250     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6251         my ($uuid, $uuid_str);
6252         UUID::generate($uuid);
6253         UUID::unparse($uuid, $uuid_str);
6254         my $smbios1 = parse_smbios1($2);
6255         $smbios1->{uuid} = $uuid_str;
6256         $res .= $1.print_smbios1($smbios1)."\n";
6257     } else {
6258         $res .= $line;
6259     }
6260
6261     return $res;
6262 }
6263
6264 my $restore_deactivate_volumes = sub {
6265     my ($storecfg, $devinfo) = @_;
6266
6267     my $vollist = [];
6268     foreach my $devname (keys %$devinfo) {
6269         my $volid = $devinfo->{$devname}->{volid};
6270         push @$vollist, $volid if $volid;
6271     }
6272
6273     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6274 };
6275
6276 my $restore_destroy_volumes = sub {
6277     my ($storecfg, $devinfo) = @_;
6278
6279     foreach my $devname (keys %$devinfo) {
6280         my $volid = $devinfo->{$devname}->{volid};
6281         next if !$volid;
6282         eval {
6283             if ($volid =~ m|^/|) {
6284                 unlink $volid || die 'unlink failed\n';
6285             } else {
6286                 PVE::Storage::vdisk_free($storecfg, $volid);
6287             }
6288             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6289         };
6290         print STDERR "unable to cleanup '$volid' - $@" if $@;
6291     }
6292 };
6293
6294 sub scan_volids {
6295     my ($cfg, $vmid) = @_;
6296
6297     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6298
6299     my $volid_hash = {};
6300     foreach my $storeid (keys %$info) {
6301         foreach my $item (@{$info->{$storeid}}) {
6302             next if !($item->{volid} && $item->{size});
6303             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6304             $volid_hash->{$item->{volid}} = $item;
6305         }
6306     }
6307
6308     return $volid_hash;
6309 }
6310
6311 sub update_disk_config {
6312     my ($vmid, $conf, $volid_hash) = @_;
6313
6314     my $changes;
6315     my $prefix = "VM $vmid";
6316
6317     # used and unused disks
6318     my $referenced = {};
6319
6320     # Note: it is allowed to define multiple storages with same path (alias), so
6321     # we need to check both 'volid' and real 'path' (two different volid can point
6322     # to the same path).
6323
6324     my $referencedpath = {};
6325
6326     # update size info
6327     PVE::QemuConfig->foreach_volume($conf, sub {
6328         my ($opt, $drive) = @_;
6329
6330         my $volid = $drive->{file};
6331         return if !$volid;
6332         my $volume = $volid_hash->{$volid};
6333
6334         # mark volid as "in-use" for next step
6335         $referenced->{$volid} = 1;
6336         if ($volume && (my $path = $volume->{path})) {
6337             $referencedpath->{$path} = 1;
6338         }
6339
6340         return if drive_is_cdrom($drive);
6341         return if !$volume;
6342
6343         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6344         if (defined($updated)) {
6345             $changes = 1;
6346             $conf->{$opt} = print_drive($updated);
6347             print "$prefix ($opt): $msg\n";
6348         }
6349     });
6350
6351     # remove 'unusedX' entry if volume is used
6352     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6353         my ($opt, $drive) = @_;
6354
6355         my $volid = $drive->{file};
6356         return if !$volid;
6357
6358         my $path;
6359         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6360         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6361             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6362             $changes = 1;
6363             delete $conf->{$opt};
6364         }
6365
6366         $referenced->{$volid} = 1;
6367         $referencedpath->{$path} = 1 if $path;
6368     });
6369
6370     foreach my $volid (sort keys %$volid_hash) {
6371         next if $volid =~ m/vm-$vmid-state-/;
6372         next if $referenced->{$volid};
6373         my $path = $volid_hash->{$volid}->{path};
6374         next if !$path; # just to be sure
6375         next if $referencedpath->{$path};
6376         $changes = 1;
6377         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6378         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6379         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6380     }
6381
6382     return $changes;
6383 }
6384
6385 sub rescan {
6386     my ($vmid, $nolock, $dryrun) = @_;
6387
6388     my $cfg = PVE::Storage::config();
6389
6390     print "rescan volumes...\n";
6391     my $volid_hash = scan_volids($cfg, $vmid);
6392
6393     my $updatefn =  sub {
6394         my ($vmid) = @_;
6395
6396         my $conf = PVE::QemuConfig->load_config($vmid);
6397
6398         PVE::QemuConfig->check_lock($conf);
6399
6400         my $vm_volids = {};
6401         foreach my $volid (keys %$volid_hash) {
6402             my $info = $volid_hash->{$volid};
6403             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6404         }
6405
6406         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6407
6408         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6409     };
6410
6411     if (defined($vmid)) {
6412         if ($nolock) {
6413             &$updatefn($vmid);
6414         } else {
6415             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6416         }
6417     } else {
6418         my $vmlist = config_list();
6419         foreach my $vmid (keys %$vmlist) {
6420             if ($nolock) {
6421                 &$updatefn($vmid);
6422             } else {
6423                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6424             }
6425         }
6426     }
6427 }
6428
6429 sub restore_proxmox_backup_archive {
6430     my ($archive, $vmid, $user, $options) = @_;
6431
6432     my $storecfg = PVE::Storage::config();
6433
6434     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6435     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6436
6437     my $fingerprint = $scfg->{fingerprint};
6438     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6439
6440     my $repo = PVE::PBSClient::get_repository($scfg);
6441
6442     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6443     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6444     local $ENV{PBS_PASSWORD} = $password;
6445     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6446
6447     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6448         PVE::Storage::parse_volname($storecfg, $archive);
6449
6450     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6451
6452     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6453
6454     my $tmpdir = "/var/tmp/vzdumptmp$$";
6455     rmtree $tmpdir;
6456     mkpath $tmpdir;
6457
6458     my $conffile = PVE::QemuConfig->config_file($vmid);
6459      # disable interrupts (always do cleanups)
6460     local $SIG{INT} =
6461         local $SIG{TERM} =
6462         local $SIG{QUIT} =
6463         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6464
6465     # Note: $oldconf is undef if VM does not exists
6466     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6467     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6468     my $new_conf_raw = '';
6469
6470     my $rpcenv = PVE::RPCEnvironment::get();
6471     my $devinfo = {};
6472
6473     eval {
6474         # enable interrupts
6475         local $SIG{INT} =
6476             local $SIG{TERM} =
6477             local $SIG{QUIT} =
6478             local $SIG{HUP} =
6479             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6480
6481         my $cfgfn = "$tmpdir/qemu-server.conf";
6482         my $firewall_config_fn = "$tmpdir/fw.conf";
6483         my $index_fn = "$tmpdir/index.json";
6484
6485         my $cmd = "restore";
6486
6487         my $param = [$pbs_backup_name, "index.json", $index_fn];
6488         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6489         my $index = PVE::Tools::file_get_contents($index_fn);
6490         $index = decode_json($index);
6491
6492         # print Dumper($index);
6493         foreach my $info (@{$index->{files}}) {
6494             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6495                 my $devname = $1;
6496                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6497                     $devinfo->{$devname}->{size} = $1;
6498                 } else {
6499                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6500                 }
6501             }
6502         }
6503
6504         my $is_qemu_server_backup = scalar(
6505             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6506         );
6507         if (!$is_qemu_server_backup) {
6508             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6509         }
6510         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6511
6512         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6513         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6514
6515         if ($has_firewall_config) {
6516             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6517             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6518
6519             my $pve_firewall_dir = '/etc/pve/firewall';
6520             mkdir $pve_firewall_dir; # make sure the dir exists
6521             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6522         }
6523
6524         my $fh = IO::File->new($cfgfn, "r") ||
6525             die "unable to read qemu-server.conf - $!\n";
6526
6527         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6528
6529         # fixme: rate limit?
6530
6531         # create empty/temp config
6532         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6533
6534         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6535
6536         # allocate volumes
6537         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6538
6539         foreach my $virtdev (sort keys %$virtdev_hash) {
6540             my $d = $virtdev_hash->{$virtdev};
6541             next if $d->{is_cloudinit}; # no need to restore cloudinit
6542
6543             # this fails if storage is unavailable
6544             my $volid = $d->{volid};
6545             my $path = PVE::Storage::path($storecfg, $volid);
6546
6547             # for live-restore we only want to preload the efidisk and TPM state
6548             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6549
6550             my $pbs_restore_cmd = [
6551                 '/usr/bin/pbs-restore',
6552                 '--repository', $repo,
6553                 $pbs_backup_name,
6554                 "$d->{devname}.img.fidx",
6555                 $path,
6556                 '--verbose',
6557                 ];
6558
6559             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6560             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6561
6562             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6563                 push @$pbs_restore_cmd, '--skip-zero';
6564             }
6565
6566             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6567             print "restore proxmox backup image: $dbg_cmdstring\n";
6568             run_command($pbs_restore_cmd);
6569         }
6570
6571         $fh->seek(0, 0) || die "seek failed - $!\n";
6572
6573         my $cookie = { netcount => 0 };
6574         while (defined(my $line = <$fh>)) {
6575             $new_conf_raw .= restore_update_config_line(
6576                 $cookie,
6577                 $map,
6578                 $line,
6579                 $options->{unique},
6580             );
6581         }
6582
6583         $fh->close();
6584     };
6585     my $err = $@;
6586
6587     if ($err || !$options->{live}) {
6588         $restore_deactivate_volumes->($storecfg, $devinfo);
6589     }
6590
6591     rmtree $tmpdir;
6592
6593     if ($err) {
6594         $restore_destroy_volumes->($storecfg, $devinfo);
6595         die $err;
6596     }
6597
6598     if ($options->{live}) {
6599         # keep lock during live-restore
6600         $new_conf_raw .= "\nlock: create";
6601     }
6602
6603     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6604
6605     PVE::Cluster::cfs_update(); # make sure we read new file
6606
6607     eval { rescan($vmid, 1); };
6608     warn $@ if $@;
6609
6610     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6611
6612     if ($options->{live}) {
6613         # enable interrupts
6614         local $SIG{INT} =
6615             local $SIG{TERM} =
6616             local $SIG{QUIT} =
6617             local $SIG{HUP} =
6618             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6619
6620         my $conf = PVE::QemuConfig->load_config($vmid);
6621         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6622
6623         # these special drives are already restored before start
6624         delete $devinfo->{'drive-efidisk0'};
6625         delete $devinfo->{'drive-tpmstate0-backup'};
6626         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6627
6628         PVE::QemuConfig->remove_lock($vmid, "create");
6629     }
6630 }
6631
6632 sub pbs_live_restore {
6633     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6634
6635     print "starting VM for live-restore\n";
6636     print "repository: '$repo', snapshot: '$snap'\n";
6637
6638     my $pbs_backing = {};
6639     for my $ds (keys %$restored_disks) {
6640         $ds =~ m/^drive-(.*)$/;
6641         my $confname = $1;
6642         $pbs_backing->{$confname} = {
6643             repository => $repo,
6644             snapshot => $snap,
6645             archive => "$ds.img.fidx",
6646         };
6647         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6648
6649         my $drive = parse_drive($confname, $conf->{$confname});
6650         print "restoring '$ds' to '$drive->{file}'\n";
6651     }
6652
6653     my $drives_streamed = 0;
6654     eval {
6655         # make sure HA doesn't interrupt our restore by stopping the VM
6656         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6657             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6658         }
6659
6660         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6661         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6662         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6663
6664         my $qmeventd_fd = register_qmeventd_handle($vmid);
6665
6666         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6667         # this will effectively collapse the backing image chain consisting of
6668         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6669         # removes itself once all backing images vanish with 'auto-remove=on')
6670         my $jobs = {};
6671         for my $ds (sort keys %$restored_disks) {
6672             my $job_id = "restore-$ds";
6673             mon_cmd($vmid, 'block-stream',
6674                 'job-id' => $job_id,
6675                 device => "$ds",
6676             );
6677             $jobs->{$job_id} = {};
6678         }
6679
6680         mon_cmd($vmid, 'cont');
6681         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6682
6683         print "restore-drive jobs finished successfully, removing all tracking block devices"
6684             ." to disconnect from Proxmox Backup Server\n";
6685
6686         for my $ds (sort keys %$restored_disks) {
6687             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6688         }
6689
6690         close($qmeventd_fd);
6691     };
6692
6693     my $err = $@;
6694
6695     if ($err) {
6696         warn "An error occured during live-restore: $err\n";
6697         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6698         die "live-restore failed\n";
6699     }
6700 }
6701
6702 sub restore_vma_archive {
6703     my ($archive, $vmid, $user, $opts, $comp) = @_;
6704
6705     my $readfrom = $archive;
6706
6707     my $cfg = PVE::Storage::config();
6708     my $commands = [];
6709     my $bwlimit = $opts->{bwlimit};
6710
6711     my $dbg_cmdstring = '';
6712     my $add_pipe = sub {
6713         my ($cmd) = @_;
6714         push @$commands, $cmd;
6715         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6716         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6717         $readfrom = '-';
6718     };
6719
6720     my $input = undef;
6721     if ($archive eq '-') {
6722         $input = '<&STDIN';
6723     } else {
6724         # If we use a backup from a PVE defined storage we also consider that
6725         # storage's rate limit:
6726         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6727         if (defined($volid)) {
6728             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6729             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6730             if ($readlimit) {
6731                 print STDERR "applying read rate limit: $readlimit\n";
6732                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6733                 $add_pipe->($cstream);
6734             }
6735         }
6736     }
6737
6738     if ($comp) {
6739         my $info = PVE::Storage::decompressor_info('vma', $comp);
6740         my $cmd = $info->{decompressor};
6741         push @$cmd, $readfrom;
6742         $add_pipe->($cmd);
6743     }
6744
6745     my $tmpdir = "/var/tmp/vzdumptmp$$";
6746     rmtree $tmpdir;
6747
6748     # disable interrupts (always do cleanups)
6749     local $SIG{INT} =
6750         local $SIG{TERM} =
6751         local $SIG{QUIT} =
6752         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6753
6754     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6755     POSIX::mkfifo($mapfifo, 0600);
6756     my $fifofh;
6757     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6758
6759     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6760
6761     my $oldtimeout;
6762     my $timeout = 5;
6763
6764     my $devinfo = {};
6765
6766     my $rpcenv = PVE::RPCEnvironment::get();
6767
6768     my $conffile = PVE::QemuConfig->config_file($vmid);
6769
6770     # Note: $oldconf is undef if VM does not exist
6771     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6772     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6773     my $new_conf_raw = '';
6774
6775     my %storage_limits;
6776
6777     my $print_devmap = sub {
6778         my $cfgfn = "$tmpdir/qemu-server.conf";
6779
6780         # we can read the config - that is already extracted
6781         my $fh = IO::File->new($cfgfn, "r") ||
6782             die "unable to read qemu-server.conf - $!\n";
6783
6784         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6785         if (-f $fwcfgfn) {
6786             my $pve_firewall_dir = '/etc/pve/firewall';
6787             mkdir $pve_firewall_dir; # make sure the dir exists
6788             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6789         }
6790
6791         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6792
6793         foreach my $info (values %{$virtdev_hash}) {
6794             my $storeid = $info->{storeid};
6795             next if defined($storage_limits{$storeid});
6796
6797             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6798             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6799             $storage_limits{$storeid} = $limit * 1024;
6800         }
6801
6802         foreach my $devname (keys %$devinfo) {
6803             die "found no device mapping information for device '$devname'\n"
6804                 if !$devinfo->{$devname}->{virtdev};
6805         }
6806
6807         # create empty/temp config
6808         if ($oldconf) {
6809             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6810             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6811         }
6812
6813         # allocate volumes
6814         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6815
6816         # print restore information to $fifofh
6817         foreach my $virtdev (sort keys %$virtdev_hash) {
6818             my $d = $virtdev_hash->{$virtdev};
6819             next if $d->{is_cloudinit}; # no need to restore cloudinit
6820
6821             my $storeid = $d->{storeid};
6822             my $volid = $d->{volid};
6823
6824             my $map_opts = '';
6825             if (my $limit = $storage_limits{$storeid}) {
6826                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6827             }
6828
6829             my $write_zeros = 1;
6830             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6831                 $write_zeros = 0;
6832             }
6833
6834             my $path = PVE::Storage::path($cfg, $volid);
6835
6836             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6837
6838             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6839         }
6840
6841         $fh->seek(0, 0) || die "seek failed - $!\n";
6842
6843         my $cookie = { netcount => 0 };
6844         while (defined(my $line = <$fh>)) {
6845             $new_conf_raw .= restore_update_config_line(
6846                 $cookie,
6847                 $map,
6848                 $line,
6849                 $opts->{unique},
6850             );
6851         }
6852
6853         $fh->close();
6854     };
6855
6856     eval {
6857         # enable interrupts
6858         local $SIG{INT} =
6859             local $SIG{TERM} =
6860             local $SIG{QUIT} =
6861             local $SIG{HUP} =
6862             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6863         local $SIG{ALRM} = sub { die "got timeout\n"; };
6864
6865         $oldtimeout = alarm($timeout);
6866
6867         my $parser = sub {
6868             my $line = shift;
6869
6870             print "$line\n";
6871
6872             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6873                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6874                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6875             } elsif ($line =~ m/^CTIME: /) {
6876                 # we correctly received the vma config, so we can disable
6877                 # the timeout now for disk allocation (set to 10 minutes, so
6878                 # that we always timeout if something goes wrong)
6879                 alarm(600);
6880                 &$print_devmap();
6881                 print $fifofh "done\n";
6882                 my $tmp = $oldtimeout || 0;
6883                 $oldtimeout = undef;
6884                 alarm($tmp);
6885                 close($fifofh);
6886                 $fifofh = undef;
6887             }
6888         };
6889
6890         print "restore vma archive: $dbg_cmdstring\n";
6891         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6892     };
6893     my $err = $@;
6894
6895     alarm($oldtimeout) if $oldtimeout;
6896
6897     $restore_deactivate_volumes->($cfg, $devinfo);
6898
6899     close($fifofh) if $fifofh;
6900     unlink $mapfifo;
6901     rmtree $tmpdir;
6902
6903     if ($err) {
6904         $restore_destroy_volumes->($cfg, $devinfo);
6905         die $err;
6906     }
6907
6908     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6909
6910     PVE::Cluster::cfs_update(); # make sure we read new file
6911
6912     eval { rescan($vmid, 1); };
6913     warn $@ if $@;
6914
6915     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6916 }
6917
6918 sub restore_tar_archive {
6919     my ($archive, $vmid, $user, $opts) = @_;
6920
6921     if ($archive ne '-') {
6922         my $firstfile = tar_archive_read_firstfile($archive);
6923         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6924             if $firstfile ne 'qemu-server.conf';
6925     }
6926
6927     my $storecfg = PVE::Storage::config();
6928
6929     # avoid zombie disks when restoring over an existing VM -> cleanup first
6930     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6931     # skiplock=1 because qmrestore has set the 'create' lock itself already
6932     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6933     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6934
6935     my $tocmd = "/usr/lib/qemu-server/qmextract";
6936
6937     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6938     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6939     $tocmd .= ' --prealloc' if $opts->{prealloc};
6940     $tocmd .= ' --info' if $opts->{info};
6941
6942     # tar option "xf" does not autodetect compression when read from STDIN,
6943     # so we pipe to zcat
6944     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6945         PVE::Tools::shellquote("--to-command=$tocmd");
6946
6947     my $tmpdir = "/var/tmp/vzdumptmp$$";
6948     mkpath $tmpdir;
6949
6950     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6951     local $ENV{VZDUMP_VMID} = $vmid;
6952     local $ENV{VZDUMP_USER} = $user;
6953
6954     my $conffile = PVE::QemuConfig->config_file($vmid);
6955     my $new_conf_raw = '';
6956
6957     # disable interrupts (always do cleanups)
6958     local $SIG{INT} =
6959         local $SIG{TERM} =
6960         local $SIG{QUIT} =
6961         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6962
6963     eval {
6964         # enable interrupts
6965         local $SIG{INT} =
6966             local $SIG{TERM} =
6967             local $SIG{QUIT} =
6968             local $SIG{HUP} =
6969             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6970
6971         if ($archive eq '-') {
6972             print "extracting archive from STDIN\n";
6973             run_command($cmd, input => "<&STDIN");
6974         } else {
6975             print "extracting archive '$archive'\n";
6976             run_command($cmd);
6977         }
6978
6979         return if $opts->{info};
6980
6981         # read new mapping
6982         my $map = {};
6983         my $statfile = "$tmpdir/qmrestore.stat";
6984         if (my $fd = IO::File->new($statfile, "r")) {
6985             while (defined (my $line = <$fd>)) {
6986                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6987                     $map->{$1} = $2 if $1;
6988                 } else {
6989                     print STDERR "unable to parse line in statfile - $line\n";
6990                 }
6991             }
6992             $fd->close();
6993         }
6994
6995         my $confsrc = "$tmpdir/qemu-server.conf";
6996
6997         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6998
6999         my $cookie = { netcount => 0 };
7000         while (defined (my $line = <$srcfd>)) {
7001             $new_conf_raw .= restore_update_config_line(
7002                 $cookie,
7003                 $map,
7004                 $line,
7005                 $opts->{unique},
7006             );
7007         }
7008
7009         $srcfd->close();
7010     };
7011     if (my $err = $@) {
7012         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7013         die $err;
7014     }
7015
7016     rmtree $tmpdir;
7017
7018     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7019
7020     PVE::Cluster::cfs_update(); # make sure we read new file
7021
7022     eval { rescan($vmid, 1); };
7023     warn $@ if $@;
7024 };
7025
7026 sub foreach_storage_used_by_vm {
7027     my ($conf, $func) = @_;
7028
7029     my $sidhash = {};
7030
7031     PVE::QemuConfig->foreach_volume($conf, sub {
7032         my ($ds, $drive) = @_;
7033         return if drive_is_cdrom($drive);
7034
7035         my $volid = $drive->{file};
7036
7037         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7038         $sidhash->{$sid} = $sid if $sid;
7039     });
7040
7041     foreach my $sid (sort keys %$sidhash) {
7042         &$func($sid);
7043     }
7044 }
7045
7046 my $qemu_snap_storage = {
7047     rbd => 1,
7048 };
7049 sub do_snapshots_with_qemu {
7050     my ($storecfg, $volid, $deviceid) = @_;
7051
7052     return if $deviceid =~ m/tpmstate0/;
7053
7054     my $storage_name = PVE::Storage::parse_volume_id($volid);
7055     my $scfg = $storecfg->{ids}->{$storage_name};
7056     die "could not find storage '$storage_name'\n" if !defined($scfg);
7057
7058     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7059         return 1;
7060     }
7061
7062     if ($volid =~ m/\.(qcow2|qed)$/){
7063         return 1;
7064     }
7065
7066     return;
7067 }
7068
7069 sub qga_check_running {
7070     my ($vmid, $nowarn) = @_;
7071
7072     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7073     if ($@) {
7074         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7075         return 0;
7076     }
7077     return 1;
7078 }
7079
7080 sub template_create {
7081     my ($vmid, $conf, $disk) = @_;
7082
7083     my $storecfg = PVE::Storage::config();
7084
7085     PVE::QemuConfig->foreach_volume($conf, sub {
7086         my ($ds, $drive) = @_;
7087
7088         return if drive_is_cdrom($drive);
7089         return if $disk && $ds ne $disk;
7090
7091         my $volid = $drive->{file};
7092         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7093
7094         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7095         $drive->{file} = $voliddst;
7096         $conf->{$ds} = print_drive($drive);
7097         PVE::QemuConfig->write_config($vmid, $conf);
7098     });
7099 }
7100
7101 sub convert_iscsi_path {
7102     my ($path) = @_;
7103
7104     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7105         my $portal = $1;
7106         my $target = $2;
7107         my $lun = $3;
7108
7109         my $initiator_name = get_initiator_name();
7110
7111         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7112                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7113     }
7114
7115     die "cannot convert iscsi path '$path', unkown format\n";
7116 }
7117
7118 sub qemu_img_convert {
7119     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7120
7121     my $storecfg = PVE::Storage::config();
7122     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7123     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7124
7125     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7126
7127     my $cachemode;
7128     my $src_path;
7129     my $src_is_iscsi = 0;
7130     my $src_format;
7131
7132     if ($src_storeid) {
7133         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7134         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7135         $src_format = qemu_img_format($src_scfg, $src_volname);
7136         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7137         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7138         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7139     } elsif (-f $src_volid) {
7140         $src_path = $src_volid;
7141         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7142             $src_format = $1;
7143         }
7144     }
7145
7146     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7147
7148     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7149     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7150     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7151     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7152
7153     my $cmd = [];
7154     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7155     push @$cmd, '-l', "snapshot.name=$snapname"
7156         if $snapname && $src_format && $src_format eq "qcow2";
7157     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7158     push @$cmd, '-T', $cachemode if defined($cachemode);
7159
7160     if ($src_is_iscsi) {
7161         push @$cmd, '--image-opts';
7162         $src_path = convert_iscsi_path($src_path);
7163     } elsif ($src_format) {
7164         push @$cmd, '-f', $src_format;
7165     }
7166
7167     if ($dst_is_iscsi) {
7168         push @$cmd, '--target-image-opts';
7169         $dst_path = convert_iscsi_path($dst_path);
7170     } else {
7171         push @$cmd, '-O', $dst_format;
7172     }
7173
7174     push @$cmd, $src_path;
7175
7176     if (!$dst_is_iscsi && $is_zero_initialized) {
7177         push @$cmd, "zeroinit:$dst_path";
7178     } else {
7179         push @$cmd, $dst_path;
7180     }
7181
7182     my $parser = sub {
7183         my $line = shift;
7184         if($line =~ m/\((\S+)\/100\%\)/){
7185             my $percent = $1;
7186             my $transferred = int($size * $percent / 100);
7187             my $total_h = render_bytes($size, 1);
7188             my $transferred_h = render_bytes($transferred, 1);
7189
7190             print "transferred $transferred_h of $total_h ($percent%)\n";
7191         }
7192
7193     };
7194
7195     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7196     my $err = $@;
7197     die "copy failed: $err" if $err;
7198 }
7199
7200 sub qemu_img_format {
7201     my ($scfg, $volname) = @_;
7202
7203     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7204         return $1;
7205     } else {
7206         return "raw";
7207     }
7208 }
7209
7210 sub qemu_drive_mirror {
7211     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7212
7213     $jobs = {} if !$jobs;
7214
7215     my $qemu_target;
7216     my $format;
7217     $jobs->{"drive-$drive"} = {};
7218
7219     if ($dst_volid =~ /^nbd:/) {
7220         $qemu_target = $dst_volid;
7221         $format = "nbd";
7222     } else {
7223         my $storecfg = PVE::Storage::config();
7224         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7225
7226         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7227
7228         $format = qemu_img_format($dst_scfg, $dst_volname);
7229
7230         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7231
7232         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7233     }
7234
7235     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7236     $opts->{format} = $format if $format;
7237
7238     if (defined($src_bitmap)) {
7239         $opts->{sync} = 'incremental';
7240         $opts->{bitmap} = $src_bitmap;
7241         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7242     }
7243
7244     if (defined($bwlimit)) {
7245         $opts->{speed} = $bwlimit * 1024;
7246         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7247     } else {
7248         print "drive mirror is starting for drive-$drive\n";
7249     }
7250
7251     # if a job already runs for this device we get an error, catch it for cleanup
7252     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7253     if (my $err = $@) {
7254         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7255         warn "$@\n" if $@;
7256         die "mirroring error: $err\n";
7257     }
7258
7259     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7260 }
7261
7262 # $completion can be either
7263 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7264 # 'cancel': wait until all jobs are ready, block-job-cancel them
7265 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7266 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7267 sub qemu_drive_mirror_monitor {
7268     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7269
7270     $completion //= 'complete';
7271     $op //= "mirror";
7272
7273     eval {
7274         my $err_complete = 0;
7275
7276         my $starttime = time ();
7277         while (1) {
7278             die "block job ('$op') timed out\n" if $err_complete > 300;
7279
7280             my $stats = mon_cmd($vmid, "query-block-jobs");
7281             my $ctime = time();
7282
7283             my $running_jobs = {};
7284             for my $stat (@$stats) {
7285                 next if $stat->{type} ne $op;
7286                 $running_jobs->{$stat->{device}} = $stat;
7287             }
7288
7289             my $readycounter = 0;
7290
7291             for my $job_id (sort keys %$jobs) {
7292                 my $job = $running_jobs->{$job_id};
7293
7294                 my $vanished = !defined($job);
7295                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7296                 if($complete || ($vanished && $completion eq 'auto')) {
7297                     print "$job_id: $op-job finished\n";
7298                     delete $jobs->{$job_id};
7299                     next;
7300                 }
7301
7302                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7303
7304                 my $busy = $job->{busy};
7305                 my $ready = $job->{ready};
7306                 if (my $total = $job->{len}) {
7307                     my $transferred = $job->{offset} || 0;
7308                     my $remaining = $total - $transferred;
7309                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7310
7311                     my $duration = $ctime - $starttime;
7312                     my $total_h = render_bytes($total, 1);
7313                     my $transferred_h = render_bytes($transferred, 1);
7314
7315                     my $status = sprintf(
7316                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7317                         render_duration($duration),
7318                     );
7319
7320                     if ($ready) {
7321                         if ($busy) {
7322                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7323                         } else {
7324                             $status .= ", ready";
7325                         }
7326                     }
7327                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7328                     $jobs->{$job_id}->{ready} = $ready;
7329                 }
7330
7331                 $readycounter++ if $job->{ready};
7332             }
7333
7334             last if scalar(keys %$jobs) == 0;
7335
7336             if ($readycounter == scalar(keys %$jobs)) {
7337                 print "all '$op' jobs are ready\n";
7338
7339                 # do the complete later (or has already been done)
7340                 last if $completion eq 'skip' || $completion eq 'auto';
7341
7342                 if ($vmiddst && $vmiddst != $vmid) {
7343                     my $agent_running = $qga && qga_check_running($vmid);
7344                     if ($agent_running) {
7345                         print "freeze filesystem\n";
7346                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7347                     } else {
7348                         print "suspend vm\n";
7349                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7350                     }
7351
7352                     # if we clone a disk for a new target vm, we don't switch the disk
7353                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7354
7355                     if ($agent_running) {
7356                         print "unfreeze filesystem\n";
7357                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7358                     } else {
7359                         print "resume vm\n";
7360                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7361                     }
7362
7363                     last;
7364                 } else {
7365
7366                     for my $job_id (sort keys %$jobs) {
7367                         # try to switch the disk if source and destination are on the same guest
7368                         print "$job_id: Completing block job_id...\n";
7369
7370                         my $op;
7371                         if ($completion eq 'complete') {
7372                             $op = 'block-job-complete';
7373                         } elsif ($completion eq 'cancel') {
7374                             $op = 'block-job-cancel';
7375                         } else {
7376                             die "invalid completion value: $completion\n";
7377                         }
7378                         eval { mon_cmd($vmid, $op, device => $job_id) };
7379                         if ($@ =~ m/cannot be completed/) {
7380                             print "$job_id: block job cannot be completed, trying again.\n";
7381                             $err_complete++;
7382                         }else {
7383                             print "$job_id: Completed successfully.\n";
7384                             $jobs->{$job_id}->{complete} = 1;
7385                         }
7386                     }
7387                 }
7388             }
7389             sleep 1;
7390         }
7391     };
7392     my $err = $@;
7393
7394     if ($err) {
7395         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7396         die "block job ($op) error: $err";
7397     }
7398 }
7399
7400 sub qemu_blockjobs_cancel {
7401     my ($vmid, $jobs) = @_;
7402
7403     foreach my $job (keys %$jobs) {
7404         print "$job: Cancelling block job\n";
7405         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7406         $jobs->{$job}->{cancel} = 1;
7407     }
7408
7409     while (1) {
7410         my $stats = mon_cmd($vmid, "query-block-jobs");
7411
7412         my $running_jobs = {};
7413         foreach my $stat (@$stats) {
7414             $running_jobs->{$stat->{device}} = $stat;
7415         }
7416
7417         foreach my $job (keys %$jobs) {
7418
7419             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7420                 print "$job: Done.\n";
7421                 delete $jobs->{$job};
7422             }
7423         }
7424
7425         last if scalar(keys %$jobs) == 0;
7426
7427         sleep 1;
7428     }
7429 }
7430
7431 sub clone_disk {
7432     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7433         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7434
7435     my $newvolid;
7436
7437     if (!$full) {
7438         print "create linked clone of drive $drivename ($drive->{file})\n";
7439         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7440         push @$newvollist, $newvolid;
7441     } else {
7442
7443         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7444         $storeid = $storage if $storage;
7445
7446         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7447
7448         print "create full clone of drive $drivename ($drive->{file})\n";
7449         my $name = undef;
7450         my $size = undef;
7451         if (drive_is_cloudinit($drive)) {
7452             $name = "vm-$newvmid-cloudinit";
7453             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7454             if ($scfg->{path}) {
7455                 $name .= ".$dst_format";
7456             }
7457             $snapname = undef;
7458             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7459         } elsif ($drivename eq 'efidisk0') {
7460             $size = get_efivars_size($conf);
7461         } elsif ($drivename eq 'tpmstate0') {
7462             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7463         } else {
7464             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7465         }
7466         $newvolid = PVE::Storage::vdisk_alloc(
7467             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7468         );
7469         push @$newvollist, $newvolid;
7470
7471         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7472
7473         if (drive_is_cloudinit($drive)) {
7474             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7475             # if this is the case, we have to complete any block-jobs still there from
7476             # previous drive-mirrors
7477             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7478                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7479             }
7480             goto no_data_clone;
7481         }
7482
7483         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7484         if (!$running || $snapname) {
7485             # TODO: handle bwlimits
7486             if ($drivename eq 'efidisk0') {
7487                 # the relevant data on the efidisk may be smaller than the source
7488                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7489                 # that is given by the OVMF_VARS.fd
7490                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7491                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7492
7493                 # better for Ceph if block size is not too small, see bug #3324
7494                 my $bs = 1024*1024;
7495
7496                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7497                     "if=$src_path", "of=$dst_path"]);
7498             } else {
7499                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7500             }
7501         } else {
7502
7503             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7504
7505             my $kvmver = get_running_qemu_version ($vmid);
7506             if (!min_version($kvmver, 2, 7)) {
7507                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7508                     if $drive->{iothread};
7509             }
7510
7511             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7512                 $completion, $qga, $bwlimit);
7513         }
7514     }
7515
7516 no_data_clone:
7517     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7518
7519     my $disk = $drive;
7520     $disk->{format} = undef;
7521     $disk->{file} = $newvolid;
7522     $disk->{size} = $size if defined($size);
7523
7524     return $disk;
7525 }
7526
7527 sub get_running_qemu_version {
7528     my ($vmid) = @_;
7529     my $res = mon_cmd($vmid, "query-version");
7530     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7531 }
7532
7533 sub qemu_use_old_bios_files {
7534     my ($machine_type) = @_;
7535
7536     return if !$machine_type;
7537
7538     my $use_old_bios_files = undef;
7539
7540     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7541         $machine_type = $1;
7542         $use_old_bios_files = 1;
7543     } else {
7544         my $version = extract_version($machine_type, kvm_user_version());
7545         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7546         # load new efi bios files on migration. So this hack is required to allow
7547         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7548         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7549         $use_old_bios_files = !min_version($version, 2, 4);
7550     }
7551
7552     return ($use_old_bios_files, $machine_type);
7553 }
7554
7555 sub get_efivars_size {
7556     my ($conf) = @_;
7557     my $arch = get_vm_arch($conf);
7558     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7559     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk);
7560     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7561     return -s $ovmf_vars;
7562 }
7563
7564 sub update_efidisk_size {
7565     my ($conf) = @_;
7566
7567     return if !defined($conf->{efidisk0});
7568
7569     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7570     $disk->{size} = get_efivars_size($conf);
7571     $conf->{efidisk0} = print_drive($disk);
7572
7573     return;
7574 }
7575
7576 sub update_tpmstate_size {
7577     my ($conf) = @_;
7578
7579     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7580     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7581     $conf->{tpmstate0} = print_drive($disk);
7582 }
7583
7584 sub create_efidisk($$$$$$) {
7585     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk) = @_;
7586
7587     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk);
7588     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7589
7590     my $vars_size_b = -s $ovmf_vars;
7591     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7592     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7593     PVE::Storage::activate_volumes($storecfg, [$volid]);
7594
7595     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7596     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7597
7598     return ($volid, $size/1024);
7599 }
7600
7601 sub vm_iothreads_list {
7602     my ($vmid) = @_;
7603
7604     my $res = mon_cmd($vmid, 'query-iothreads');
7605
7606     my $iothreads = {};
7607     foreach my $iothread (@$res) {
7608         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7609     }
7610
7611     return $iothreads;
7612 }
7613
7614 sub scsihw_infos {
7615     my ($conf, $drive) = @_;
7616
7617     my $maxdev = 0;
7618
7619     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7620         $maxdev = 7;
7621     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7622         $maxdev = 1;
7623     } else {
7624         $maxdev = 256;
7625     }
7626
7627     my $controller = int($drive->{index} / $maxdev);
7628     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7629         ? "virtioscsi"
7630         : "scsihw";
7631
7632     return ($maxdev, $controller, $controller_prefix);
7633 }
7634
7635 sub windows_version {
7636     my ($ostype) = @_;
7637
7638     return 0 if !$ostype;
7639
7640     my $winversion = 0;
7641
7642     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7643         $winversion = 5;
7644     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7645         $winversion = 6;
7646     } elsif ($ostype =~ m/^win(\d+)$/) {
7647         $winversion = $1;
7648     }
7649
7650     return $winversion;
7651 }
7652
7653 sub resolve_dst_disk_format {
7654         my ($storecfg, $storeid, $src_volname, $format) = @_;
7655         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7656
7657         if (!$format) {
7658             # if no target format is specified, use the source disk format as hint
7659             if ($src_volname) {
7660                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7661                 $format = qemu_img_format($scfg, $src_volname);
7662             } else {
7663                 return $defFormat;
7664             }
7665         }
7666
7667         # test if requested format is supported - else use default
7668         my $supported = grep { $_ eq $format } @$validFormats;
7669         $format = $defFormat if !$supported;
7670         return $format;
7671 }
7672
7673 # NOTE: if this logic changes, please update docs & possibly gui logic
7674 sub find_vmstate_storage {
7675     my ($conf, $storecfg) = @_;
7676
7677     # first, return storage from conf if set
7678     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7679
7680     my ($target, $shared, $local);
7681
7682     foreach_storage_used_by_vm($conf, sub {
7683         my ($sid) = @_;
7684         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7685         my $dst = $scfg->{shared} ? \$shared : \$local;
7686         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7687     });
7688
7689     # second, use shared storage where VM has at least one disk
7690     # third, use local storage where VM has at least one disk
7691     # fall back to local storage
7692     $target = $shared // $local // 'local';
7693
7694     return $target;
7695 }
7696
7697 sub generate_uuid {
7698     my ($uuid, $uuid_str);
7699     UUID::generate($uuid);
7700     UUID::unparse($uuid, $uuid_str);
7701     return $uuid_str;
7702 }
7703
7704 sub generate_smbios1_uuid {
7705     return "uuid=".generate_uuid();
7706 }
7707
7708 sub nbd_stop {
7709     my ($vmid) = @_;
7710
7711     mon_cmd($vmid, 'nbd-server-stop');
7712 }
7713
7714 sub create_reboot_request {
7715     my ($vmid) = @_;
7716     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7717         or die "failed to create reboot trigger file: $!\n";
7718     close($fh);
7719 }
7720
7721 sub clear_reboot_request {
7722     my ($vmid) = @_;
7723     my $path = "/run/qemu-server/$vmid.reboot";
7724     my $res = 0;
7725
7726     $res = unlink($path);
7727     die "could not remove reboot request for $vmid: $!"
7728         if !$res && $! != POSIX::ENOENT;
7729
7730     return $res;
7731 }
7732
7733 sub bootorder_from_legacy {
7734     my ($conf, $bootcfg) = @_;
7735
7736     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7737     my $bootindex_hash = {};
7738     my $i = 1;
7739     foreach my $o (split(//, $boot)) {
7740         $bootindex_hash->{$o} = $i*100;
7741         $i++;
7742     }
7743
7744     my $bootorder = {};
7745
7746     PVE::QemuConfig->foreach_volume($conf, sub {
7747         my ($ds, $drive) = @_;
7748
7749         if (drive_is_cdrom ($drive, 1)) {
7750             if ($bootindex_hash->{d}) {
7751                 $bootorder->{$ds} = $bootindex_hash->{d};
7752                 $bootindex_hash->{d} += 1;
7753             }
7754         } elsif ($bootindex_hash->{c}) {
7755             $bootorder->{$ds} = $bootindex_hash->{c}
7756                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7757             $bootindex_hash->{c} += 1;
7758         }
7759     });
7760
7761     if ($bootindex_hash->{n}) {
7762         for (my $i = 0; $i < $MAX_NETS; $i++) {
7763             my $netname = "net$i";
7764             next if !$conf->{$netname};
7765             $bootorder->{$netname} = $bootindex_hash->{n};
7766             $bootindex_hash->{n} += 1;
7767         }
7768     }
7769
7770     return $bootorder;
7771 }
7772
7773 # Generate default device list for 'boot: order=' property. Matches legacy
7774 # default boot order, but with explicit device names. This is important, since
7775 # the fallback for when neither 'order' nor the old format is specified relies
7776 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7777 sub get_default_bootdevices {
7778     my ($conf) = @_;
7779
7780     my @ret = ();
7781
7782     # harddisk
7783     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7784     push @ret, $first if $first;
7785
7786     # cdrom
7787     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7788     push @ret, $first if $first;
7789
7790     # network
7791     for (my $i = 0; $i < $MAX_NETS; $i++) {
7792         my $netname = "net$i";
7793         next if !$conf->{$netname};
7794         push @ret, $netname;
7795         last;
7796     }
7797
7798     return \@ret;
7799 }
7800
7801 sub device_bootorder {
7802     my ($conf) = @_;
7803
7804     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7805
7806     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7807
7808     my $bootorder = {};
7809     if (!defined($boot) || $boot->{legacy}) {
7810         $bootorder = bootorder_from_legacy($conf, $boot);
7811     } elsif ($boot->{order}) {
7812         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7813         for my $dev (PVE::Tools::split_list($boot->{order})) {
7814             $bootorder->{$dev} = $i++;
7815         }
7816     }
7817
7818     return $bootorder;
7819 }
7820
7821 sub register_qmeventd_handle {
7822     my ($vmid) = @_;
7823
7824     my $fh;
7825     my $peer = "/var/run/qmeventd.sock";
7826     my $count = 0;
7827
7828     for (;;) {
7829         $count++;
7830         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7831         last if $fh;
7832         if ($! != EINTR && $! != EAGAIN) {
7833             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7834         }
7835         if ($count > 4) {
7836             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7837               . "after $count retries\n";
7838         }
7839         usleep(25000);
7840     }
7841
7842     # send handshake to mark VM as backing up
7843     print $fh to_json({vzdump => {vmid => "$vmid"}});
7844
7845     # return handle to be closed later when inhibit is no longer required
7846     return $fh;
7847 }
7848
7849 # bash completion helper
7850
7851 sub complete_backup_archives {
7852     my ($cmdname, $pname, $cvalue) = @_;
7853
7854     my $cfg = PVE::Storage::config();
7855
7856     my $storeid;
7857
7858     if ($cvalue =~ m/^([^:]+):/) {
7859         $storeid = $1;
7860     }
7861
7862     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7863
7864     my $res = [];
7865     foreach my $id (keys %$data) {
7866         foreach my $item (@{$data->{$id}}) {
7867             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7868             push @$res, $item->{volid} if defined($item->{volid});
7869         }
7870     }
7871
7872     return $res;
7873 }
7874
7875 my $complete_vmid_full = sub {
7876     my ($running) = @_;
7877
7878     my $idlist = vmstatus();
7879
7880     my $res = [];
7881
7882     foreach my $id (keys %$idlist) {
7883         my $d = $idlist->{$id};
7884         if (defined($running)) {
7885             next if $d->{template};
7886             next if $running && $d->{status} ne 'running';
7887             next if !$running && $d->{status} eq 'running';
7888         }
7889         push @$res, $id;
7890
7891     }
7892     return $res;
7893 };
7894
7895 sub complete_vmid {
7896     return &$complete_vmid_full();
7897 }
7898
7899 sub complete_vmid_stopped {
7900     return &$complete_vmid_full(0);
7901 }
7902
7903 sub complete_vmid_running {
7904     return &$complete_vmid_full(1);
7905 }
7906
7907 sub complete_storage {
7908
7909     my $cfg = PVE::Storage::config();
7910     my $ids = $cfg->{ids};
7911
7912     my $res = [];
7913     foreach my $sid (keys %$ids) {
7914         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7915         next if !$ids->{$sid}->{content}->{images};
7916         push @$res, $sid;
7917     }
7918
7919     return $res;
7920 }
7921
7922 sub complete_migration_storage {
7923     my ($cmd, $param, $current_value, $all_args) = @_;
7924
7925     my $targetnode = @$all_args[1];
7926
7927     my $cfg = PVE::Storage::config();
7928     my $ids = $cfg->{ids};
7929
7930     my $res = [];
7931     foreach my $sid (keys %$ids) {
7932         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7933         next if !$ids->{$sid}->{content}->{images};
7934         push @$res, $sid;
7935     }
7936
7937     return $res;
7938 }
7939
7940 sub vm_is_paused {
7941     my ($vmid) = @_;
7942     my $qmpstatus = eval {
7943         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7944         mon_cmd($vmid, "query-status");
7945     };
7946     warn "$@\n" if $@;
7947     return $qmpstatus && $qmpstatus->{status} eq "paused";
7948 }
7949
7950 sub check_volume_storage_type {
7951     my ($storecfg, $vol) = @_;
7952
7953     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7954     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7955     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7956
7957     die "storage '$storeid' does not support content-type '$vtype'\n"
7958         if !$scfg->{content}->{$vtype};
7959
7960     return 1;
7961 }
7962
7963 1;