PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday usleep);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::Format qw(render_duration render_bytes);
  34 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  35 use PVE::INotify;
  36 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  37 use PVE::ProcFSTools;
  38 use PVE::PBSClient;
  39 use PVE::RPCEnvironment;
  40 use PVE::Storage;
  41 use PVE::SysFSTools;
  42 use PVE::Systemd;
  43 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  44
  45 use PVE::QMPClient;
  46 use PVE::QemuConfig;
  47 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  48 use PVE::QemuServer::Cloudinit;
  49 use PVE::QemuServer::CGroup;
  50 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  51 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom drive_is_read_only parse_drive print_drive);
  52 use PVE::QemuServer::Machine;
  53 use PVE::QemuServer::Memory;
  54 use PVE::QemuServer::Monitor qw(mon_cmd);
  55 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  56 use PVE::QemuServer::USB qw(parse_usb_device);
  57
  58 my $have_sdn;
  59 eval {
  60     require PVE::Network::SDN::Zones;
  61     $have_sdn = 1;
  62 };
  63
  64 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  65 my $OVMF = {
  66     x86_64 => {
  67         '4m' => [
  68             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  69             "$EDK2_FW_BASE/OVMF_VARS_4M.fd",
  70         ],
  71         '4m-ms' => [
  72             "$EDK2_FW_BASE/OVMF_CODE_4M.secboot.fd",
  73             "$EDK2_FW_BASE/OVMF_VARS_4M.ms.fd",
  74         ],
  75         default => [
  76             "$EDK2_FW_BASE/OVMF_CODE.fd",
  77             "$EDK2_FW_BASE/OVMF_VARS.fd",
  78         ],
  79     },
  80     aarch64 => {
  81         default => [
  82             "$EDK2_FW_BASE/AAVMF_CODE.fd",
  83             "$EDK2_FW_BASE/AAVMF_VARS.fd",
  84         ],
  85     },
  86 };
  87
  88 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  89
  90 # Note about locking: we use flock on the config file protect
  91 # against concurent actions.
  92 # Aditionaly, we have a 'lock' setting in the config file. This
  93 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  94 # allowed when such lock is set. But you can ignore this kind of
  95 # lock with the --skiplock flag.
  96
  97 cfs_register_file('/qemu-server/',
  98                   \&parse_vm_config,
  99                   \&write_vm_config);
 100
 101 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
 102     description => "Some command save/restore state from this location.",
 103     type => 'string',
 104     maxLength => 128,
 105     optional => 1,
 106 });
 107
 108 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
 109         description => "Specifies the Qemu machine type.",
 110         type => 'string',
 111         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
 112         maxLength => 40,
 113         optional => 1,
 114 });
 115
 116
 117 sub map_storage {
 118     my ($map, $source) = @_;
 119
 120     return $source if !defined($map);
 121
 122     return $map->{entries}->{$source}
 123         if $map->{entries} && defined($map->{entries}->{$source});
 124
 125     return $map->{default} if $map->{default};
 126
 127     # identity (fallback)
 128     return $source;
 129 }
 130
 131 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 132     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 133     type => 'string',
 134     format => 'storagepair-list',
 135     optional => 1,
 136 });
 137
 138 #no warnings 'redefine';
 139
 140 my $nodename_cache;
 141 sub nodename {
 142     $nodename_cache //= PVE::INotify::nodename();
 143     return $nodename_cache;
 144 }
 145
 146 my $watchdog_fmt = {
 147     model => {
 148         default_key => 1,
 149         type => 'string',
 150         enum => [qw(i6300esb ib700)],
 151         description => "Watchdog type to emulate.",
 152         default => 'i6300esb',
 153         optional => 1,
 154     },
 155     action => {
 156         type => 'string',
 157         enum => [qw(reset shutdown poweroff pause debug none)],
 158         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 159         optional => 1,
 160     },
 161 };
 162 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 163
 164 my $agent_fmt = {
 165     enabled => {
 166         description => "Enable/disable Qemu GuestAgent.",
 167         type => 'boolean',
 168         default => 0,
 169         default_key => 1,
 170     },
 171     fstrim_cloned_disks => {
 172         description => "Run fstrim after moving a disk or migrating the VM.",
 173         type => 'boolean',
 174         optional => 1,
 175         default => 0
 176     },
 177     type => {
 178         description => "Select the agent type",
 179         type => 'string',
 180         default => 'virtio',
 181         optional => 1,
 182         enum => [qw(virtio isa)],
 183     },
 184 };
 185
 186 my $vga_fmt = {
 187     type => {
 188         description => "Select the VGA type.",
 189         type => 'string',
 190         default => 'std',
 191         optional => 1,
 192         default_key => 1,
 193         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 194     },
 195     memory => {
 196         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 197         type => 'integer',
 198         optional => 1,
 199         minimum => 4,
 200         maximum => 512,
 201     },
 202 };
 203
 204 my $ivshmem_fmt = {
 205     size => {
 206         type => 'integer',
 207         minimum => 1,
 208         description => "The size of the file in MB.",
 209     },
 210     name => {
 211         type => 'string',
 212         pattern => '[a-zA-Z0-9\-]+',
 213         optional => 1,
 214         format_description => 'string',
 215         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 216     },
 217 };
 218
 219 my $audio_fmt = {
 220     device => {
 221         type => 'string',
 222         enum => [qw(ich9-intel-hda intel-hda AC97)],
 223         description =>  "Configure an audio device."
 224     },
 225     driver =>  {
 226         type => 'string',
 227         enum => ['spice', 'none'],
 228         default => 'spice',
 229         optional => 1,
 230         description => "Driver backend for the audio device."
 231     },
 232 };
 233
 234 my $spice_enhancements_fmt = {
 235     foldersharing => {
 236         type => 'boolean',
 237         optional => 1,
 238         default => '0',
 239         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 240     },
 241     videostreaming =>  {
 242         type => 'string',
 243         enum => ['off', 'all', 'filter'],
 244         default => 'off',
 245         optional => 1,
 246         description => "Enable video streaming. Uses compression for detected video streams."
 247     },
 248 };
 249
 250 my $rng_fmt = {
 251     source => {
 252         type => 'string',
 253         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 254         default_key => 1,
 255         description => "The file on the host to gather entropy from. In most"
 256                      . " cases /dev/urandom should be preferred over /dev/random"
 257                      . " to avoid entropy-starvation issues on the host. Using"
 258                      . " urandom does *not* decrease security in any meaningful"
 259                      . " way, as it's still seeded from real entropy, and the"
 260                      . " bytes provided will most likely be mixed with real"
 261                      . " entropy on the guest as well. /dev/hwrng can be used"
 262                      . " to pass through a hardware RNG from the host.",
 263     },
 264     max_bytes => {
 265         type => 'integer',
 266         description => "Maximum bytes of entropy injected into the guest every"
 267                      . " 'period' milliseconds. Prefer a lower value when using"
 268                      . " /dev/random as source. Use 0 to disable limiting"
 269                      . " (potentially dangerous!).",
 270         optional => 1,
 271
 272         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 273         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 274         # of overwhelming the host, provided we're reading from /dev/urandom
 275         default => 1024,
 276     },
 277     period => {
 278         type => 'integer',
 279         description => "Every 'period' milliseconds the entropy-injection quota"
 280                      . " is reset, allowing the guest to retrieve another"
 281                      . " 'max_bytes' of entropy.",
 282         optional => 1,
 283         default => 1000,
 284     },
 285 };
 286
 287 my $confdesc = {
 288     onboot => {
 289         optional => 1,
 290         type => 'boolean',
 291         description => "Specifies whether a VM will be started during system bootup.",
 292         default => 0,
 293     },
 294     autostart => {
 295         optional => 1,
 296         type => 'boolean',
 297         description => "Automatic restart after crash (currently ignored).",
 298         default => 0,
 299     },
 300     hotplug => {
 301         optional => 1,
 302         type => 'string', format => 'pve-hotplug-features',
 303         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 304         default => 'network,disk,usb',
 305     },
 306     reboot => {
 307         optional => 1,
 308         type => 'boolean',
 309         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 310         default => 1,
 311     },
 312     lock => {
 313         optional => 1,
 314         type => 'string',
 315         description => "Lock/unlock the VM.",
 316         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 317     },
 318     cpulimit => {
 319         optional => 1,
 320         type => 'number',
 321         description => "Limit of CPU usage.",
 322         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 323         minimum => 0,
 324         maximum => 128,
 325         default => 0,
 326     },
 327     cpuunits => {
 328         optional => 1,
 329         type => 'integer',
 330         description => "CPU weight for a VM, will be clamped to [1, 10000] in cgroup v2.",
 331         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler."
 332             ." The larger the number is, the more CPU time this VM gets. Number is relative to"
 333             ." weights of all the other running VMs.",
 334         minimum => 2,
 335         maximum => 262144,
 336         default => 'cgroup v1: 1024, cgroup v2: 100',
 337     },
 338     memory => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 342         minimum => 16,
 343         default => 512,
 344     },
 345     balloon => {
 346         optional => 1,
 347         type => 'integer',
 348         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 349         minimum => 0,
 350     },
 351     shares => {
 352         optional => 1,
 353         type => 'integer',
 354         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 355         minimum => 0,
 356         maximum => 50000,
 357         default => 1000,
 358     },
 359     keyboard => {
 360         optional => 1,
 361         type => 'string',
 362         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 363                        "It should not be necessary to set it.",
 364         enum => PVE::Tools::kvmkeymaplist(),
 365         default => undef,
 366     },
 367     name => {
 368         optional => 1,
 369         type => 'string', format => 'dns-name',
 370         description => "Set a name for the VM. Only used on the configuration web interface.",
 371     },
 372     scsihw => {
 373         optional => 1,
 374         type => 'string',
 375         description => "SCSI controller model",
 376         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 377         default => 'lsi',
 378     },
 379     description => {
 380         optional => 1,
 381         type => 'string',
 382         description => "Description for the VM. Shown in the web-interface VM's summary."
 383             ." This is saved as comment inside the configuration file.",
 384         maxLength => 1024 * 8,
 385     },
 386     ostype => {
 387         optional => 1,
 388         type => 'string',
 389         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 390         description => "Specify guest operating system.",
 391         verbose_description => <<EODESC,
 392 Specify guest operating system. This is used to enable special
 393 optimization/features for specific operating systems:
 394
 395 [horizontal]
 396 other;; unspecified OS
 397 wxp;; Microsoft Windows XP
 398 w2k;; Microsoft Windows 2000
 399 w2k3;; Microsoft Windows 2003
 400 w2k8;; Microsoft Windows 2008
 401 wvista;; Microsoft Windows Vista
 402 win7;; Microsoft Windows 7
 403 win8;; Microsoft Windows 8/2012/2012r2
 404 win10;; Microsoft Windows 10/2016/2019
 405 l24;; Linux 2.4 Kernel
 406 l26;; Linux 2.6 - 5.X Kernel
 407 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 408 EODESC
 409     },
 410     boot => {
 411         optional => 1,
 412         type => 'string', format => 'pve-qm-boot',
 413         description => "Specify guest boot order. Use with 'order=', usage with"
 414                      . " no key or 'legacy=' is deprecated.",
 415     },
 416     bootdisk => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-bootdisk',
 419         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 420         pattern => '(ide|sata|scsi|virtio)\d+',
 421     },
 422     smp => {
 423         optional => 1,
 424         type => 'integer',
 425         description => "The number of CPUs. Please use option -sockets instead.",
 426         minimum => 1,
 427         default => 1,
 428     },
 429     sockets => {
 430         optional => 1,
 431         type => 'integer',
 432         description => "The number of CPU sockets.",
 433         minimum => 1,
 434         default => 1,
 435     },
 436     cores => {
 437         optional => 1,
 438         type => 'integer',
 439         description => "The number of cores per socket.",
 440         minimum => 1,
 441         default => 1,
 442     },
 443     numa => {
 444         optional => 1,
 445         type => 'boolean',
 446         description => "Enable/disable NUMA.",
 447         default => 0,
 448     },
 449     hugepages => {
 450         optional => 1,
 451         type => 'string',
 452         description => "Enable/disable hugepages memory.",
 453         enum => [qw(any 2 1024)],
 454     },
 455     keephugepages => {
 456         optional => 1,
 457         type => 'boolean',
 458         default => 0,
 459         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 460             ." after VM shutdown and can be used for subsequent starts.",
 461     },
 462     vcpus => {
 463         optional => 1,
 464         type => 'integer',
 465         description => "Number of hotplugged vcpus.",
 466         minimum => 1,
 467         default => 0,
 468     },
 469     acpi => {
 470         optional => 1,
 471         type => 'boolean',
 472         description => "Enable/disable ACPI.",
 473         default => 1,
 474     },
 475     agent => {
 476         optional => 1,
 477         description => "Enable/disable Qemu GuestAgent and its properties.",
 478         type => 'string',
 479         format => $agent_fmt,
 480     },
 481     kvm => {
 482         optional => 1,
 483         type => 'boolean',
 484         description => "Enable/disable KVM hardware virtualization.",
 485         default => 1,
 486     },
 487     tdf => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Enable/disable time drift fix.",
 491         default => 0,
 492     },
 493     localtime => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Set the real time clock to local time. This is enabled by default if ostype"
 497             ." indicates a Microsoft OS.",
 498     },
 499     freeze => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 503     },
 504     vga => {
 505         optional => 1,
 506         type => 'string', format => $vga_fmt,
 507         description => "Configure the VGA hardware.",
 508         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 509             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 510             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 511             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 512             ." display server. For win* OS you can select how many independent displays you want,"
 513             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 514             ." using a serial device as terminal.",
 515     },
 516     watchdog => {
 517         optional => 1,
 518         type => 'string', format => 'pve-qm-watchdog',
 519         description => "Create a virtual hardware watchdog device.",
 520         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 521             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 522             ." else the watchdog will reset the guest (or execute the respective action specified)",
 523     },
 524     startdate => {
 525         optional => 1,
 526         type => 'string',
 527         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 528         description => "Set the initial date of the real time clock. Valid format for date are:"
 529             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 530         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 531         default => 'now',
 532     },
 533     startup =>  get_standard_option('pve-startup-order'),
 534     template => {
 535         optional => 1,
 536         type => 'boolean',
 537         description => "Enable/disable Template.",
 538         default => 0,
 539     },
 540     args => {
 541         optional => 1,
 542         type => 'string',
 543         description => "Arbitrary arguments passed to kvm.",
 544         verbose_description => <<EODESCR,
 545 Arbitrary arguments passed to kvm, for example:
 546
 547 args: -no-reboot -no-hpet
 548
 549 NOTE: this option is for experts only.
 550 EODESCR
 551     },
 552     tablet => {
 553         optional => 1,
 554         type => 'boolean',
 555         default => 1,
 556         description => "Enable/disable the USB tablet device.",
 557         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 558             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 559             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 560             ." may consider disabling this to save some context switches. This is turned off by"
 561             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 562     },
 563     migrate_speed => {
 564         optional => 1,
 565         type => 'integer',
 566         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 567         minimum => 0,
 568         default => 0,
 569     },
 570     migrate_downtime => {
 571         optional => 1,
 572         type => 'number',
 573         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 574         minimum => 0,
 575         default => 0.1,
 576     },
 577     cdrom => {
 578         optional => 1,
 579         type => 'string', format => 'pve-qm-ide',
 580         typetext => '<volume>',
 581         description => "This is an alias for option -ide2",
 582     },
 583     cpu => {
 584         optional => 1,
 585         description => "Emulated CPU type.",
 586         type => 'string',
 587         format => 'pve-vm-cpu-conf',
 588     },
 589     parent => get_standard_option('pve-snapshot-name', {
 590         optional => 1,
 591         description => "Parent snapshot name. This is used internally, and should not be modified.",
 592     }),
 593     snaptime => {
 594         optional => 1,
 595         description => "Timestamp for snapshots.",
 596         type => 'integer',
 597         minimum => 0,
 598     },
 599     vmstate => {
 600         optional => 1,
 601         type => 'string', format => 'pve-volume-id',
 602         description => "Reference to a volume which stores the VM state. This is used internally"
 603             ." for snapshots.",
 604     },
 605     vmstatestorage => get_standard_option('pve-storage-id', {
 606         description => "Default storage for VM state volumes/files.",
 607         optional => 1,
 608     }),
 609     runningmachine => get_standard_option('pve-qemu-machine', {
 610         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 611             ." for snapshots.",
 612     }),
 613     runningcpu => {
 614         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 615             ." internally for snapshots.",
 616         optional => 1,
 617         type => 'string',
 618         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 619         format_description => 'QEMU -cpu parameter'
 620     },
 621     machine => get_standard_option('pve-qemu-machine'),
 622     arch => {
 623         description => "Virtual processor architecture. Defaults to the host.",
 624         optional => 1,
 625         type => 'string',
 626         enum => [qw(x86_64 aarch64)],
 627     },
 628     smbios1 => {
 629         description => "Specify SMBIOS type 1 fields.",
 630         type => 'string', format => 'pve-qm-smbios1',
 631         maxLength => 512,
 632         optional => 1,
 633     },
 634     protection => {
 635         optional => 1,
 636         type => 'boolean',
 637         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 638             ." remove disk operations.",
 639         default => 0,
 640     },
 641     bios => {
 642         optional => 1,
 643         type => 'string',
 644         enum => [ qw(seabios ovmf) ],
 645         description => "Select BIOS implementation.",
 646         default => 'seabios',
 647     },
 648     vmgenid => {
 649         type => 'string',
 650         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 651         format_description => 'UUID',
 652         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 653             ." to disable explicitly.",
 654         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 655             ." value identifier to the guest OS. This allows to notify the guest operating system"
 656             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 657             ." execution or creation from a template). The guest operating system notices the"
 658             ." change, and is then able to react as appropriate by marking its copies of"
 659             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 660             ."Note that auto-creation only works when done through API/CLI create or update methods"
 661             .", but not when manually editing the config file.",
 662         default => "1 (autogenerated)",
 663         optional => 1,
 664     },
 665     hookscript => {
 666         type => 'string',
 667         format => 'pve-volume-id',
 668         optional => 1,
 669         description => "Script that will be executed during various steps in the vms lifetime.",
 670     },
 671     ivshmem => {
 672         type => 'string',
 673         format => $ivshmem_fmt,
 674         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 675             ." the host.",
 676         optional => 1,
 677     },
 678     audio0 => {
 679         type => 'string',
 680         format => $audio_fmt,
 681         description => "Configure a audio device, useful in combination with QXL/Spice.",
 682         optional => 1
 683     },
 684     spice_enhancements => {
 685         type => 'string',
 686         format => $spice_enhancements_fmt,
 687         description => "Configure additional enhancements for SPICE.",
 688         optional => 1
 689     },
 690     tags => {
 691         type => 'string', format => 'pve-tag-list',
 692         description => 'Tags of the VM. This is only meta information.',
 693         optional => 1,
 694     },
 695     rng0 => {
 696         type => 'string',
 697         format => $rng_fmt,
 698         description => "Configure a VirtIO-based Random Number Generator.",
 699         optional => 1,
 700     },
 701 };
 702
 703 my $cicustom_fmt = {
 704     meta => {
 705         type => 'string',
 706         optional => 1,
 707         description => 'Specify a custom file containing all meta data passed to the VM via"
 708             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 709         format => 'pve-volume-id',
 710         format_description => 'volume',
 711     },
 712     network => {
 713         type => 'string',
 714         optional => 1,
 715         description => 'Specify a custom file containing all network data passed to the VM via'
 716             .' cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720     user => {
 721         type => 'string',
 722         optional => 1,
 723         description => 'Specify a custom file containing all user data passed to the VM via'
 724             .' cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728 };
 729 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 730
 731 my $confdesc_cloudinit = {
 732     citype => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'Specifies the cloud-init configuration format. The default depends on the'
 736             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 737             .' and `configdrive2` for windows.',
 738         enum => ['configdrive2', 'nocloud', 'opennebula'],
 739     },
 740     ciuser => {
 741         optional => 1,
 742         type => 'string',
 743         description => "cloud-init: User name to change ssh keys and password for instead of the"
 744             ." image's configured default user.",
 745     },
 746     cipassword => {
 747         optional => 1,
 748         type => 'string',
 749         description => 'cloud-init: Password to assign the user. Using this is generally not'
 750             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 751             .' support hashed passwords.',
 752     },
 753     cicustom => {
 754         optional => 1,
 755         type => 'string',
 756         description => 'cloud-init: Specify custom files to replace the automatically generated'
 757             .' ones at start.',
 758         format => 'pve-qm-cicustom',
 759     },
 760     searchdomain => {
 761         optional => 1,
 762         type => 'string',
 763         description => "cloud-init: Sets DNS search domains for a container. Create will'
 764             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 765             .' are set.",
 766     },
 767     nameserver => {
 768         optional => 1,
 769         type => 'string', format => 'address-list',
 770         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 771             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 772             .' are set.",
 773     },
 774     sshkeys => {
 775         optional => 1,
 776         type => 'string',
 777         format => 'urlencoded',
 778         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 779     },
 780 };
 781
 782 # what about other qemu settings ?
 783 #cpu => 'string',
 784 #machine => 'string',
 785 #fda => 'file',
 786 #fdb => 'file',
 787 #mtdblock => 'file',
 788 #sd => 'file',
 789 #pflash => 'file',
 790 #snapshot => 'bool',
 791 #bootp => 'file',
 792 ##tftp => 'dir',
 793 ##smb => 'dir',
 794 #kernel => 'file',
 795 #append => 'string',
 796 #initrd => 'file',
 797 ##soundhw => 'string',
 798
 799 while (my ($k, $v) = each %$confdesc) {
 800     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 801 }
 802
 803 my $MAX_USB_DEVICES = 5;
 804 my $MAX_NETS = 32;
 805 my $MAX_SERIAL_PORTS = 4;
 806 my $MAX_PARALLEL_PORTS = 3;
 807 my $MAX_NUMA = 8;
 808
 809 my $numa_fmt = {
 810     cpus => {
 811         type => "string",
 812         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 813         description => "CPUs accessing this NUMA node.",
 814         format_description => "id[-id];...",
 815     },
 816     memory => {
 817         type => "number",
 818         description => "Amount of memory this NUMA node provides.",
 819         optional => 1,
 820     },
 821     hostnodes => {
 822         type => "string",
 823         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 824         description => "Host NUMA nodes to use.",
 825         format_description => "id[-id];...",
 826         optional => 1,
 827     },
 828     policy => {
 829         type => 'string',
 830         enum => [qw(preferred bind interleave)],
 831         description => "NUMA allocation policy.",
 832         optional => 1,
 833     },
 834 };
 835 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 836 my $numadesc = {
 837     optional => 1,
 838     type => 'string', format => $numa_fmt,
 839     description => "NUMA topology.",
 840 };
 841 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 842
 843 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 844     $confdesc->{"numa$i"} = $numadesc;
 845 }
 846
 847 my $nic_model_list = [
 848     'e1000',
 849     'e1000-82540em',
 850     'e1000-82544gc',
 851     'e1000-82545em',
 852     'e1000e',
 853     'i82551',
 854     'i82557b',
 855     'i82559er',
 856     'ne2k_isa',
 857     'ne2k_pci',
 858     'pcnet',
 859     'rtl8139',
 860     'virtio',
 861     'vmxnet3',
 862 ];
 863 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 864
 865 my $net_fmt_bridge_descr = <<__EOD__;
 866 Bridge to attach the network device to. The Proxmox VE standard bridge
 867 is called 'vmbr0'.
 868
 869 If you do not specify a bridge, we create a kvm user (NATed) network
 870 device, which provides DHCP and DNS services. The following addresses
 871 are used:
 872
 873  10.0.2.2   Gateway
 874  10.0.2.3   DNS Server
 875  10.0.2.4   SMB Server
 876
 877 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 878 __EOD__
 879
 880 my $net_fmt = {
 881     macaddr  => get_standard_option('mac-addr', {
 882         description => "MAC address. That address must be unique withing your network. This is"
 883             ." automatically generated if not specified.",
 884     }),
 885     model => {
 886         type => 'string',
 887         description => "Network Card Model. The 'virtio' model provides the best performance with"
 888             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 889             ." best to use 'e1000'.",
 890         enum => $nic_model_list,
 891         default_key => 1,
 892     },
 893     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 894     bridge => {
 895         type => 'string',
 896         description => $net_fmt_bridge_descr,
 897         format_description => 'bridge',
 898         pattern => '[-_.\w\d]+',
 899         optional => 1,
 900     },
 901     queues => {
 902         type => 'integer',
 903         minimum => 0, maximum => 16,
 904         description => 'Number of packet queues to be used on the device.',
 905         optional => 1,
 906     },
 907     rate => {
 908         type => 'number',
 909         minimum => 0,
 910         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 911         optional => 1,
 912     },
 913     tag => {
 914         type => 'integer',
 915         minimum => 1, maximum => 4094,
 916         description => 'VLAN tag to apply to packets on this interface.',
 917         optional => 1,
 918     },
 919     trunks => {
 920         type => 'string',
 921         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 922         description => 'VLAN trunks to pass through this interface.',
 923         format_description => 'vlanid[;vlanid...]',
 924         optional => 1,
 925     },
 926     firewall => {
 927         type => 'boolean',
 928         description => 'Whether this interface should be protected by the firewall.',
 929         optional => 1,
 930     },
 931     link_down => {
 932         type => 'boolean',
 933         description => 'Whether this interface should be disconnected (like pulling the plug).',
 934         optional => 1,
 935     },
 936     mtu => {
 937         type => 'integer',
 938         minimum => 1, maximum => 65520,
 939         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 940         optional => 1,
 941     },
 942 };
 943
 944 my $netdesc = {
 945     optional => 1,
 946     type => 'string', format => $net_fmt,
 947     description => "Specify network devices.",
 948 };
 949
 950 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 951
 952 my $ipconfig_fmt = {
 953     ip => {
 954         type => 'string',
 955         format => 'pve-ipv4-config',
 956         format_description => 'IPv4Format/CIDR',
 957         description => 'IPv4 address in CIDR format.',
 958         optional => 1,
 959         default => 'dhcp',
 960     },
 961     gw => {
 962         type => 'string',
 963         format => 'ipv4',
 964         format_description => 'GatewayIPv4',
 965         description => 'Default gateway for IPv4 traffic.',
 966         optional => 1,
 967         requires => 'ip',
 968     },
 969     ip6 => {
 970         type => 'string',
 971         format => 'pve-ipv6-config',
 972         format_description => 'IPv6Format/CIDR',
 973         description => 'IPv6 address in CIDR format.',
 974         optional => 1,
 975         default => 'dhcp',
 976     },
 977     gw6 => {
 978         type => 'string',
 979         format => 'ipv6',
 980         format_description => 'GatewayIPv6',
 981         description => 'Default gateway for IPv6 traffic.',
 982         optional => 1,
 983         requires => 'ip6',
 984     },
 985 };
 986 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 987 my $ipconfigdesc = {
 988     optional => 1,
 989     type => 'string', format => 'pve-qm-ipconfig',
 990     description => <<'EODESCR',
 991 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 992
 993 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 994
 995 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 996 gateway should be provided.
 997 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration. This requires
 998 cloud-init 19.4 or newer.
 999
1000 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
1001 dhcp on IPv4.
1002 EODESCR
1003 };
1004 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
1005
1006 for (my $i = 0; $i < $MAX_NETS; $i++)  {
1007     $confdesc->{"net$i"} = $netdesc;
1008     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
1009 }
1010
1011 foreach my $key (keys %$confdesc_cloudinit) {
1012     $confdesc->{$key} = $confdesc_cloudinit->{$key};
1013 }
1014
1015 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
1016 sub verify_volume_id_or_qm_path {
1017     my ($volid, $noerr) = @_;
1018
1019     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
1020         return $volid;
1021     }
1022
1023     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
1024     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
1025     if ($@) {
1026         return if $noerr;
1027         die $@;
1028     }
1029     return $volid;
1030 }
1031
1032 my $usb_fmt = {
1033     host => {
1034         default_key => 1,
1035         type => 'string', format => 'pve-qm-usb-device',
1036         format_description => 'HOSTUSBDEVICE|spice',
1037         description => <<EODESCR,
1038 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1039
1040  'bus-port(.port)*' (decimal numbers) or
1041  'vendor_id:product_id' (hexadeciaml numbers) or
1042  'spice'
1043
1044 You can use the 'lsusb -t' command to list existing usb devices.
1045
1046 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1047 machines - use with special care.
1048
1049 The value 'spice' can be used to add a usb redirection devices for spice.
1050 EODESCR
1051     },
1052     usb3 => {
1053         optional => 1,
1054         type => 'boolean',
1055         description => "Specifies whether if given host option is a USB3 device or port.",
1056         default => 0,
1057     },
1058 };
1059
1060 my $usbdesc = {
1061     optional => 1,
1062     type => 'string', format => $usb_fmt,
1063     description => "Configure an USB device (n is 0 to 4).",
1064 };
1065 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1066
1067 my $serialdesc = {
1068         optional => 1,
1069         type => 'string',
1070         pattern => '(/dev/.+|socket)',
1071         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1072         verbose_description =>  <<EODESCR,
1073 Create a serial device inside the VM (n is 0 to 3), and pass through a
1074 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1075 host side (use 'qm terminal' to open a terminal connection).
1076
1077 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1078 use with special care.
1079
1080 CAUTION: Experimental! User reported problems with this option.
1081 EODESCR
1082 };
1083
1084 my $paralleldesc= {
1085         optional => 1,
1086         type => 'string',
1087         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1088         description =>  "Map host parallel devices (n is 0 to 2).",
1089         verbose_description =>  <<EODESCR,
1090 Map host parallel devices (n is 0 to 2).
1091
1092 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1093 machines - use with special care.
1094
1095 CAUTION: Experimental! User reported problems with this option.
1096 EODESCR
1097 };
1098
1099 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1100     $confdesc->{"parallel$i"} = $paralleldesc;
1101 }
1102
1103 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1104     $confdesc->{"serial$i"} = $serialdesc;
1105 }
1106
1107 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1108     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1109 }
1110
1111 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1112     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1113 }
1114
1115 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1116     $confdesc->{"usb$i"} = $usbdesc;
1117 }
1118
1119 my $boot_fmt = {
1120     legacy => {
1121         optional => 1,
1122         default_key => 1,
1123         type => 'string',
1124         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1125                      . " Deprecated, use 'order=' instead.",
1126         pattern => '[acdn]{1,4}',
1127         format_description => "[acdn]{1,4}",
1128
1129         # note: this is also the fallback if boot: is not given at all
1130         default => 'cdn',
1131     },
1132     order => {
1133         optional => 1,
1134         type => 'string',
1135         format => 'pve-qm-bootdev-list',
1136         format_description => "device[;device...]",
1137         description => <<EODESC,
1138 The guest will attempt to boot from devices in the order they appear here.
1139
1140 Disks, optical drives and passed-through storage USB devices will be directly
1141 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1142 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1143
1144 Note that only devices in this list will be marked as bootable and thus loaded
1145 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1146 (e.g. software-raid), you need to specify all of them here.
1147
1148 Overrides the deprecated 'legacy=[acdn]*' value when given.
1149 EODESC
1150     },
1151 };
1152 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1153
1154 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1155 sub verify_bootdev {
1156     my ($dev, $noerr) = @_;
1157
1158     my $special = $dev =~ m/^efidisk/ || $dev =~ m/^tpmstate/;
1159     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && !$special;
1160
1161     my $check = sub {
1162         my ($base) = @_;
1163         return 0 if $dev !~ m/^$base\d+$/;
1164         return 0 if !$confdesc->{$dev};
1165         return 1;
1166     };
1167
1168     return $dev if $check->("net");
1169     return $dev if $check->("usb");
1170     return $dev if $check->("hostpci");
1171
1172     return if $noerr;
1173     die "invalid boot device '$dev'\n";
1174 }
1175
1176 sub print_bootorder {
1177     my ($devs) = @_;
1178     return "" if !@$devs;
1179     my $data = { order => join(';', @$devs) };
1180     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1181 }
1182
1183 my $kvm_api_version = 0;
1184
1185 sub kvm_version {
1186     return $kvm_api_version if $kvm_api_version;
1187
1188     open my $fh, '<', '/dev/kvm' or return;
1189
1190     # 0xae00 => KVM_GET_API_VERSION
1191     $kvm_api_version = ioctl($fh, 0xae00, 0);
1192     close($fh);
1193
1194     return $kvm_api_version;
1195 }
1196
1197 my $kvm_user_version = {};
1198 my $kvm_mtime = {};
1199
1200 sub kvm_user_version {
1201     my ($binary) = @_;
1202
1203     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1204     my $st = stat($binary);
1205
1206     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1207     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1208         $cachedmtime == $st->mtime;
1209
1210     $kvm_user_version->{$binary} = 'unknown';
1211     $kvm_mtime->{$binary} = $st->mtime;
1212
1213     my $code = sub {
1214         my $line = shift;
1215         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1216             $kvm_user_version->{$binary} = $2;
1217         }
1218     };
1219
1220     eval { run_command([$binary, '--version'], outfunc => $code); };
1221     warn $@ if $@;
1222
1223     return $kvm_user_version->{$binary};
1224
1225 }
1226 my sub extract_version {
1227     my ($machine_type, $version) = @_;
1228     $version = kvm_user_version() if !defined($version);
1229     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1230 }
1231
1232 sub kernel_has_vhost_net {
1233     return -c '/dev/vhost-net';
1234 }
1235
1236 sub option_exists {
1237     my $key = shift;
1238     return defined($confdesc->{$key});
1239 }
1240
1241 my $cdrom_path;
1242 sub get_cdrom_path {
1243
1244     return  $cdrom_path if $cdrom_path;
1245
1246     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1247     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1248     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269      if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1314             if !$vtype;
1315         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1316         verify_media_type($opt, $vtype, $drive->{media});
1317         $drive->{file} = $volid;
1318     }
1319
1320     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1321 }
1322
1323 sub parse_hotplug_features {
1324     my ($data) = @_;
1325
1326     my $res = {};
1327
1328     return $res if $data eq '0';
1329
1330     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1331
1332     foreach my $feature (PVE::Tools::split_list($data)) {
1333         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1334             $res->{$1} = 1;
1335         } else {
1336             die "invalid hotplug feature '$feature'\n";
1337         }
1338     }
1339     return $res;
1340 }
1341
1342 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1343 sub pve_verify_hotplug_features {
1344     my ($value, $noerr) = @_;
1345
1346     return $value if parse_hotplug_features($value);
1347
1348     return if $noerr;
1349
1350     die "unable to parse hotplug option\n";
1351 }
1352
1353 sub scsi_inquiry {
1354     my($fh, $noerr) = @_;
1355
1356     my $SG_IO = 0x2285;
1357     my $SG_GET_VERSION_NUM = 0x2282;
1358
1359     my $versionbuf = "\x00" x 8;
1360     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1361     if (!$ret) {
1362         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1363         return;
1364     }
1365     my $version = unpack("I", $versionbuf);
1366     if ($version < 30000) {
1367         die "scsi generic interface too old\n"  if !$noerr;
1368         return;
1369     }
1370
1371     my $buf = "\x00" x 36;
1372     my $sensebuf = "\x00" x 8;
1373     my $cmd = pack("C x3 C x1", 0x12, 36);
1374
1375     # see /usr/include/scsi/sg.h
1376     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1377
1378     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1379                       length($sensebuf), 0, length($buf), $buf,
1380                       $cmd, $sensebuf, 6000);
1381
1382     $ret = ioctl($fh, $SG_IO, $packet);
1383     if (!$ret) {
1384         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1385         return;
1386     }
1387
1388     my @res = unpack($sg_io_hdr_t, $packet);
1389     if ($res[17] || $res[18]) {
1390         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1391         return;
1392     }
1393
1394     my $res = {};
1395     (my $byte0, my $byte1, $res->{vendor},
1396      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1397
1398     $res->{removable} = $byte1 & 128 ? 1 : 0;
1399     $res->{type} = $byte0 & 31;
1400
1401     return $res;
1402 }
1403
1404 sub path_is_scsi {
1405     my ($path) = @_;
1406
1407     my $fh = IO::File->new("+<$path") || return;
1408     my $res = scsi_inquiry($fh, 1);
1409     close($fh);
1410
1411     return $res;
1412 }
1413
1414 sub print_tabletdevice_full {
1415     my ($conf, $arch) = @_;
1416
1417     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1418
1419     # we use uhci for old VMs because tablet driver was buggy in older qemu
1420     my $usbbus;
1421     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1422         $usbbus = 'ehci';
1423     } else {
1424         $usbbus = 'uhci';
1425     }
1426
1427     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1428 }
1429
1430 sub print_keyboarddevice_full {
1431     my ($conf, $arch, $machine) = @_;
1432
1433     return if $arch ne 'aarch64';
1434
1435     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1436 }
1437
1438 my sub get_drive_id {
1439     my ($drive) = @_;
1440     return "$drive->{interface}$drive->{index}";
1441 }
1442
1443 sub print_drivedevice_full {
1444     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1445
1446     my $device = '';
1447     my $maxdev = 0;
1448
1449     my $drive_id = get_drive_id($drive);
1450     if ($drive->{interface} eq 'virtio') {
1451         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1452         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1453         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1454     } elsif ($drive->{interface} eq 'scsi') {
1455
1456         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1457         my $unit = $drive->{index} % $maxdev;
1458         my $devicetype = 'hd';
1459         my $path = '';
1460         if (drive_is_cdrom($drive)) {
1461             $devicetype = 'cd';
1462         } else {
1463             if ($drive->{file} =~ m|^/|) {
1464                 $path = $drive->{file};
1465                 if (my $info = path_is_scsi($path)) {
1466                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1467                         $devicetype = 'block';
1468                     } elsif ($info->{type} == 1) { # tape
1469                         $devicetype = 'generic';
1470                     }
1471                 }
1472             } else {
1473                  $path = PVE::Storage::path($storecfg, $drive->{file});
1474             }
1475
1476             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1477             my $version = extract_version($machine_type, kvm_user_version());
1478             if ($path =~ m/^iscsi\:\/\// &&
1479                !min_version($version, 4, 1)) {
1480                 $devicetype = 'generic';
1481             }
1482         }
1483
1484         if (!$conf->{scsihw} || $conf->{scsihw} =~ m/^lsi/ || $conf->{scsihw} eq 'pvscsi') {
1485             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1486         } else {
1487             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1488                 .",lun=$drive->{index}";
1489         }
1490         $device .= ",drive=drive-$drive_id,id=$drive_id";
1491
1492         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1493             $device .= ",rotation_rate=1";
1494         }
1495         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1496
1497     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1498         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1499         my $controller = int($drive->{index} / $maxdev);
1500         my $unit = $drive->{index} % $maxdev;
1501         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1502
1503         $device = "ide-$devicetype";
1504         if ($drive->{interface} eq 'ide') {
1505             $device .= ",bus=ide.$controller,unit=$unit";
1506         } else {
1507             $device .= ",bus=ahci$controller.$unit";
1508         }
1509         $device .= ",drive=drive-$drive_id,id=$drive_id";
1510
1511         if ($devicetype eq 'hd') {
1512             if (my $model = $drive->{model}) {
1513                 $model = URI::Escape::uri_unescape($model);
1514                 $device .= ",model=$model";
1515             }
1516             if ($drive->{ssd}) {
1517                 $device .= ",rotation_rate=1";
1518             }
1519         }
1520         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1521     } elsif ($drive->{interface} eq 'usb') {
1522         die "implement me";
1523         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1524     } else {
1525         die "unsupported interface type";
1526     }
1527
1528     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1529
1530     if (my $serial = $drive->{serial}) {
1531         $serial = URI::Escape::uri_unescape($serial);
1532         $device .= ",serial=$serial";
1533     }
1534
1535
1536     return $device;
1537 }
1538
1539 sub get_initiator_name {
1540     my $initiator;
1541
1542     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1543     while (defined(my $line = <$fh>)) {
1544         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1545         $initiator = $1;
1546         last;
1547     }
1548     $fh->close();
1549
1550     return $initiator;
1551 }
1552
1553 sub print_drive_commandline_full {
1554     my ($storecfg, $vmid, $drive, $pbs_name, $io_uring) = @_;
1555
1556     my $path;
1557     my $volid = $drive->{file};
1558     my $format = $drive->{format};
1559     my $drive_id = get_drive_id($drive);
1560
1561     my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1562     my $scfg = $storeid ? PVE::Storage::storage_config($storecfg, $storeid) : undef;
1563
1564     if (drive_is_cdrom($drive)) {
1565         $path = get_iso_path($storecfg, $vmid, $volid);
1566         die "$drive_id: cannot back cdrom drive with PBS snapshot\n" if $pbs_name;
1567     } else {
1568         if ($storeid) {
1569             $path = PVE::Storage::path($storecfg, $volid);
1570             $format //= qemu_img_format($scfg, $volname);
1571         } else {
1572             $path = $volid;
1573             $format //= "raw";
1574         }
1575    }
1576
1577    my $is_rbd = $path =~ m/^rbd:/;
1578
1579     my $opts = '';
1580     my @qemu_drive_options = qw(heads secs cyls trans media cache rerror werror aio discard);
1581     foreach my $o (@qemu_drive_options) {
1582         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1583     }
1584
1585     # snapshot only accepts on|off
1586     if (defined($drive->{snapshot})) {
1587         my $v = $drive->{snapshot} ? 'on' : 'off';
1588         $opts .= ",snapshot=$v";
1589     }
1590
1591     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1592         my ($dir, $qmpname) = @$type;
1593         if (my $v = $drive->{"mbps$dir"}) {
1594             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1595         }
1596         if (my $v = $drive->{"mbps${dir}_max"}) {
1597             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1598         }
1599         if (my $v = $drive->{"bps${dir}_max_length"}) {
1600             $opts .= ",throttling.bps$qmpname-max-length=$v";
1601         }
1602         if (my $v = $drive->{"iops${dir}"}) {
1603             $opts .= ",throttling.iops$qmpname=$v";
1604         }
1605         if (my $v = $drive->{"iops${dir}_max"}) {
1606             $opts .= ",throttling.iops$qmpname-max=$v";
1607         }
1608         if (my $v = $drive->{"iops${dir}_max_length"}) {
1609             $opts .= ",throttling.iops$qmpname-max-length=$v";
1610         }
1611     }
1612
1613     if ($pbs_name) {
1614         $format = "rbd" if $is_rbd;
1615         die "$drive_id: Proxmox Backup Server backed drive cannot auto-detect the format\n"
1616             if !$format;
1617         $opts .= ",format=alloc-track,file.driver=$format";
1618     } elsif ($format) {
1619         $opts .= ",format=$format";
1620     }
1621
1622     my $cache_direct = 0;
1623
1624     if (my $cache = $drive->{cache}) {
1625         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1626     } elsif (!drive_is_cdrom($drive) && !($scfg && $scfg->{type} eq 'btrfs' && !$scfg->{nocow})) {
1627         $opts .= ",cache=none";
1628         $cache_direct = 1;
1629     }
1630
1631     # io_uring with cache mode writeback or writethrough on krbd will hang...
1632     my $rbd_no_io_uring = $scfg && $scfg->{type} eq 'rbd' && $scfg->{krbd} && !$cache_direct;
1633
1634     # io_uring with cache mode writeback or writethrough on LVM will hang, without cache only
1635     # sometimes, just plain disable...
1636     my $lvm_no_io_uring = $scfg && $scfg->{type} eq 'lvm';
1637
1638     if (!$drive->{aio}) {
1639         if ($io_uring && !$rbd_no_io_uring && !$lvm_no_io_uring) {
1640             # io_uring supports all cache modes
1641             $opts .= ",aio=io_uring";
1642         } else {
1643             # aio native works only with O_DIRECT
1644             if($cache_direct) {
1645                 $opts .= ",aio=native";
1646             } else {
1647                 $opts .= ",aio=threads";
1648             }
1649         }
1650     }
1651
1652     if (!drive_is_cdrom($drive)) {
1653         my $detectzeroes;
1654         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1655             $detectzeroes = 'off';
1656         } elsif ($drive->{discard}) {
1657             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1658         } else {
1659             # This used to be our default with discard not being specified:
1660             $detectzeroes = 'on';
1661         }
1662
1663         # note: 'detect-zeroes' works per blockdev and we want it to persist
1664         # after the alloc-track is removed, so put it on 'file' directly
1665         my $dz_param = $pbs_name ? "file.detect-zeroes" : "detect-zeroes";
1666         $opts .= ",$dz_param=$detectzeroes" if $detectzeroes;
1667     }
1668
1669     if ($pbs_name) {
1670         $opts .= ",backing=$pbs_name";
1671         $opts .= ",auto-remove=on";
1672     }
1673
1674     # my $file_param = $pbs_name ? "file.file.filename" : "file";
1675     my $file_param = "file";
1676     if ($pbs_name) {
1677         # non-rbd drivers require the underlying file to be a seperate block
1678         # node, so add a second .file indirection
1679         $file_param .= ".file" if !$is_rbd;
1680         $file_param .= ".filename";
1681     }
1682     my $pathinfo = $path ? "$file_param=$path," : '';
1683
1684     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1685 }
1686
1687 sub print_pbs_blockdev {
1688     my ($pbs_conf, $pbs_name) = @_;
1689     my $blockdev = "driver=pbs,node-name=$pbs_name,read-only=on";
1690     $blockdev .= ",repository=$pbs_conf->{repository}";
1691     $blockdev .= ",snapshot=$pbs_conf->{snapshot}";
1692     $blockdev .= ",archive=$pbs_conf->{archive}";
1693     $blockdev .= ",keyfile=$pbs_conf->{keyfile}" if $pbs_conf->{keyfile};
1694     return $blockdev;
1695 }
1696
1697 sub print_netdevice_full {
1698     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1699
1700     my $device = $net->{model};
1701     if ($net->{model} eq 'virtio') {
1702          $device = 'virtio-net-pci';
1703      };
1704
1705     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1706     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1707     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1708         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1709         # and out of each queue plus one config interrupt and control vector queue
1710         my $vectors = $net->{queues} * 2 + 2;
1711         $tmpstr .= ",vectors=$vectors,mq=on";
1712     }
1713     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1714
1715     if (my $mtu = $net->{mtu}) {
1716         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1717             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1718             if ($mtu == 1) {
1719                  $mtu = $bridge_mtu;
1720             } elsif ($mtu < 576) {
1721                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1722             } elsif ($mtu > $bridge_mtu) {
1723                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1724             }
1725             $tmpstr .= ",host_mtu=$mtu";
1726         } else {
1727             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1728         }
1729     }
1730
1731     if ($use_old_bios_files) {
1732         my $romfile;
1733         if ($device eq 'virtio-net-pci') {
1734             $romfile = 'pxe-virtio.rom';
1735         } elsif ($device eq 'e1000') {
1736             $romfile = 'pxe-e1000.rom';
1737         } elsif ($device eq 'e1000e') {
1738             $romfile = 'pxe-e1000e.rom';
1739         } elsif ($device eq 'ne2k') {
1740             $romfile = 'pxe-ne2k_pci.rom';
1741         } elsif ($device eq 'pcnet') {
1742             $romfile = 'pxe-pcnet.rom';
1743         } elsif ($device eq 'rtl8139') {
1744             $romfile = 'pxe-rtl8139.rom';
1745         }
1746         $tmpstr .= ",romfile=$romfile" if $romfile;
1747     }
1748
1749     return $tmpstr;
1750 }
1751
1752 sub print_netdev_full {
1753     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1754
1755     my $i = '';
1756     if ($netid =~ m/^net(\d+)$/) {
1757         $i = int($1);
1758     }
1759
1760     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1761
1762     my $ifname = "tap${vmid}i$i";
1763
1764     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1765     die "interface name '$ifname' is too long (max 15 character)\n"
1766         if length($ifname) >= 16;
1767
1768     my $vhostparam = '';
1769     if (is_native($arch)) {
1770         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1771     }
1772
1773     my $vmname = $conf->{name} || "vm$vmid";
1774
1775     my $netdev = "";
1776     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1777
1778     if ($net->{bridge}) {
1779         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1780             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1781     } else {
1782         $netdev = "type=user,id=$netid,hostname=$vmname";
1783     }
1784
1785     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1786
1787     return $netdev;
1788 }
1789
1790 my $vga_map = {
1791     'cirrus' => 'cirrus-vga',
1792     'std' => 'VGA',
1793     'vmware' => 'vmware-svga',
1794     'virtio' => 'virtio-vga',
1795 };
1796
1797 sub print_vga_device {
1798     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1799
1800     my $type = $vga_map->{$vga->{type}};
1801     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1802         $type = 'virtio-gpu';
1803     }
1804     my $vgamem_mb = $vga->{memory};
1805
1806     my $max_outputs = '';
1807     if ($qxlnum) {
1808         $type = $id ? 'qxl' : 'qxl-vga';
1809
1810         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1811             # set max outputs so linux can have up to 4 qxl displays with one device
1812             if (min_version($machine_version, 4, 1)) {
1813                 $max_outputs = ",max_outputs=4";
1814             }
1815         }
1816     }
1817
1818     die "no devicetype for $vga->{type}\n" if !$type;
1819
1820     my $memory = "";
1821     if ($vgamem_mb) {
1822         if ($vga->{type} eq 'virtio') {
1823             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1824             $memory = ",max_hostmem=$bytes";
1825         } elsif ($qxlnum) {
1826             # from https://www.spice-space.org/multiple-monitors.html
1827             $memory = ",vgamem_mb=$vga->{memory}";
1828             my $ram = $vgamem_mb * 4;
1829             my $vram = $vgamem_mb * 2;
1830             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1831         } else {
1832             $memory = ",vgamem_mb=$vga->{memory}";
1833         }
1834     } elsif ($qxlnum && $id) {
1835         $memory = ",ram_size=67108864,vram_size=33554432";
1836     }
1837
1838     my $edidoff = "";
1839     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1840         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1841     }
1842
1843     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1844     my $vgaid = "vga" . ($id // '');
1845     my $pciaddr;
1846
1847     if ($q35 && $vgaid eq 'vga') {
1848         # the first display uses pcie.0 bus on q35 machines
1849         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1850     } else {
1851         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1852     }
1853
1854     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1855 }
1856
1857 sub parse_number_sets {
1858     my ($set) = @_;
1859     my $res = [];
1860     foreach my $part (split(/;/, $set)) {
1861         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1862             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1863             push @$res, [ $1, $2 ];
1864         } else {
1865             die "invalid range: $part\n";
1866         }
1867     }
1868     return $res;
1869 }
1870
1871 sub parse_numa {
1872     my ($data) = @_;
1873
1874     my $res = parse_property_string($numa_fmt, $data);
1875     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1876     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1877     return $res;
1878 }
1879
1880 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1881 sub parse_net {
1882     my ($data) = @_;
1883
1884     my $res = eval { parse_property_string($net_fmt, $data) };
1885     if ($@) {
1886         warn $@;
1887         return;
1888     }
1889     if (!defined($res->{macaddr})) {
1890         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1891         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1892     }
1893     return $res;
1894 }
1895
1896 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1897 sub parse_ipconfig {
1898     my ($data) = @_;
1899
1900     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1901     if ($@) {
1902         warn $@;
1903         return;
1904     }
1905
1906     if ($res->{gw} && !$res->{ip}) {
1907         warn 'gateway specified without specifying an IP address';
1908         return;
1909     }
1910     if ($res->{gw6} && !$res->{ip6}) {
1911         warn 'IPv6 gateway specified without specifying an IPv6 address';
1912         return;
1913     }
1914     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1915         warn 'gateway specified together with DHCP';
1916         return;
1917     }
1918     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1919         # gw6 + auto/dhcp
1920         warn "IPv6 gateway specified together with $res->{ip6} address";
1921         return;
1922     }
1923
1924     if (!$res->{ip} && !$res->{ip6}) {
1925         return { ip => 'dhcp', ip6 => 'dhcp' };
1926     }
1927
1928     return $res;
1929 }
1930
1931 sub print_net {
1932     my $net = shift;
1933
1934     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1935 }
1936
1937 sub add_random_macs {
1938     my ($settings) = @_;
1939
1940     foreach my $opt (keys %$settings) {
1941         next if $opt !~ m/^net(\d+)$/;
1942         my $net = parse_net($settings->{$opt});
1943         next if !$net;
1944         $settings->{$opt} = print_net($net);
1945     }
1946 }
1947
1948 sub vm_is_volid_owner {
1949     my ($storecfg, $vmid, $volid) = @_;
1950
1951     if ($volid !~  m|^/|) {
1952         my ($path, $owner);
1953         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1954         if ($owner && ($owner == $vmid)) {
1955             return 1;
1956         }
1957     }
1958
1959     return;
1960 }
1961
1962 sub vmconfig_register_unused_drive {
1963     my ($storecfg, $vmid, $conf, $drive) = @_;
1964
1965     if (drive_is_cloudinit($drive)) {
1966         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1967         warn $@ if $@;
1968     } elsif (!drive_is_cdrom($drive)) {
1969         my $volid = $drive->{file};
1970         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1971             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1972         }
1973     }
1974 }
1975
1976 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1977 my $smbios1_fmt = {
1978     uuid => {
1979         type => 'string',
1980         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1981         format_description => 'UUID',
1982         description => "Set SMBIOS1 UUID.",
1983         optional => 1,
1984     },
1985     version => {
1986         type => 'string',
1987         pattern => '[A-Za-z0-9+\/]+={0,2}',
1988         format_description => 'Base64 encoded string',
1989         description => "Set SMBIOS1 version.",
1990         optional => 1,
1991     },
1992     serial => {
1993         type => 'string',
1994         pattern => '[A-Za-z0-9+\/]+={0,2}',
1995         format_description => 'Base64 encoded string',
1996         description => "Set SMBIOS1 serial number.",
1997         optional => 1,
1998     },
1999     manufacturer => {
2000         type => 'string',
2001         pattern => '[A-Za-z0-9+\/]+={0,2}',
2002         format_description => 'Base64 encoded string',
2003         description => "Set SMBIOS1 manufacturer.",
2004         optional => 1,
2005     },
2006     product => {
2007         type => 'string',
2008         pattern => '[A-Za-z0-9+\/]+={0,2}',
2009         format_description => 'Base64 encoded string',
2010         description => "Set SMBIOS1 product ID.",
2011         optional => 1,
2012     },
2013     sku => {
2014         type => 'string',
2015         pattern => '[A-Za-z0-9+\/]+={0,2}',
2016         format_description => 'Base64 encoded string',
2017         description => "Set SMBIOS1 SKU string.",
2018         optional => 1,
2019     },
2020     family => {
2021         type => 'string',
2022         pattern => '[A-Za-z0-9+\/]+={0,2}',
2023         format_description => 'Base64 encoded string',
2024         description => "Set SMBIOS1 family string.",
2025         optional => 1,
2026     },
2027     base64 => {
2028         type => 'boolean',
2029         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2030         optional => 1,
2031     },
2032 };
2033
2034 sub parse_smbios1 {
2035     my ($data) = @_;
2036
2037     my $res = eval { parse_property_string($smbios1_fmt, $data) };
2038     warn $@ if $@;
2039     return $res;
2040 }
2041
2042 sub print_smbios1 {
2043     my ($smbios1) = @_;
2044     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2045 }
2046
2047 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2048
2049 sub parse_watchdog {
2050     my ($value) = @_;
2051
2052     return if !$value;
2053
2054     my $res = eval { parse_property_string($watchdog_fmt, $value) };
2055     warn $@ if $@;
2056     return $res;
2057 }
2058
2059 sub parse_guest_agent {
2060     my ($conf) = @_;
2061
2062     return {} if !defined($conf->{agent});
2063
2064     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
2065     warn $@ if $@;
2066
2067     # if the agent is disabled ignore the other potentially set properties
2068     return {} if !$res->{enabled};
2069     return $res;
2070 }
2071
2072 sub get_qga_key {
2073     my ($conf, $key) = @_;
2074     return undef if !defined($conf->{agent});
2075
2076     my $agent = parse_guest_agent($conf);
2077     return $agent->{$key};
2078 }
2079
2080 sub parse_vga {
2081     my ($value) = @_;
2082
2083     return {} if !$value;
2084     my $res = eval { parse_property_string($vga_fmt, $value) };
2085     warn $@ if $@;
2086     return $res;
2087 }
2088
2089 sub parse_rng {
2090     my ($value) = @_;
2091
2092     return if !$value;
2093
2094     my $res = eval { parse_property_string($rng_fmt, $value) };
2095     warn $@ if $@;
2096     return $res;
2097 }
2098
2099 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2100 sub verify_usb_device {
2101     my ($value, $noerr) = @_;
2102
2103     return $value if parse_usb_device($value);
2104
2105     return if $noerr;
2106
2107     die "unable to parse usb device\n";
2108 }
2109
2110 # add JSON properties for create and set function
2111 sub json_config_properties {
2112     my $prop = shift;
2113
2114     foreach my $opt (keys %$confdesc) {
2115         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2116             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2117         $prop->{$opt} = $confdesc->{$opt};
2118     }
2119
2120     return $prop;
2121 }
2122
2123 # return copy of $confdesc_cloudinit to generate documentation
2124 sub cloudinit_config_properties {
2125
2126     return dclone($confdesc_cloudinit);
2127 }
2128
2129 sub check_type {
2130     my ($key, $value) = @_;
2131
2132     die "unknown setting '$key'\n" if !$confdesc->{$key};
2133
2134     my $type = $confdesc->{$key}->{type};
2135
2136     if (!defined($value)) {
2137         die "got undefined value\n";
2138     }
2139
2140     if ($value =~ m/[\n\r]/) {
2141         die "property contains a line feed\n";
2142     }
2143
2144     if ($type eq 'boolean') {
2145         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2146         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2147         die "type check ('boolean') failed - got '$value'\n";
2148     } elsif ($type eq 'integer') {
2149         return int($1) if $value =~ m/^(\d+)$/;
2150         die "type check ('integer') failed - got '$value'\n";
2151     } elsif ($type eq 'number') {
2152         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2153         die "type check ('number') failed - got '$value'\n";
2154     } elsif ($type eq 'string') {
2155         if (my $fmt = $confdesc->{$key}->{format}) {
2156             PVE::JSONSchema::check_format($fmt, $value);
2157             return $value;
2158         }
2159         $value =~ s/^\"(.*)\"$/$1/;
2160         return $value;
2161     } else {
2162         die "internal error"
2163     }
2164 }
2165
2166 sub destroy_vm {
2167     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2168
2169     my $conf = PVE::QemuConfig->load_config($vmid);
2170
2171     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2172
2173     if ($conf->{template}) {
2174         # check if any base image is still used by a linked clone
2175         PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2176                 my ($ds, $drive) = @_;
2177                 return if drive_is_cdrom($drive);
2178
2179                 my $volid = $drive->{file};
2180                 return if !$volid || $volid =~ m|^/|;
2181
2182                 die "base volume '$volid' is still in use by linked cloned\n"
2183                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2184
2185         });
2186     }
2187
2188     my $volids = {};
2189     my $remove_owned_drive = sub {
2190         my ($ds, $drive) = @_;
2191         return if drive_is_cdrom($drive, 1);
2192
2193         my $volid = $drive->{file};
2194         return if !$volid || $volid =~ m|^/|;
2195         return if $volids->{$volid};
2196
2197         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2198         return if !$path || !$owner || ($owner != $vmid);
2199
2200         $volids->{$volid} = 1;
2201         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2202         warn "Could not remove disk '$volid', check manually: $@" if $@;
2203     };
2204
2205     # only remove disks owned by this VM (referenced in the config)
2206     my $include_opts = {
2207         include_unused => 1,
2208         extra_keys => ['vmstate'],
2209     };
2210     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $remove_owned_drive);
2211
2212     for my $snap (values %{$conf->{snapshots}}) {
2213         next if !defined($snap->{vmstate});
2214         my $drive = PVE::QemuConfig->parse_volume('vmstate', $snap->{vmstate}, 1);
2215         next if !defined($drive);
2216         $remove_owned_drive->('vmstate', $drive);
2217     }
2218
2219     PVE::QemuConfig->foreach_volume_full($conf->{pending}, $include_opts, $remove_owned_drive);
2220
2221     if ($purge_unreferenced) { # also remove unreferenced disk
2222         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid, undef, 'images');
2223         PVE::Storage::foreach_volid($vmdisks, sub {
2224             my ($volid, $sid, $volname, $d) = @_;
2225             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2226             warn $@ if $@;
2227         });
2228     }
2229
2230     if (defined $replacement_conf) {
2231         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2232     } else {
2233         PVE::QemuConfig->destroy_config($vmid);
2234     }
2235 }
2236
2237 sub parse_vm_config {
2238     my ($filename, $raw) = @_;
2239
2240     return if !defined($raw);
2241
2242     my $res = {
2243         digest => Digest::SHA::sha1_hex($raw),
2244         snapshots => {},
2245         pending => {},
2246     };
2247
2248     $filename =~ m|/qemu-server/(\d+)\.conf$|
2249         || die "got strange filename '$filename'";
2250
2251     my $vmid = $1;
2252
2253     my $conf = $res;
2254     my $descr;
2255     my $section = '';
2256
2257     my @lines = split(/\n/, $raw);
2258     foreach my $line (@lines) {
2259         next if $line =~ m/^\s*$/;
2260
2261         if ($line =~ m/^\[PENDING\]\s*$/i) {
2262             $section = 'pending';
2263             if (defined($descr)) {
2264                 $descr =~ s/\s+$//;
2265                 $conf->{description} = $descr;
2266             }
2267             $descr = undef;
2268             $conf = $res->{$section} = {};
2269             next;
2270
2271         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2272             $section = $1;
2273             if (defined($descr)) {
2274                 $descr =~ s/\s+$//;
2275                 $conf->{description} = $descr;
2276             }
2277             $descr = undef;
2278             $conf = $res->{snapshots}->{$section} = {};
2279             next;
2280         }
2281
2282         if ($line =~ m/^\#(.*)\s*$/) {
2283             $descr = '' if !defined($descr);
2284             $descr .= PVE::Tools::decode_text($1) . "\n";
2285             next;
2286         }
2287
2288         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2289             $descr = '' if !defined($descr);
2290             $descr .= PVE::Tools::decode_text($2);
2291         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2292             $conf->{snapstate} = $1;
2293         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2294             my $key = $1;
2295             my $value = $2;
2296             $conf->{$key} = $value;
2297         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2298             my $value = $1;
2299             if ($section eq 'pending') {
2300                 $conf->{delete} = $value; # we parse this later
2301             } else {
2302                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2303             }
2304         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2305             my $key = $1;
2306             my $value = $2;
2307             eval { $value = check_type($key, $value); };
2308             if ($@) {
2309                 warn "vm $vmid - unable to parse value of '$key' - $@";
2310             } else {
2311                 $key = 'ide2' if $key eq 'cdrom';
2312                 my $fmt = $confdesc->{$key}->{format};
2313                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2314                     my $v = parse_drive($key, $value);
2315                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2316                         $v->{file} = $volid;
2317                         $value = print_drive($v);
2318                     } else {
2319                         warn "vm $vmid - unable to parse value of '$key'\n";
2320                         next;
2321                     }
2322                 }
2323
2324                 $conf->{$key} = $value;
2325             }
2326         } else {
2327             warn "vm $vmid - unable to parse config: $line\n";
2328         }
2329     }
2330
2331     if (defined($descr)) {
2332         $descr =~ s/\s+$//;
2333         $conf->{description} = $descr;
2334     }
2335     delete $res->{snapstate}; # just to be sure
2336
2337     return $res;
2338 }
2339
2340 sub write_vm_config {
2341     my ($filename, $conf) = @_;
2342
2343     delete $conf->{snapstate}; # just to be sure
2344
2345     if ($conf->{cdrom}) {
2346         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2347         $conf->{ide2} = $conf->{cdrom};
2348         delete $conf->{cdrom};
2349     }
2350
2351     # we do not use 'smp' any longer
2352     if ($conf->{sockets}) {
2353         delete $conf->{smp};
2354     } elsif ($conf->{smp}) {
2355         $conf->{sockets} = $conf->{smp};
2356         delete $conf->{cores};
2357         delete $conf->{smp};
2358     }
2359
2360     my $used_volids = {};
2361
2362     my $cleanup_config = sub {
2363         my ($cref, $pending, $snapname) = @_;
2364
2365         foreach my $key (keys %$cref) {
2366             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2367                 $key eq 'snapstate' || $key eq 'pending';
2368             my $value = $cref->{$key};
2369             if ($key eq 'delete') {
2370                 die "propertry 'delete' is only allowed in [PENDING]\n"
2371                     if !$pending;
2372                 # fixme: check syntax?
2373                 next;
2374             }
2375             eval { $value = check_type($key, $value); };
2376             die "unable to parse value of '$key' - $@" if $@;
2377
2378             $cref->{$key} = $value;
2379
2380             if (!$snapname && is_valid_drivename($key)) {
2381                 my $drive = parse_drive($key, $value);
2382                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2383             }
2384         }
2385     };
2386
2387     &$cleanup_config($conf);
2388
2389     &$cleanup_config($conf->{pending}, 1);
2390
2391     foreach my $snapname (keys %{$conf->{snapshots}}) {
2392         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2393         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2394     }
2395
2396     # remove 'unusedX' settings if we re-add a volume
2397     foreach my $key (keys %$conf) {
2398         my $value = $conf->{$key};
2399         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2400             delete $conf->{$key};
2401         }
2402     }
2403
2404     my $generate_raw_config = sub {
2405         my ($conf, $pending) = @_;
2406
2407         my $raw = '';
2408
2409         # add description as comment to top of file
2410         if (defined(my $descr = $conf->{description})) {
2411             if ($descr) {
2412                 foreach my $cl (split(/\n/, $descr)) {
2413                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2414                 }
2415             } else {
2416                 $raw .= "#\n" if $pending;
2417             }
2418         }
2419
2420         foreach my $key (sort keys %$conf) {
2421             next if $key =~ /^(digest|description|pending|snapshots)$/;
2422             $raw .= "$key: $conf->{$key}\n";
2423         }
2424         return $raw;
2425     };
2426
2427     my $raw = &$generate_raw_config($conf);
2428
2429     if (scalar(keys %{$conf->{pending}})){
2430         $raw .= "\n[PENDING]\n";
2431         $raw .= &$generate_raw_config($conf->{pending}, 1);
2432     }
2433
2434     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2435         $raw .= "\n[$snapname]\n";
2436         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2437     }
2438
2439     return $raw;
2440 }
2441
2442 sub load_defaults {
2443
2444     my $res = {};
2445
2446     # we use static defaults from our JSON schema configuration
2447     foreach my $key (keys %$confdesc) {
2448         if (defined(my $default = $confdesc->{$key}->{default})) {
2449             $res->{$key} = $default;
2450         }
2451     }
2452
2453     return $res;
2454 }
2455
2456 sub config_list {
2457     my $vmlist = PVE::Cluster::get_vmlist();
2458     my $res = {};
2459     return $res if !$vmlist || !$vmlist->{ids};
2460     my $ids = $vmlist->{ids};
2461     my $nodename = nodename();
2462
2463     foreach my $vmid (keys %$ids) {
2464         my $d = $ids->{$vmid};
2465         next if !$d->{node} || $d->{node} ne $nodename;
2466         next if !$d->{type} || $d->{type} ne 'qemu';
2467         $res->{$vmid}->{exists} = 1;
2468     }
2469     return $res;
2470 }
2471
2472 # test if VM uses local resources (to prevent migration)
2473 sub check_local_resources {
2474     my ($conf, $noerr) = @_;
2475
2476     my @loc_res = ();
2477
2478     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2479     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2480
2481     push @loc_res, "ivshmem" if $conf->{ivshmem};
2482
2483     foreach my $k (keys %$conf) {
2484         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2485         # sockets are safe: they will recreated be on the target side post-migrate
2486         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2487         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2488     }
2489
2490     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2491
2492     return \@loc_res;
2493 }
2494
2495 # check if used storages are available on all nodes (use by migrate)
2496 sub check_storage_availability {
2497     my ($storecfg, $conf, $node) = @_;
2498
2499     PVE::QemuConfig->foreach_volume($conf, sub {
2500         my ($ds, $drive) = @_;
2501
2502         my $volid = $drive->{file};
2503         return if !$volid;
2504
2505         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2506         return if !$sid;
2507
2508         # check if storage is available on both nodes
2509         my $scfg = PVE::Storage::storage_check_enabled($storecfg, $sid);
2510         PVE::Storage::storage_check_enabled($storecfg, $sid, $node);
2511
2512         my ($vtype) = PVE::Storage::parse_volname($storecfg, $volid);
2513
2514         die "$volid: content type '$vtype' is not available on storage '$sid'\n"
2515             if !$scfg->{content}->{$vtype};
2516    });
2517 }
2518
2519 # list nodes where all VM images are available (used by has_feature API)
2520 sub shared_nodes {
2521     my ($conf, $storecfg) = @_;
2522
2523     my $nodelist = PVE::Cluster::get_nodelist();
2524     my $nodehash = { map { $_ => 1 } @$nodelist };
2525     my $nodename = nodename();
2526
2527     PVE::QemuConfig->foreach_volume($conf, sub {
2528         my ($ds, $drive) = @_;
2529
2530         my $volid = $drive->{file};
2531         return if !$volid;
2532
2533         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2534         if ($storeid) {
2535             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2536             if ($scfg->{disable}) {
2537                 $nodehash = {};
2538             } elsif (my $avail = $scfg->{nodes}) {
2539                 foreach my $node (keys %$nodehash) {
2540                     delete $nodehash->{$node} if !$avail->{$node};
2541                 }
2542             } elsif (!$scfg->{shared}) {
2543                 foreach my $node (keys %$nodehash) {
2544                     delete $nodehash->{$node} if $node ne $nodename
2545                 }
2546             }
2547         }
2548     });
2549
2550     return $nodehash
2551 }
2552
2553 sub check_local_storage_availability {
2554     my ($conf, $storecfg) = @_;
2555
2556     my $nodelist = PVE::Cluster::get_nodelist();
2557     my $nodehash = { map { $_ => {} } @$nodelist };
2558
2559     PVE::QemuConfig->foreach_volume($conf, sub {
2560         my ($ds, $drive) = @_;
2561
2562         my $volid = $drive->{file};
2563         return if !$volid;
2564
2565         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2566         if ($storeid) {
2567             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2568
2569             if ($scfg->{disable}) {
2570                 foreach my $node (keys %$nodehash) {
2571                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2572                 }
2573             } elsif (my $avail = $scfg->{nodes}) {
2574                 foreach my $node (keys %$nodehash) {
2575                     if (!$avail->{$node}) {
2576                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2577                     }
2578                 }
2579             }
2580         }
2581     });
2582
2583     foreach my $node (values %$nodehash) {
2584         if (my $unavail = $node->{unavailable_storages}) {
2585             $node->{unavailable_storages} = [ sort keys %$unavail ];
2586         }
2587     }
2588
2589     return $nodehash
2590 }
2591
2592 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2593 sub check_running {
2594     my ($vmid, $nocheck, $node) = @_;
2595
2596     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2597     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2598 }
2599
2600 sub vzlist {
2601
2602     my $vzlist = config_list();
2603
2604     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2605
2606     while (defined(my $de = $fd->read)) {
2607         next if $de !~ m/^(\d+)\.pid$/;
2608         my $vmid = $1;
2609         next if !defined($vzlist->{$vmid});
2610         if (my $pid = check_running($vmid)) {
2611             $vzlist->{$vmid}->{pid} = $pid;
2612         }
2613     }
2614
2615     return $vzlist;
2616 }
2617
2618 our $vmstatus_return_properties = {
2619     vmid => get_standard_option('pve-vmid'),
2620     status => {
2621         description => "Qemu process status.",
2622         type => 'string',
2623         enum => ['stopped', 'running'],
2624     },
2625     maxmem => {
2626         description => "Maximum memory in bytes.",
2627         type => 'integer',
2628         optional => 1,
2629         renderer => 'bytes',
2630     },
2631     maxdisk => {
2632         description => "Root disk size in bytes.",
2633         type => 'integer',
2634         optional => 1,
2635         renderer => 'bytes',
2636     },
2637     name => {
2638         description => "VM name.",
2639         type => 'string',
2640         optional => 1,
2641     },
2642     qmpstatus => {
2643         description => "Qemu QMP agent status.",
2644         type => 'string',
2645         optional => 1,
2646     },
2647     pid => {
2648         description => "PID of running qemu process.",
2649         type => 'integer',
2650         optional => 1,
2651     },
2652     uptime => {
2653         description => "Uptime.",
2654         type => 'integer',
2655         optional => 1,
2656         renderer => 'duration',
2657     },
2658     cpus => {
2659         description => "Maximum usable CPUs.",
2660         type => 'number',
2661         optional => 1,
2662     },
2663     lock => {
2664         description => "The current config lock, if any.",
2665         type => 'string',
2666         optional => 1,
2667     },
2668     tags => {
2669         description  => "The current configured tags, if any",
2670         type => 'string',
2671         optional => 1,
2672     },
2673     'running-machine' => {
2674         description => "The currently running machine type (if running).",
2675         type => 'string',
2676         optional => 1,
2677     },
2678     'running-qemu' => {
2679         description => "The currently running QEMU version (if running).",
2680         type => 'string',
2681         optional => 1,
2682     },
2683 };
2684
2685 my $last_proc_pid_stat;
2686
2687 # get VM status information
2688 # This must be fast and should not block ($full == false)
2689 # We only query KVM using QMP if $full == true (this can be slow)
2690 sub vmstatus {
2691     my ($opt_vmid, $full) = @_;
2692
2693     my $res = {};
2694
2695     my $storecfg = PVE::Storage::config();
2696
2697     my $list = vzlist();
2698     my $defaults = load_defaults();
2699
2700     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2701
2702     my $cpucount = $cpuinfo->{cpus} || 1;
2703
2704     foreach my $vmid (keys %$list) {
2705         next if $opt_vmid && ($vmid ne $opt_vmid);
2706
2707         my $conf = PVE::QemuConfig->load_config($vmid);
2708
2709         my $d = { vmid => int($vmid) };
2710         $d->{pid} = int($list->{$vmid}->{pid}) if $list->{$vmid}->{pid};
2711
2712         # fixme: better status?
2713         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2714
2715         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2716         if (defined($size)) {
2717             $d->{disk} = 0; # no info available
2718             $d->{maxdisk} = $size;
2719         } else {
2720             $d->{disk} = 0;
2721             $d->{maxdisk} = 0;
2722         }
2723
2724         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2725             * ($conf->{cores} || $defaults->{cores});
2726         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2727         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2728
2729         $d->{name} = $conf->{name} || "VM $vmid";
2730         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2731             : $defaults->{memory}*(1024*1024);
2732
2733         if ($conf->{balloon}) {
2734             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2735             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2736                 : $defaults->{shares};
2737         }
2738
2739         $d->{uptime} = 0;
2740         $d->{cpu} = 0;
2741         $d->{mem} = 0;
2742
2743         $d->{netout} = 0;
2744         $d->{netin} = 0;
2745
2746         $d->{diskread} = 0;
2747         $d->{diskwrite} = 0;
2748
2749         $d->{template} = 1 if PVE::QemuConfig->is_template($conf);
2750
2751         $d->{serial} = 1 if conf_has_serial($conf);
2752         $d->{lock} = $conf->{lock} if $conf->{lock};
2753         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2754
2755         $res->{$vmid} = $d;
2756     }
2757
2758     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2759     foreach my $dev (keys %$netdev) {
2760         next if $dev !~ m/^tap([1-9]\d*)i/;
2761         my $vmid = $1;
2762         my $d = $res->{$vmid};
2763         next if !$d;
2764
2765         $d->{netout} += $netdev->{$dev}->{receive};
2766         $d->{netin} += $netdev->{$dev}->{transmit};
2767
2768         if ($full) {
2769             $d->{nics}->{$dev}->{netout} = int($netdev->{$dev}->{receive});
2770             $d->{nics}->{$dev}->{netin} = int($netdev->{$dev}->{transmit});
2771         }
2772
2773     }
2774
2775     my $ctime = gettimeofday;
2776
2777     foreach my $vmid (keys %$list) {
2778
2779         my $d = $res->{$vmid};
2780         my $pid = $d->{pid};
2781         next if !$pid;
2782
2783         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2784         next if !$pstat; # not running
2785
2786         my $used = $pstat->{utime} + $pstat->{stime};
2787
2788         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2789
2790         if ($pstat->{vsize}) {
2791             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2792         }
2793
2794         my $old = $last_proc_pid_stat->{$pid};
2795         if (!$old) {
2796             $last_proc_pid_stat->{$pid} = {
2797                 time => $ctime,
2798                 used => $used,
2799                 cpu => 0,
2800             };
2801             next;
2802         }
2803
2804         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2805
2806         if ($dtime > 1000) {
2807             my $dutime = $used -  $old->{used};
2808
2809             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2810             $last_proc_pid_stat->{$pid} = {
2811                 time => $ctime,
2812                 used => $used,
2813                 cpu => $d->{cpu},
2814             };
2815         } else {
2816             $d->{cpu} = $old->{cpu};
2817         }
2818     }
2819
2820     return $res if !$full;
2821
2822     my $qmpclient = PVE::QMPClient->new();
2823
2824     my $ballooncb = sub {
2825         my ($vmid, $resp) = @_;
2826
2827         my $info = $resp->{'return'};
2828         return if !$info->{max_mem};
2829
2830         my $d = $res->{$vmid};
2831
2832         # use memory assigned to VM
2833         $d->{maxmem} = $info->{max_mem};
2834         $d->{balloon} = $info->{actual};
2835
2836         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2837             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2838             $d->{freemem} = $info->{free_mem};
2839         }
2840
2841         $d->{ballooninfo} = $info;
2842     };
2843
2844     my $blockstatscb = sub {
2845         my ($vmid, $resp) = @_;
2846         my $data = $resp->{'return'} || [];
2847         my $totalrdbytes = 0;
2848         my $totalwrbytes = 0;
2849
2850         for my $blockstat (@$data) {
2851             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2852             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2853
2854             $blockstat->{device} =~ s/drive-//;
2855             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2856         }
2857         $res->{$vmid}->{diskread} = $totalrdbytes;
2858         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2859     };
2860
2861     my $machinecb = sub {
2862         my ($vmid, $resp) = @_;
2863         my $data = $resp->{'return'} || [];
2864
2865         $res->{$vmid}->{'running-machine'} =
2866             PVE::QemuServer::Machine::current_from_query_machines($data);
2867     };
2868
2869     my $versioncb = sub {
2870         my ($vmid, $resp) = @_;
2871         my $data = $resp->{'return'} // {};
2872         my $version = 'unknown';
2873
2874         if (my $v = $data->{qemu}) {
2875             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2876         }
2877
2878         $res->{$vmid}->{'running-qemu'} = $version;
2879     };
2880
2881     my $statuscb = sub {
2882         my ($vmid, $resp) = @_;
2883
2884         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2885         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2886         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2887         # this fails if ballon driver is not loaded, so this must be
2888         # the last commnand (following command are aborted if this fails).
2889         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2890
2891         my $status = 'unknown';
2892         if (!defined($status = $resp->{'return'}->{status})) {
2893             warn "unable to get VM status\n";
2894             return;
2895         }
2896
2897         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2898     };
2899
2900     foreach my $vmid (keys %$list) {
2901         next if $opt_vmid && ($vmid ne $opt_vmid);
2902         next if !$res->{$vmid}->{pid}; # not running
2903         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2904     }
2905
2906     $qmpclient->queue_execute(undef, 2);
2907
2908     foreach my $vmid (keys %$list) {
2909         next if $opt_vmid && ($vmid ne $opt_vmid);
2910         next if !$res->{$vmid}->{pid}; #not running
2911
2912         # we can't use the $qmpclient since it might have already aborted on
2913         # 'query-balloon', but this might also fail for older versions...
2914         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2915         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2916     }
2917
2918     foreach my $vmid (keys %$list) {
2919         next if $opt_vmid && ($vmid ne $opt_vmid);
2920         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2921     }
2922
2923     return $res;
2924 }
2925
2926 sub conf_has_serial {
2927     my ($conf) = @_;
2928
2929     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2930         if ($conf->{"serial$i"}) {
2931             return 1;
2932         }
2933     }
2934
2935     return 0;
2936 }
2937
2938 sub conf_has_audio {
2939     my ($conf, $id) = @_;
2940
2941     $id //= 0;
2942     my $audio = $conf->{"audio$id"};
2943     return if !defined($audio);
2944
2945     my $audioproperties = parse_property_string($audio_fmt, $audio);
2946     my $audiodriver = $audioproperties->{driver} // 'spice';
2947
2948     return {
2949         dev => $audioproperties->{device},
2950         dev_id => "audiodev$id",
2951         backend => $audiodriver,
2952         backend_id => "$audiodriver-backend${id}",
2953     };
2954 }
2955
2956 sub audio_devs {
2957     my ($audio, $audiopciaddr, $machine_version) = @_;
2958
2959     my $devs = [];
2960
2961     my $id = $audio->{dev_id};
2962     my $audiodev = "";
2963     if (min_version($machine_version, 4, 2)) {
2964         $audiodev = ",audiodev=$audio->{backend_id}";
2965     }
2966
2967     if ($audio->{dev} eq 'AC97') {
2968         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2969     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2970         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2971         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2972         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2973     } else {
2974         die "unkown audio device '$audio->{dev}', implement me!";
2975     }
2976
2977     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2978
2979     return $devs;
2980 }
2981
2982 sub get_tpm_paths {
2983     my ($vmid) = @_;
2984     return {
2985         socket => "/var/run/qemu-server/$vmid.swtpm",
2986         pid => "/var/run/qemu-server/$vmid.swtpm.pid",
2987     };
2988 }
2989
2990 sub add_tpm_device {
2991     my ($vmid, $devices, $conf) = @_;
2992
2993     return if !$conf->{tpmstate0};
2994
2995     my $paths = get_tpm_paths($vmid);
2996
2997     push @$devices, "-chardev", "socket,id=tpmchar,path=$paths->{socket}";
2998     push @$devices, "-tpmdev", "emulator,id=tpmdev,chardev=tpmchar";
2999     push @$devices, "-device", "tpm-tis,tpmdev=tpmdev";
3000 }
3001
3002 sub start_swtpm {
3003     my ($storecfg, $vmid, $tpmdrive, $migration) = @_;
3004
3005     return if !$tpmdrive;
3006
3007     my $state;
3008     my $tpm = parse_drive("tpmstate0", $tpmdrive);
3009     my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
3010     if ($storeid) {
3011         $state = PVE::Storage::map_volume($storecfg, $tpm->{file});
3012     } else {
3013         $state = $tpm->{file};
3014     }
3015
3016     my $paths = get_tpm_paths($vmid);
3017
3018     # during migration, we will get state from remote
3019     #
3020     if (!$migration) {
3021         # run swtpm_setup to create a new TPM state if it doesn't exist yet
3022         my $setup_cmd = [
3023             "swtpm_setup",
3024             "--tpmstate",
3025             "file://$state",
3026             "--createek",
3027             "--create-ek-cert",
3028             "--create-platform-cert",
3029             "--lock-nvram",
3030             "--config",
3031             "/etc/swtpm_setup.conf", # do not use XDG configs
3032             "--runas",
3033             "0", # force creation as root, error if not possible
3034             "--not-overwrite", # ignore existing state, do not modify
3035         ];
3036
3037         push @$setup_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3038         # TPM 2.0 supports ECC crypto, use if possible
3039         push @$setup_cmd, "--ecc" if $tpm->{version} eq 'v2.0';
3040
3041         run_command($setup_cmd, outfunc => sub {
3042             print "swtpm_setup: $1\n";
3043         });
3044     }
3045
3046     my $emulator_cmd = [
3047         "swtpm",
3048         "socket",
3049         "--tpmstate",
3050         "backend-uri=file://$state,mode=0600",
3051         "--ctrl",
3052         "type=unixio,path=$paths->{socket},mode=0600",
3053         "--pid",
3054         "file=$paths->{pid}",
3055         "--terminate", # terminate on QEMU disconnect
3056         "--daemon",
3057     ];
3058     push @$emulator_cmd, "--tpm2" if $tpm->{version} eq 'v2.0';
3059     run_command($emulator_cmd, outfunc => sub { print $1; });
3060
3061     # swtpm may take a bit to start before daemonizing, wait up to 5s for pid
3062     my $tries = 100;
3063     while (! -e $paths->{pid}) {
3064         usleep(50000);
3065         die "failed to start swtpm: pid file '$paths->{pid}' wasn't created.\n"
3066             if --$tries == 0;
3067     }
3068
3069     # return untainted PID of swtpm daemon so it can be killed on error
3070     file_read_firstline($paths->{pid}) =~ m/(\d+)/;
3071     return $1;
3072 }
3073
3074 sub vga_conf_has_spice {
3075     my ($vga) = @_;
3076
3077     my $vgaconf = parse_vga($vga);
3078     my $vgatype = $vgaconf->{type};
3079     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3080
3081     return $1 || 1;
3082 }
3083
3084 sub is_native($) {
3085     my ($arch) = @_;
3086     return get_host_arch() eq $arch;
3087 }
3088
3089 sub get_vm_arch {
3090     my ($conf) = @_;
3091     return $conf->{arch} // get_host_arch();
3092 }
3093
3094 my $default_machines = {
3095     x86_64 => 'pc',
3096     aarch64 => 'virt',
3097 };
3098
3099 sub get_installed_machine_version {
3100     my ($kvmversion) = @_;
3101     $kvmversion = kvm_user_version() if !defined($kvmversion);
3102     $kvmversion =~ m/^(\d+\.\d+)/;
3103     return $1;
3104 }
3105
3106 sub windows_get_pinned_machine_version {
3107     my ($machine, $base_version, $kvmversion) = @_;
3108
3109     my $pin_version = $base_version;
3110     if (!defined($base_version) ||
3111         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
3112     ) {
3113         $pin_version = get_installed_machine_version($kvmversion);
3114     }
3115     if (!$machine || $machine eq 'pc') {
3116         $machine = "pc-i440fx-$pin_version";
3117     } elsif ($machine eq 'q35') {
3118         $machine = "pc-q35-$pin_version";
3119     } elsif ($machine eq 'virt') {
3120         $machine = "virt-$pin_version";
3121     } else {
3122         warn "unknown machine type '$machine', not touching that!\n";
3123     }
3124
3125     return $machine;
3126 }
3127
3128 sub get_vm_machine {
3129     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
3130
3131     my $machine = $forcemachine || $conf->{machine};
3132
3133     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
3134         $kvmversion //= kvm_user_version();
3135         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
3136         # layout which confuses windows quite a bit and may result in various regressions..
3137         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
3138         if (windows_version($conf->{ostype})) {
3139             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
3140         }
3141         $arch //= 'x86_64';
3142         $machine ||= $default_machines->{$arch};
3143         if ($add_pve_version) {
3144             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
3145             $machine .= "+pve$pvever";
3146         }
3147     }
3148
3149     if ($add_pve_version && $machine !~ m/\+pve\d+?(?:\.pxe)?$/) {
3150         my $is_pxe = $machine =~ m/^(.*?)\.pxe$/;
3151         $machine = $1 if $is_pxe;
3152
3153         # for version-pinned machines that do not include a pve-version (e.g.
3154         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
3155         $machine .= '+pve0';
3156
3157         $machine .= '.pxe' if $is_pxe;
3158     }
3159
3160     return $machine;
3161 }
3162
3163 sub get_ovmf_files($$) {
3164     my ($arch, $efidisk) = @_;
3165
3166     my $types = $OVMF->{$arch}
3167         or die "no OVMF images known for architecture '$arch'\n";
3168
3169     my $type = 'default';
3170     if (defined($efidisk->{efitype}) && $efidisk->{efitype} eq '4m') {
3171         $type = $efidisk->{'pre-enrolled-keys'} ? "4m-ms" : "4m";
3172     }
3173
3174     return $types->{$type}->@*;
3175 }
3176
3177 my $Arch2Qemu = {
3178     aarch64 => '/usr/bin/qemu-system-aarch64',
3179     x86_64 => '/usr/bin/qemu-system-x86_64',
3180 };
3181 sub get_command_for_arch($) {
3182     my ($arch) = @_;
3183     return '/usr/bin/kvm' if is_native($arch);
3184
3185     my $cmd = $Arch2Qemu->{$arch}
3186         or die "don't know how to emulate architecture '$arch'\n";
3187     return $cmd;
3188 }
3189
3190 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
3191 # to use in a QEMU command line (-cpu element), first array_intersect the result
3192 # of query_supported_ with query_understood_. This is necessary because:
3193 #
3194 # a) query_understood_ returns flags the host cannot use and
3195 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
3196 #    flags, but CPU settings - with most of them being flags. Those settings
3197 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
3198 #
3199 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
3200 # expensive. If you need the value returned from this, you can get it much
3201 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
3202 # $accel being 'kvm' or 'tcg'.
3203 #
3204 # pvestatd calls this function on startup and whenever the QEMU/KVM version
3205 # changes, automatically populating pmxcfs.
3206 #
3207 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
3208 # since kvm and tcg machines support different flags
3209 #
3210 sub query_supported_cpu_flags {
3211     my ($arch) = @_;
3212
3213     $arch //= get_host_arch();
3214     my $default_machine = $default_machines->{$arch};
3215
3216     my $flags = {};
3217
3218     # FIXME: Once this is merged, the code below should work for ARM as well:
3219     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3220     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3221         $arch eq "aarch64";
3222
3223     my $kvm_supported = defined(kvm_version());
3224     my $qemu_cmd = get_command_for_arch($arch);
3225     my $fakevmid = -1;
3226     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3227
3228     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3229     my $query_supported_run_qemu = sub {
3230         my ($kvm) = @_;
3231
3232         my $flags = {};
3233         my $cmd = [
3234             $qemu_cmd,
3235             '-machine', $default_machine,
3236             '-display', 'none',
3237             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server=on,wait=off",
3238             '-mon', 'chardev=qmp,mode=control',
3239             '-pidfile', $pidfile,
3240             '-S', '-daemonize'
3241         ];
3242
3243         if (!$kvm) {
3244             push @$cmd, '-accel', 'tcg';
3245         }
3246
3247         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3248         die "QEMU flag querying VM exited with code " . $rc if $rc;
3249
3250         eval {
3251             my $cmd_result = mon_cmd(
3252                 $fakevmid,
3253                 'query-cpu-model-expansion',
3254                 type => 'full',
3255                 model => { name => 'host' }
3256             );
3257
3258             my $props = $cmd_result->{model}->{props};
3259             foreach my $prop (keys %$props) {
3260                 next if $props->{$prop} ne '1';
3261                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3262                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3263                 # We only keep those with underscores, to match /proc/cpuinfo
3264                 $prop =~ s/\.|-/_/g;
3265                 $flags->{$prop} = 1;
3266             }
3267         };
3268         my $err = $@;
3269
3270         # force stop with 10 sec timeout and 'nocheck'
3271         # always stop, even if QMP failed
3272         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3273
3274         die $err if $err;
3275
3276         return [ sort keys %$flags ];
3277     };
3278
3279     # We need to query QEMU twice, since KVM and TCG have different supported flags
3280     PVE::QemuConfig->lock_config($fakevmid, sub {
3281         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3282         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3283
3284         if ($kvm_supported) {
3285             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3286             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3287         }
3288     });
3289
3290     return $flags;
3291 }
3292
3293 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3294 my $understood_cpu_flag_dir = "/usr/share/kvm";
3295 sub query_understood_cpu_flags {
3296     my $arch = get_host_arch();
3297     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3298
3299     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3300         if ! -e $filepath;
3301
3302     my $raw = file_get_contents($filepath);
3303     $raw =~ s/^\s+|\s+$//g;
3304     my @flags = split(/\s+/, $raw);
3305
3306     return \@flags;
3307 }
3308
3309 my sub get_cpuunits {
3310     my ($conf) = @_;
3311     return $conf->{cpuunits} // (PVE::CGroup::cgroup_mode() == 2 ? 100 : 1024);
3312 }
3313 sub config_to_command {
3314     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu,
3315         $pbs_backing) = @_;
3316
3317     my $cmd = [];
3318     my $globalFlags = [];
3319     my $machineFlags = [];
3320     my $rtcFlags = [];
3321     my $devices = [];
3322     my $pciaddr = '';
3323     my $bridges = {};
3324     my $ostype = $conf->{ostype};
3325     my $winversion = windows_version($ostype);
3326     my $kvm = $conf->{kvm};
3327     my $nodename = nodename();
3328
3329     my $arch = get_vm_arch($conf);
3330     my $kvm_binary = get_command_for_arch($arch);
3331     my $kvmver = kvm_user_version($kvm_binary);
3332
3333     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3334         $kvmver //= "undefined";
3335         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3336     }
3337
3338     my $add_pve_version = min_version($kvmver, 4, 1);
3339
3340     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3341     my $machine_version = extract_version($machine_type, $kvmver);
3342     $kvm //= 1 if is_native($arch);
3343
3344     $machine_version =~ m/(\d+)\.(\d+)/;
3345     my ($machine_major, $machine_minor) = ($1, $2);
3346
3347     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3348         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3349     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3350         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3351             ." please upgrade node '$nodename'\n"
3352     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3353         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3354         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3355             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3356             ." node '$nodename'\n";
3357     }
3358
3359     # if a specific +pve version is required for a feature, use $version_guard
3360     # instead of min_version to allow machines to be run with the minimum
3361     # required version
3362     my $required_pve_version = 0;
3363     my $version_guard = sub {
3364         my ($major, $minor, $pve) = @_;
3365         return 0 if !min_version($machine_version, $major, $minor, $pve);
3366         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3367         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3368         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3369         return 1;
3370     };
3371
3372     if ($kvm && !defined kvm_version()) {
3373         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3374             ." or enable in BIOS.\n";
3375     }
3376
3377     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3378     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3379     my $use_old_bios_files = undef;
3380     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3381
3382     my $cpuunits = get_cpuunits($conf);
3383
3384     push @$cmd, $kvm_binary;
3385
3386     push @$cmd, '-id', $vmid;
3387
3388     my $vmname = $conf->{name} || "vm$vmid";
3389
3390     push @$cmd, '-name', $vmname;
3391
3392     push @$cmd, '-no-shutdown';
3393
3394     my $use_virtio = 0;
3395
3396     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3397     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server=on,wait=off";
3398     push @$cmd, '-mon', "chardev=qmp,mode=control";
3399
3400     if (min_version($machine_version, 2, 12)) {
3401         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3402         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3403     }
3404
3405     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3406
3407     push @$cmd, '-daemonize';
3408
3409     if ($conf->{smbios1}) {
3410         my $smbios_conf = parse_smbios1($conf->{smbios1});
3411         if ($smbios_conf->{base64}) {
3412             # Do not pass base64 flag to qemu
3413             delete $smbios_conf->{base64};
3414             my $smbios_string = "";
3415             foreach my $key (keys %$smbios_conf) {
3416                 my $value;
3417                 if ($key eq "uuid") {
3418                     $value = $smbios_conf->{uuid}
3419                 } else {
3420                     $value = decode_base64($smbios_conf->{$key});
3421                 }
3422                 # qemu accepts any binary data, only commas need escaping by double comma
3423                 $value =~ s/,/,,/g;
3424                 $smbios_string .= "," . $key . "=" . $value if $value;
3425             }
3426             push @$cmd, '-smbios', "type=1" . $smbios_string;
3427         } else {
3428             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3429         }
3430     }
3431
3432     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3433         my $d;
3434         if (my $efidisk = $conf->{efidisk0}) {
3435             $d = parse_drive('efidisk0', $efidisk);
3436         }
3437
3438         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch, $d);
3439         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3440
3441         my ($path, $format);
3442         my $read_only_str = '';
3443         if ($d) {
3444             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3445             $format = $d->{format};
3446             if ($storeid) {
3447                 $path = PVE::Storage::path($storecfg, $d->{file});
3448                 if (!defined($format)) {
3449                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3450                     $format = qemu_img_format($scfg, $volname);
3451                 }
3452             } else {
3453                 $path = $d->{file};
3454                 die "efidisk format must be specified\n"
3455                     if !defined($format);
3456             }
3457
3458             $read_only_str = ',readonly=on' if drive_is_read_only($conf, $d);
3459         } else {
3460             warn "no efidisk configured! Using temporary efivars disk.\n";
3461             $path = "/tmp/$vmid-ovmf.fd";
3462             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3463             $format = 'raw';
3464         }
3465
3466         my $size_str = "";
3467
3468         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3469             $size_str = ",size=" . (-s $ovmf_vars);
3470         }
3471
3472         # SPI flash does lots of read-modify-write OPs, without writeback this gets really slow #3329
3473         my $cache = "";
3474         if ($path =~ m/^rbd:/) {
3475                 $cache = ',cache=writeback';
3476                 $path .= ':rbd_cache_policy=writeback'; # avoid write-around, we *need* to cache writes too
3477         }
3478
3479         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly=on,file=$ovmf_code";
3480         push @$cmd, '-drive', "if=pflash,unit=1$cache,format=$format,id=drive-efidisk0$size_str,file=${path}${read_only_str}";
3481     }
3482
3483     # load q35 config
3484     if ($q35) {
3485         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3486         if (min_version($machine_version, 4, 0)) {
3487             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3488         } else {
3489             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3490         }
3491     }
3492
3493     if ($conf->{vmgenid}) {
3494         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3495     }
3496
3497     # add usb controllers
3498     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3499         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3500     push @$devices, @usbcontrollers if @usbcontrollers;
3501     my $vga = parse_vga($conf->{vga});
3502
3503     my $qxlnum = vga_conf_has_spice($conf->{vga});
3504     $vga->{type} = 'qxl' if $qxlnum;
3505
3506     if (!$vga->{type}) {
3507         if ($arch eq 'aarch64') {
3508             $vga->{type} = 'virtio';
3509         } elsif (min_version($machine_version, 2, 9)) {
3510             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3511         } else {
3512             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3513         }
3514     }
3515
3516     # enable absolute mouse coordinates (needed by vnc)
3517     my $tablet;
3518     if (defined($conf->{tablet})) {
3519         $tablet = $conf->{tablet};
3520     } else {
3521         $tablet = $defaults->{tablet};
3522         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3523         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3524     }
3525
3526     if ($tablet) {
3527         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3528         my $kbd = print_keyboarddevice_full($conf, $arch);
3529         push @$devices, '-device', $kbd if defined($kbd);
3530     }
3531
3532     my $bootorder = device_bootorder($conf);
3533
3534     # host pci device passthrough
3535     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3536         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3537
3538     # usb devices
3539     my $usb_dev_features = {};
3540     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3541
3542     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3543         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3544     push @$devices, @usbdevices if @usbdevices;
3545
3546     # serial devices
3547     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3548         if (my $path = $conf->{"serial$i"}) {
3549             if ($path eq 'socket') {
3550                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3551                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server=on,wait=off";
3552                 # On aarch64, serial0 is the UART device. Qemu only allows
3553                 # connecting UART devices via the '-serial' command line, as
3554                 # the device has a fixed slot on the hardware...
3555                 if ($arch eq 'aarch64' && $i == 0) {
3556                     push @$devices, '-serial', "chardev:serial$i";
3557                 } else {
3558                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3559                 }
3560             } else {
3561                 die "no such serial device\n" if ! -c $path;
3562                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3563                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3564             }
3565         }
3566     }
3567
3568     # parallel devices
3569     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3570         if (my $path = $conf->{"parallel$i"}) {
3571             die "no such parallel device\n" if ! -c $path;
3572             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3573             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3574             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3575         }
3576     }
3577
3578     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3579         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3580         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3581         push @$devices, @$audio_devs;
3582     }
3583
3584     add_tpm_device($vmid, $devices, $conf);
3585
3586     my $sockets = 1;
3587     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3588     $sockets = $conf->{sockets} if  $conf->{sockets};
3589
3590     my $cores = $conf->{cores} || 1;
3591
3592     my $maxcpus = $sockets * $cores;
3593
3594     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3595
3596     my $allowed_vcpus = $cpuinfo->{cpus};
3597
3598     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3599         if ($allowed_vcpus < $maxcpus);
3600
3601     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3602
3603         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3604         for (my $i = 2; $i <= $vcpus; $i++)  {
3605             my $cpustr = print_cpu_device($conf,$i);
3606             push @$cmd, '-device', $cpustr;
3607         }
3608
3609     } else {
3610
3611         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3612     }
3613     push @$cmd, '-nodefaults';
3614
3615     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3616
3617     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3618
3619     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3620
3621     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3622         push @$devices, '-device', print_vga_device(
3623             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3624         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3625         push @$cmd,  '-vnc', "unix:$socket,password=on";
3626     } else {
3627         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3628         push @$cmd, '-nographic';
3629     }
3630
3631     # time drift fix
3632     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3633     my $useLocaltime = $conf->{localtime};
3634
3635     if ($winversion >= 5) { # windows
3636         $useLocaltime = 1 if !defined($conf->{localtime});
3637
3638         # use time drift fix when acpi is enabled
3639         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3640             $tdf = 1 if !defined($conf->{tdf});
3641         }
3642     }
3643
3644     if ($winversion >= 6) {
3645         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3646         push @$cmd, '-no-hpet';
3647     }
3648
3649     push @$rtcFlags, 'driftfix=slew' if $tdf;
3650
3651     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3652         push @$rtcFlags, "base=$conf->{startdate}";
3653     } elsif ($useLocaltime) {
3654         push @$rtcFlags, 'base=localtime';
3655     }
3656
3657     if ($forcecpu) {
3658         push @$cmd, '-cpu', $forcecpu;
3659     } else {
3660         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3661     }
3662
3663     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3664
3665     push @$cmd, '-S' if $conf->{freeze};
3666
3667     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3668
3669     my $guest_agent = parse_guest_agent($conf);
3670
3671     if ($guest_agent->{enabled}) {
3672         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3673         push @$devices, '-chardev', "socket,path=$qgasocket,server=on,wait=off,id=qga0";
3674
3675         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3676             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3677             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3678             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3679         } elsif ($guest_agent->{type} eq 'isa') {
3680             push @$devices, '-device', "isa-serial,chardev=qga0";
3681         }
3682     }
3683
3684     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3685     if ($rng && $version_guard->(4, 1, 2)) {
3686         check_rng_source($rng->{source});
3687
3688         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3689         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3690         my $limiter_str = "";
3691         if ($max_bytes) {
3692             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3693         }
3694
3695         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3696         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3697         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3698     }
3699
3700     my $spice_port;
3701
3702     if ($qxlnum) {
3703         if ($qxlnum > 1) {
3704             if ($winversion){
3705                 for (my $i = 1; $i < $qxlnum; $i++){
3706                     push @$devices, '-device', print_vga_device(
3707                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3708                 }
3709             } else {
3710                 # assume other OS works like Linux
3711                 my ($ram, $vram) = ("134217728", "67108864");
3712                 if ($vga->{memory}) {
3713                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3714                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3715                 }
3716                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3717                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3718             }
3719         }
3720
3721         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3722
3723         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3724         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3725         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3726
3727         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3728         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3729         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3730
3731         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3732         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3733
3734         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3735         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3736         if ($spice_enhancement->{foldersharing}) {
3737             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3738             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3739         }
3740
3741         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3742         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3743             if $spice_enhancement->{videostreaming};
3744
3745         push @$devices, '-spice', "$spice_opts";
3746     }
3747
3748     # enable balloon by default, unless explicitly disabled
3749     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3750         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3751         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3752     }
3753
3754     if ($conf->{watchdog}) {
3755         my $wdopts = parse_watchdog($conf->{watchdog});
3756         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3757         my $watchdog = $wdopts->{model} || 'i6300esb';
3758         push @$devices, '-device', "$watchdog$pciaddr";
3759         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3760     }
3761
3762     my $vollist = [];
3763     my $scsicontroller = {};
3764     my $ahcicontroller = {};
3765     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3766
3767     # Add iscsi initiator name if available
3768     if (my $initiator = get_initiator_name()) {
3769         push @$devices, '-iscsi', "initiator-name=$initiator";
3770     }
3771
3772     PVE::QemuConfig->foreach_volume($conf, sub {
3773         my ($ds, $drive) = @_;
3774
3775         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3776             check_volume_storage_type($storecfg, $drive->{file});
3777             push @$vollist, $drive->{file};
3778         }
3779
3780         # ignore efidisk here, already added in bios/fw handling code above
3781         return if $drive->{interface} eq 'efidisk';
3782         # similar for TPM
3783         return if $drive->{interface} eq 'tpmstate';
3784
3785         $use_virtio = 1 if $ds =~ m/^virtio/;
3786
3787         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3788
3789         if ($drive->{interface} eq 'virtio'){
3790            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3791         }
3792
3793         if ($drive->{interface} eq 'scsi') {
3794
3795             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3796
3797             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3798                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3799
3800             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3801             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3802
3803             my $iothread = '';
3804             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3805                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3806                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3807             } elsif ($drive->{iothread}) {
3808                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3809             }
3810
3811             my $queues = '';
3812             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3813                 $queues = ",num_queues=$drive->{queues}";
3814             }
3815
3816             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3817                 if !$scsicontroller->{$controller};
3818             $scsicontroller->{$controller}=1;
3819         }
3820
3821         if ($drive->{interface} eq 'sata') {
3822             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3823             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3824             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3825                 if !$ahcicontroller->{$controller};
3826             $ahcicontroller->{$controller}=1;
3827         }
3828
3829         my $pbs_conf = $pbs_backing->{$ds};
3830         my $pbs_name = undef;
3831         if ($pbs_conf) {
3832             $pbs_name = "drive-$ds-pbs";
3833             push @$devices, '-blockdev', print_pbs_blockdev($pbs_conf, $pbs_name);
3834         }
3835
3836         my $drive_cmd = print_drive_commandline_full(
3837             $storecfg, $vmid, $drive, $pbs_name, min_version($kvmver, 6, 0));
3838
3839         # extra protection for templates, but SATA and IDE don't support it..
3840         $drive_cmd .= ',readonly=on' if drive_is_read_only($conf, $drive);
3841
3842         push @$devices, '-drive',$drive_cmd;
3843         push @$devices, '-device', print_drivedevice_full(
3844             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3845     });
3846
3847     for (my $i = 0; $i < $MAX_NETS; $i++) {
3848         my $netname = "net$i";
3849
3850         next if !$conf->{$netname};
3851         my $d = parse_net($conf->{$netname});
3852         next if !$d;
3853
3854         $use_virtio = 1 if $d->{model} eq 'virtio';
3855
3856         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3857
3858         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3859         push @$devices, '-netdev', $netdevfull;
3860
3861         my $netdevicefull = print_netdevice_full(
3862             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3863
3864         push @$devices, '-device', $netdevicefull;
3865     }
3866
3867     if ($conf->{ivshmem}) {
3868         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3869
3870         my $bus;
3871         if ($q35) {
3872             $bus = print_pcie_addr("ivshmem");
3873         } else {
3874             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3875         }
3876
3877         my $ivshmem_name = $ivshmem->{name} // $vmid;
3878         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3879
3880         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3881         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3882             .",size=$ivshmem->{size}M";
3883     }
3884
3885     # pci.4 is nested in pci.1
3886     $bridges->{1} = 1 if $bridges->{4};
3887
3888     if (!$q35) {
3889         # add pci bridges
3890         if (min_version($machine_version, 2, 3)) {
3891            $bridges->{1} = 1;
3892            $bridges->{2} = 1;
3893         }
3894
3895         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3896
3897     }
3898
3899     for my $k (sort {$b cmp $a} keys %$bridges) {
3900         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3901
3902         my $k_name = $k;
3903         if ($k == 2 && $legacy_igd) {
3904             $k_name = "$k-igd";
3905         }
3906         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3907
3908         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3909         if ($q35) {
3910             # add after -readconfig pve-q35.cfg
3911             splice @$devices, 2, 0, '-device', $devstr;
3912         } else {
3913             unshift @$devices, '-device', $devstr if $k > 0;
3914         }
3915     }
3916
3917     if (!$kvm) {
3918         push @$machineFlags, 'accel=tcg';
3919     }
3920
3921     my $machine_type_min = $machine_type;
3922     if ($add_pve_version) {
3923         $machine_type_min =~ s/\+pve\d+$//;
3924         $machine_type_min .= "+pve$required_pve_version";
3925     }
3926     push @$machineFlags, "type=${machine_type_min}";
3927
3928     push @$cmd, @$devices;
3929     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3930     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3931     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3932
3933     if (my $vmstate = $conf->{vmstate}) {
3934         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3935         push @$vollist, $vmstate;
3936         push @$cmd, '-loadstate', $statepath;
3937         print "activating and using '$vmstate' as vmstate\n";
3938     }
3939
3940     if (PVE::QemuConfig->is_template($conf)) {
3941         # needed to workaround base volumes being read-only
3942         push @$cmd, '-snapshot';
3943     }
3944
3945     # add custom args
3946     if ($conf->{args}) {
3947         my $aa = PVE::Tools::split_args($conf->{args});
3948         push @$cmd, @$aa;
3949     }
3950
3951     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3952 }
3953
3954 sub check_rng_source {
3955     my ($source) = @_;
3956
3957     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3958     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3959         if ! -e $source;
3960
3961     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3962     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3963         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3964         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3965         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3966             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3967             ." to the host.\n";
3968     }
3969 }
3970
3971 sub spice_port {
3972     my ($vmid) = @_;
3973
3974     my $res = mon_cmd($vmid, 'query-spice');
3975
3976     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3977 }
3978
3979 sub vm_devices_list {
3980     my ($vmid) = @_;
3981
3982     my $res = mon_cmd($vmid, 'query-pci');
3983     my $devices_to_check = [];
3984     my $devices = {};
3985     foreach my $pcibus (@$res) {
3986         push @$devices_to_check, @{$pcibus->{devices}},
3987     }
3988
3989     while (@$devices_to_check) {
3990         my $to_check = [];
3991         for my $d (@$devices_to_check) {
3992             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3993             next if !$d->{'pci_bridge'};
3994
3995             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3996             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3997         }
3998         $devices_to_check = $to_check;
3999     }
4000
4001     my $resblock = mon_cmd($vmid, 'query-block');
4002     foreach my $block (@$resblock) {
4003         if($block->{device} =~ m/^drive-(\S+)/){
4004                 $devices->{$1} = 1;
4005         }
4006     }
4007
4008     my $resmice = mon_cmd($vmid, 'query-mice');
4009     foreach my $mice (@$resmice) {
4010         if ($mice->{name} eq 'QEMU HID Tablet') {
4011             $devices->{tablet} = 1;
4012             last;
4013         }
4014     }
4015
4016     # for usb devices there is no query-usb
4017     # but we can iterate over the entries in
4018     # qom-list path=/machine/peripheral
4019     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4020     foreach my $per (@$resperipheral) {
4021         if ($per->{name} =~ m/^usb\d+$/) {
4022             $devices->{$per->{name}} = 1;
4023         }
4024     }
4025
4026     return $devices;
4027 }
4028
4029 sub vm_deviceplug {
4030     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4031
4032     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
4033
4034     my $devices_list = vm_devices_list($vmid);
4035     return 1 if defined($devices_list->{$deviceid});
4036
4037     # add PCI bridge if we need it for the device
4038     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
4039
4040     if ($deviceid eq 'tablet') {
4041
4042         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4043
4044     } elsif ($deviceid eq 'keyboard') {
4045
4046         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4047
4048     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4049
4050         die "usb hotplug currently not reliable\n";
4051         # since we can't reliably hot unplug all added usb devices and usb
4052         # passthrough breaks live migration we disable usb hotplugging for now
4053         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4054
4055     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4056
4057         qemu_iothread_add($vmid, $deviceid, $device);
4058
4059         qemu_driveadd($storecfg, $vmid, $device);
4060         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4061
4062         qemu_deviceadd($vmid, $devicefull);
4063         eval { qemu_deviceaddverify($vmid, $deviceid); };
4064         if (my $err = $@) {
4065             eval { qemu_drivedel($vmid, $deviceid); };
4066             warn $@ if $@;
4067             die $err;
4068         }
4069
4070     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4071
4072
4073         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4074         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4075         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4076
4077         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4078
4079         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4080             qemu_iothread_add($vmid, $deviceid, $device);
4081             $devicefull .= ",iothread=iothread-$deviceid";
4082         }
4083
4084         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4085             $devicefull .= ",num_queues=$device->{queues}";
4086         }
4087
4088         qemu_deviceadd($vmid, $devicefull);
4089         qemu_deviceaddverify($vmid, $deviceid);
4090
4091     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4092
4093         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4094         qemu_driveadd($storecfg, $vmid, $device);
4095
4096         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
4097         eval { qemu_deviceadd($vmid, $devicefull); };
4098         if (my $err = $@) {
4099             eval { qemu_drivedel($vmid, $deviceid); };
4100             warn $@ if $@;
4101             die $err;
4102         }
4103
4104     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4105
4106         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4107
4108         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
4109         my $use_old_bios_files = undef;
4110         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4111
4112         my $netdevicefull = print_netdevice_full(
4113             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4114         qemu_deviceadd($vmid, $netdevicefull);
4115         eval {
4116             qemu_deviceaddverify($vmid, $deviceid);
4117             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4118         };
4119         if (my $err = $@) {
4120             eval { qemu_netdevdel($vmid, $deviceid); };
4121             warn $@ if $@;
4122             die $err;
4123         }
4124
4125     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4126
4127         my $bridgeid = $2;
4128         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4129         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4130
4131         qemu_deviceadd($vmid, $devicefull);
4132         qemu_deviceaddverify($vmid, $deviceid);
4133
4134     } else {
4135         die "can't hotplug device '$deviceid'\n";
4136     }
4137
4138     return 1;
4139 }
4140
4141 # fixme: this should raise exceptions on error!
4142 sub vm_deviceunplug {
4143     my ($vmid, $conf, $deviceid) = @_;
4144
4145     my $devices_list = vm_devices_list($vmid);
4146     return 1 if !defined($devices_list->{$deviceid});
4147
4148     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
4149     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
4150
4151     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4152         qemu_devicedel($vmid, $deviceid);
4153     } elsif ($deviceid =~ m/^usb\d+$/) {
4154         die "usb hotplug currently not reliable\n";
4155         # when unplugging usb devices this way, there may be remaining usb
4156         # controllers/hubs so we disable it for now
4157         #qemu_devicedel($vmid, $deviceid);
4158         #qemu_devicedelverify($vmid, $deviceid);
4159     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4160         my $device = parse_drive($deviceid, $conf->{$deviceid});
4161
4162         qemu_devicedel($vmid, $deviceid);
4163         qemu_devicedelverify($vmid, $deviceid);
4164         qemu_drivedel($vmid, $deviceid);
4165         qemu_iothread_del($vmid, $deviceid, $device);
4166     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4167         qemu_devicedel($vmid, $deviceid);
4168         qemu_devicedelverify($vmid, $deviceid);
4169     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4170         my $device = parse_drive($deviceid, $conf->{$deviceid});
4171
4172         qemu_devicedel($vmid, $deviceid);
4173         qemu_drivedel($vmid, $deviceid);
4174         qemu_deletescsihw($conf, $vmid, $deviceid);
4175
4176         qemu_iothread_del($vmid, "virtioscsi$device->{index}", $device)
4177             if $conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single');
4178     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4179         qemu_devicedel($vmid, $deviceid);
4180         qemu_devicedelverify($vmid, $deviceid);
4181         qemu_netdevdel($vmid, $deviceid);
4182     } else {
4183         die "can't unplug device '$deviceid'\n";
4184     }
4185
4186     return 1;
4187 }
4188
4189 sub qemu_deviceadd {
4190     my ($vmid, $devicefull) = @_;
4191
4192     $devicefull = "driver=".$devicefull;
4193     my %options =  split(/[=,]/, $devicefull);
4194
4195     mon_cmd($vmid, "device_add" , %options);
4196 }
4197
4198 sub qemu_devicedel {
4199     my ($vmid, $deviceid) = @_;
4200
4201     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
4202 }
4203
4204 sub qemu_iothread_add {
4205     my ($vmid, $deviceid, $device) = @_;
4206
4207     if ($device->{iothread}) {
4208         my $iothreads = vm_iothreads_list($vmid);
4209         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4210     }
4211 }
4212
4213 sub qemu_iothread_del {
4214     my ($vmid, $deviceid, $device) = @_;
4215
4216     if ($device->{iothread}) {
4217         my $iothreads = vm_iothreads_list($vmid);
4218         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4219     }
4220 }
4221
4222 sub qemu_objectadd {
4223     my ($vmid, $objectid, $qomtype) = @_;
4224
4225     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4226
4227     return 1;
4228 }
4229
4230 sub qemu_objectdel {
4231     my ($vmid, $objectid) = @_;
4232
4233     mon_cmd($vmid, "object-del", id => $objectid);
4234
4235     return 1;
4236 }
4237
4238 sub qemu_driveadd {
4239     my ($storecfg, $vmid, $device) = @_;
4240
4241     my $kvmver = get_running_qemu_version($vmid);
4242     my $io_uring = min_version($kvmver, 6, 0);
4243     my $drive = print_drive_commandline_full($storecfg, $vmid, $device, undef, $io_uring);
4244     $drive =~ s/\\/\\\\/g;
4245     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4246
4247     # If the command succeeds qemu prints: "OK"
4248     return 1 if $ret =~ m/OK/s;
4249
4250     die "adding drive failed: $ret\n";
4251 }
4252
4253 sub qemu_drivedel {
4254     my ($vmid, $deviceid) = @_;
4255
4256     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4257     $ret =~ s/^\s+//;
4258
4259     return 1 if $ret eq "";
4260
4261     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4262     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4263
4264     die "deleting drive $deviceid failed : $ret\n";
4265 }
4266
4267 sub qemu_deviceaddverify {
4268     my ($vmid, $deviceid) = @_;
4269
4270     for (my $i = 0; $i <= 5; $i++) {
4271          my $devices_list = vm_devices_list($vmid);
4272          return 1 if defined($devices_list->{$deviceid});
4273          sleep 1;
4274     }
4275
4276     die "error on hotplug device '$deviceid'\n";
4277 }
4278
4279
4280 sub qemu_devicedelverify {
4281     my ($vmid, $deviceid) = @_;
4282
4283     # need to verify that the device is correctly removed as device_del
4284     # is async and empty return is not reliable
4285
4286     for (my $i = 0; $i <= 5; $i++) {
4287          my $devices_list = vm_devices_list($vmid);
4288          return 1 if !defined($devices_list->{$deviceid});
4289          sleep 1;
4290     }
4291
4292     die "error on hot-unplugging device '$deviceid'\n";
4293 }
4294
4295 sub qemu_findorcreatescsihw {
4296     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4297
4298     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4299
4300     my $scsihwid="$controller_prefix$controller";
4301     my $devices_list = vm_devices_list($vmid);
4302
4303     if (!defined($devices_list->{$scsihwid})) {
4304         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4305     }
4306
4307     return 1;
4308 }
4309
4310 sub qemu_deletescsihw {
4311     my ($conf, $vmid, $opt) = @_;
4312
4313     my $device = parse_drive($opt, $conf->{$opt});
4314
4315     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4316         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4317         return 1;
4318     }
4319
4320     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4321
4322     my $devices_list = vm_devices_list($vmid);
4323     foreach my $opt (keys %{$devices_list}) {
4324         if (is_valid_drivename($opt)) {
4325             my $drive = parse_drive($opt, $conf->{$opt});
4326             if ($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4327                 return 1;
4328             }
4329         }
4330     }
4331
4332     my $scsihwid="scsihw$controller";
4333
4334     vm_deviceunplug($vmid, $conf, $scsihwid);
4335
4336     return 1;
4337 }
4338
4339 sub qemu_add_pci_bridge {
4340     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4341
4342     my $bridges = {};
4343
4344     my $bridgeid;
4345
4346     print_pci_addr($device, $bridges, $arch, $machine_type);
4347
4348     while (my ($k, $v) = each %$bridges) {
4349         $bridgeid = $k;
4350     }
4351     return 1 if !defined($bridgeid) || $bridgeid < 1;
4352
4353     my $bridge = "pci.$bridgeid";
4354     my $devices_list = vm_devices_list($vmid);
4355
4356     if (!defined($devices_list->{$bridge})) {
4357         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4358     }
4359
4360     return 1;
4361 }
4362
4363 sub qemu_set_link_status {
4364     my ($vmid, $device, $up) = @_;
4365
4366     mon_cmd($vmid, "set_link", name => $device,
4367                up => $up ? JSON::true : JSON::false);
4368 }
4369
4370 sub qemu_netdevadd {
4371     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4372
4373     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4374     my %options =  split(/[=,]/, $netdev);
4375
4376     if (defined(my $vhost = $options{vhost})) {
4377         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4378     }
4379
4380     if (defined(my $queues = $options{queues})) {
4381         $options{queues} = $queues + 0;
4382     }
4383
4384     mon_cmd($vmid, "netdev_add",  %options);
4385     return 1;
4386 }
4387
4388 sub qemu_netdevdel {
4389     my ($vmid, $deviceid) = @_;
4390
4391     mon_cmd($vmid, "netdev_del", id => $deviceid);
4392 }
4393
4394 sub qemu_usb_hotplug {
4395     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4396
4397     return if !$device;
4398
4399     # remove the old one first
4400     vm_deviceunplug($vmid, $conf, $deviceid);
4401
4402     # check if xhci controller is necessary and available
4403     if ($device->{usb3}) {
4404
4405         my $devicelist = vm_devices_list($vmid);
4406
4407         if (!$devicelist->{xhci}) {
4408             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4409             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4410         }
4411     }
4412     my $d = parse_usb_device($device->{host});
4413     $d->{usb3} = $device->{usb3};
4414
4415     # add the new one
4416     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4417 }
4418
4419 sub qemu_cpu_hotplug {
4420     my ($vmid, $conf, $vcpus) = @_;
4421
4422     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4423
4424     my $sockets = 1;
4425     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4426     $sockets = $conf->{sockets} if  $conf->{sockets};
4427     my $cores = $conf->{cores} || 1;
4428     my $maxcpus = $sockets * $cores;
4429
4430     $vcpus = $maxcpus if !$vcpus;
4431
4432     die "you can't add more vcpus than maxcpus\n"
4433         if $vcpus > $maxcpus;
4434
4435     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4436
4437     if ($vcpus < $currentvcpus) {
4438
4439         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4440
4441             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4442                 qemu_devicedel($vmid, "cpu$i");
4443                 my $retry = 0;
4444                 my $currentrunningvcpus = undef;
4445                 while (1) {
4446                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4447                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4448                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4449                     $retry++;
4450                     sleep 1;
4451                 }
4452                 #update conf after each succesfull cpu unplug
4453                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4454                 PVE::QemuConfig->write_config($vmid, $conf);
4455             }
4456         } else {
4457             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4458         }
4459
4460         return;
4461     }
4462
4463     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4464     die "vcpus in running vm does not match its configuration\n"
4465         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4466
4467     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4468
4469         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4470             my $cpustr = print_cpu_device($conf, $i);
4471             qemu_deviceadd($vmid, $cpustr);
4472
4473             my $retry = 0;
4474             my $currentrunningvcpus = undef;
4475             while (1) {
4476                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4477                 last if scalar(@{$currentrunningvcpus}) == $i;
4478                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4479                 sleep 1;
4480                 $retry++;
4481             }
4482             #update conf after each succesfull cpu hotplug
4483             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4484             PVE::QemuConfig->write_config($vmid, $conf);
4485         }
4486     } else {
4487
4488         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4489             mon_cmd($vmid, "cpu-add", id => int($i));
4490         }
4491     }
4492 }
4493
4494 sub qemu_block_set_io_throttle {
4495     my ($vmid, $deviceid,
4496         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4497         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4498         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4499         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4500
4501     return if !check_running($vmid) ;
4502
4503     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4504         bps => int($bps),
4505         bps_rd => int($bps_rd),
4506         bps_wr => int($bps_wr),
4507         iops => int($iops),
4508         iops_rd => int($iops_rd),
4509         iops_wr => int($iops_wr),
4510         bps_max => int($bps_max),
4511         bps_rd_max => int($bps_rd_max),
4512         bps_wr_max => int($bps_wr_max),
4513         iops_max => int($iops_max),
4514         iops_rd_max => int($iops_rd_max),
4515         iops_wr_max => int($iops_wr_max),
4516         bps_max_length => int($bps_max_length),
4517         bps_rd_max_length => int($bps_rd_max_length),
4518         bps_wr_max_length => int($bps_wr_max_length),
4519         iops_max_length => int($iops_max_length),
4520         iops_rd_max_length => int($iops_rd_max_length),
4521         iops_wr_max_length => int($iops_wr_max_length),
4522     );
4523
4524 }
4525
4526 sub qemu_block_resize {
4527     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4528
4529     my $running = check_running($vmid);
4530
4531     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4532
4533     return if !$running;
4534
4535     my $padding = (1024 - $size % 1024) % 1024;
4536     $size = $size + $padding;
4537
4538     mon_cmd(
4539         $vmid,
4540         "block_resize",
4541         device => $deviceid,
4542         size => int($size),
4543         timeout => 60,
4544     );
4545 }
4546
4547 sub qemu_volume_snapshot {
4548     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4549
4550     my $running = check_running($vmid);
4551
4552     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4553         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4554     } else {
4555         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4556     }
4557 }
4558
4559 sub qemu_volume_snapshot_delete {
4560     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4561
4562     my $running = check_running($vmid);
4563
4564     if($running) {
4565
4566         $running = undef;
4567         my $conf = PVE::QemuConfig->load_config($vmid);
4568         PVE::QemuConfig->foreach_volume($conf, sub {
4569             my ($ds, $drive) = @_;
4570             $running = 1 if $drive->{file} eq $volid;
4571         });
4572     }
4573
4574     if ($running && do_snapshots_with_qemu($storecfg, $volid, $deviceid)) {
4575         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4576     } else {
4577         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4578     }
4579 }
4580
4581 sub set_migration_caps {
4582     my ($vmid, $savevm) = @_;
4583
4584     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4585
4586     my $bitmap_prop = $savevm ? 'pbs-dirty-bitmap-savevm' : 'pbs-dirty-bitmap-migration';
4587     my $dirty_bitmaps = $qemu_support->{$bitmap_prop} ? 1 : 0;
4588
4589     my $cap_ref = [];
4590
4591     my $enabled_cap = {
4592         "auto-converge" => 1,
4593         "xbzrle" => 1,
4594         "x-rdma-pin-all" => 0,
4595         "zero-blocks" => 0,
4596         "compress" => 0,
4597         "dirty-bitmaps" => $dirty_bitmaps,
4598     };
4599
4600     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4601
4602     for my $supported_capability (@$supported_capabilities) {
4603         push @$cap_ref, {
4604             capability => $supported_capability->{capability},
4605             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4606         };
4607     }
4608
4609     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4610 }
4611
4612 sub foreach_volid {
4613     my ($conf, $func, @param) = @_;
4614
4615     my $volhash = {};
4616
4617     my $test_volid = sub {
4618         my ($key, $drive, $snapname) = @_;
4619
4620         my $volid = $drive->{file};
4621         return if !$volid;
4622
4623         $volhash->{$volid}->{cdrom} //= 1;
4624         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4625
4626         my $replicate = $drive->{replicate} // 1;
4627         $volhash->{$volid}->{replicate} //= 0;
4628         $volhash->{$volid}->{replicate} = 1 if $replicate;
4629
4630         $volhash->{$volid}->{shared} //= 0;
4631         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4632
4633         $volhash->{$volid}->{referenced_in_config} //= 0;
4634         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4635
4636         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4637             if defined($snapname);
4638
4639         my $size = $drive->{size};
4640         $volhash->{$volid}->{size} //= $size if $size;
4641
4642         $volhash->{$volid}->{is_vmstate} //= 0;
4643         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4644
4645         $volhash->{$volid}->{is_tpmstate} //= 0;
4646         $volhash->{$volid}->{is_tpmstate} = 1 if $key eq 'tpmstate0';
4647
4648         $volhash->{$volid}->{is_unused} //= 0;
4649         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4650
4651         $volhash->{$volid}->{drivename} = $key if is_valid_drivename($key);
4652     };
4653
4654     my $include_opts = {
4655         extra_keys => ['vmstate'],
4656         include_unused => 1,
4657     };
4658
4659     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4660     foreach my $snapname (keys %{$conf->{snapshots}}) {
4661         my $snap = $conf->{snapshots}->{$snapname};
4662         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4663     }
4664
4665     foreach my $volid (keys %$volhash) {
4666         &$func($volid, $volhash->{$volid}, @param);
4667     }
4668 }
4669
4670 my $fast_plug_option = {
4671     'lock' => 1,
4672     'name' => 1,
4673     'onboot' => 1,
4674     'shares' => 1,
4675     'startup' => 1,
4676     'description' => 1,
4677     'protection' => 1,
4678     'vmstatestorage' => 1,
4679     'hookscript' => 1,
4680     'tags' => 1,
4681 };
4682
4683 # hotplug changes in [PENDING]
4684 # $selection hash can be used to only apply specified options, for
4685 # example: { cores => 1 } (only apply changed 'cores')
4686 # $errors ref is used to return error messages
4687 sub vmconfig_hotplug_pending {
4688     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4689
4690     my $defaults = load_defaults();
4691     my $arch = get_vm_arch($conf);
4692     my $machine_type = get_vm_machine($conf, undef, $arch);
4693
4694     # commit values which do not have any impact on running VM first
4695     # Note: those option cannot raise errors, we we do not care about
4696     # $selection and always apply them.
4697
4698     my $add_error = sub {
4699         my ($opt, $msg) = @_;
4700         $errors->{$opt} = "hotplug problem - $msg";
4701     };
4702
4703     my $changes = 0;
4704     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4705         if ($fast_plug_option->{$opt}) {
4706             $conf->{$opt} = $conf->{pending}->{$opt};
4707             delete $conf->{pending}->{$opt};
4708             $changes = 1;
4709         }
4710     }
4711
4712     if ($changes) {
4713         PVE::QemuConfig->write_config($vmid, $conf);
4714     }
4715
4716     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4717
4718     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4719     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4720     foreach my $opt (sort keys %$pending_delete_hash) {
4721         next if $selection && !$selection->{$opt};
4722         my $force = $pending_delete_hash->{$opt}->{force};
4723         eval {
4724             if ($opt eq 'hotplug') {
4725                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4726             } elsif ($opt eq 'tablet') {
4727                 die "skip\n" if !$hotplug_features->{usb};
4728                 if ($defaults->{tablet}) {
4729                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4730                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4731                         if $arch eq 'aarch64';
4732                 } else {
4733                     vm_deviceunplug($vmid, $conf, 'tablet');
4734                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4735                 }
4736             } elsif ($opt =~ m/^usb\d+/) {
4737                 die "skip\n";
4738                 # since we cannot reliably hot unplug usb devices we are disabling it
4739                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4740                 #vm_deviceunplug($vmid, $conf, $opt);
4741             } elsif ($opt eq 'vcpus') {
4742                 die "skip\n" if !$hotplug_features->{cpu};
4743                 qemu_cpu_hotplug($vmid, $conf, undef);
4744             } elsif ($opt eq 'balloon') {
4745                 # enable balloon device is not hotpluggable
4746                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4747                 # here we reset the ballooning value to memory
4748                 my $balloon = $conf->{memory} || $defaults->{memory};
4749                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4750             } elsif ($fast_plug_option->{$opt}) {
4751                 # do nothing
4752             } elsif ($opt =~ m/^net(\d+)$/) {
4753                 die "skip\n" if !$hotplug_features->{network};
4754                 vm_deviceunplug($vmid, $conf, $opt);
4755             } elsif (is_valid_drivename($opt)) {
4756                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4757                 vm_deviceunplug($vmid, $conf, $opt);
4758                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4759             } elsif ($opt =~ m/^memory$/) {
4760                 die "skip\n" if !$hotplug_features->{memory};
4761                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4762             } elsif ($opt eq 'cpuunits') {
4763                 $cgroup->change_cpu_shares(undef, 1024);
4764             } elsif ($opt eq 'cpulimit') {
4765                 $cgroup->change_cpu_quota(-1, 100000);
4766             } else {
4767                 die "skip\n";
4768             }
4769         };
4770         if (my $err = $@) {
4771             &$add_error($opt, $err) if $err ne "skip\n";
4772         } else {
4773             delete $conf->{$opt};
4774             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4775         }
4776     }
4777
4778     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4779     $apply_pending_cloudinit = sub {
4780         return if $apply_pending_cloudinit_done; # once is enough
4781         $apply_pending_cloudinit_done = 1; # once is enough
4782
4783         my ($key, $value) = @_;
4784
4785         my @cloudinit_opts = keys %$confdesc_cloudinit;
4786         foreach my $opt (keys %{$conf->{pending}}) {
4787             next if !grep { $_ eq $opt } @cloudinit_opts;
4788             $conf->{$opt} = delete $conf->{pending}->{$opt};
4789         }
4790
4791         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4792         foreach my $opt (sort keys %$pending_delete_hash) {
4793             next if !grep { $_ eq $opt } @cloudinit_opts;
4794             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4795             delete $conf->{$opt};
4796         }
4797
4798         my $new_conf = { %$conf };
4799         $new_conf->{$key} = $value;
4800         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4801     };
4802
4803     foreach my $opt (keys %{$conf->{pending}}) {
4804         next if $selection && !$selection->{$opt};
4805         my $value = $conf->{pending}->{$opt};
4806         eval {
4807             if ($opt eq 'hotplug') {
4808                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4809             } elsif ($opt eq 'tablet') {
4810                 die "skip\n" if !$hotplug_features->{usb};
4811                 if ($value == 1) {
4812                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4813                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4814                         if $arch eq 'aarch64';
4815                 } elsif ($value == 0) {
4816                     vm_deviceunplug($vmid, $conf, 'tablet');
4817                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4818                 }
4819             } elsif ($opt =~ m/^usb\d+$/) {
4820                 die "skip\n";
4821                 # since we cannot reliably hot unplug usb devices we disable it for now
4822                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4823                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4824                 #die "skip\n" if !$d;
4825                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4826             } elsif ($opt eq 'vcpus') {
4827                 die "skip\n" if !$hotplug_features->{cpu};
4828                 qemu_cpu_hotplug($vmid, $conf, $value);
4829             } elsif ($opt eq 'balloon') {
4830                 # enable/disable balloning device is not hotpluggable
4831                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4832                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4833                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4834
4835                 # allow manual ballooning if shares is set to zero
4836                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4837                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4838                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4839                 }
4840             } elsif ($opt =~ m/^net(\d+)$/) {
4841                 # some changes can be done without hotplug
4842                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4843                                     $vmid, $opt, $value, $arch, $machine_type);
4844             } elsif (is_valid_drivename($opt)) {
4845                 die "skip\n" if $opt eq 'efidisk0' || $opt eq 'tpmstate0';
4846                 # some changes can be done without hotplug
4847                 my $drive = parse_drive($opt, $value);
4848                 if (drive_is_cloudinit($drive)) {
4849                     &$apply_pending_cloudinit($opt, $value);
4850                 }
4851                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4852                                      $vmid, $opt, $value, $arch, $machine_type);
4853             } elsif ($opt =~ m/^memory$/) { #dimms
4854                 die "skip\n" if !$hotplug_features->{memory};
4855                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4856             } elsif ($opt eq 'cpuunits') {
4857                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, 1024);
4858             } elsif ($opt eq 'cpulimit') {
4859                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4860                 $cgroup->change_cpu_quota($cpulimit, 100000);
4861             } else {
4862                 die "skip\n";  # skip non-hot-pluggable options
4863             }
4864         };
4865         if (my $err = $@) {
4866             &$add_error($opt, $err) if $err ne "skip\n";
4867         } else {
4868             $conf->{$opt} = $value;
4869             delete $conf->{pending}->{$opt};
4870         }
4871     }
4872
4873     PVE::QemuConfig->write_config($vmid, $conf);
4874 }
4875
4876 sub try_deallocate_drive {
4877     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4878
4879     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4880         my $volid = $drive->{file};
4881         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4882             my $sid = PVE::Storage::parse_volume_id($volid);
4883             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4884
4885             # check if the disk is really unused
4886             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4887                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4888             PVE::Storage::vdisk_free($storecfg, $volid);
4889             return 1;
4890         } else {
4891             # If vm is not owner of this disk remove from config
4892             return 1;
4893         }
4894     }
4895
4896     return;
4897 }
4898
4899 sub vmconfig_delete_or_detach_drive {
4900     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4901
4902     my $drive = parse_drive($opt, $conf->{$opt});
4903
4904     my $rpcenv = PVE::RPCEnvironment::get();
4905     my $authuser = $rpcenv->get_user();
4906
4907     if ($force) {
4908         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4909         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4910     } else {
4911         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4912     }
4913 }
4914
4915
4916
4917 sub vmconfig_apply_pending {
4918     my ($vmid, $conf, $storecfg, $errors) = @_;
4919
4920     my $add_apply_error = sub {
4921         my ($opt, $msg) = @_;
4922         my $err_msg = "unable to apply pending change $opt : $msg";
4923         $errors->{$opt} = $err_msg;
4924         warn $err_msg;
4925     };
4926
4927     # cold plug
4928
4929     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4930     foreach my $opt (sort keys %$pending_delete_hash) {
4931         my $force = $pending_delete_hash->{$opt}->{force};
4932         eval {
4933             if ($opt =~ m/^unused/) {
4934                 die "internal error";
4935             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4936                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4937             }
4938         };
4939         if (my $err = $@) {
4940             $add_apply_error->($opt, $err);
4941         } else {
4942             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4943             delete $conf->{$opt};
4944         }
4945     }
4946
4947     PVE::QemuConfig->cleanup_pending($conf);
4948
4949     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4950         next if $opt eq 'delete'; # just to be sure
4951         eval {
4952             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4953                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4954             }
4955         };
4956         if (my $err = $@) {
4957             $add_apply_error->($opt, $err);
4958         } else {
4959             $conf->{$opt} = delete $conf->{pending}->{$opt};
4960         }
4961     }
4962
4963     # write all changes at once to avoid unnecessary i/o
4964     PVE::QemuConfig->write_config($vmid, $conf);
4965 }
4966
4967 sub vmconfig_update_net {
4968     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4969
4970     my $newnet = parse_net($value);
4971
4972     if ($conf->{$opt}) {
4973         my $oldnet = parse_net($conf->{$opt});
4974
4975         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4976             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4977             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4978             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4979
4980             # for non online change, we try to hot-unplug
4981             die "skip\n" if !$hotplug;
4982             vm_deviceunplug($vmid, $conf, $opt);
4983         } else {
4984
4985             die "internal error" if $opt !~ m/net(\d+)/;
4986             my $iface = "tap${vmid}i$1";
4987
4988             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4989                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4990                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4991                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4992                 PVE::Network::tap_unplug($iface);
4993
4994                 if ($have_sdn) {
4995                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4996                 } else {
4997                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4998                 }
4999             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5000                 # Rate can be applied on its own but any change above needs to
5001                 # include the rate in tap_plug since OVS resets everything.
5002                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5003             }
5004
5005             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5006                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5007             }
5008
5009             return 1;
5010         }
5011     }
5012
5013     if ($hotplug) {
5014         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5015     } else {
5016         die "skip\n";
5017     }
5018 }
5019
5020 sub vmconfig_update_disk {
5021     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5022
5023     my $drive = parse_drive($opt, $value);
5024
5025     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
5026         my $media = $drive->{media} || 'disk';
5027         my $oldmedia = $old_drive->{media} || 'disk';
5028         die "unable to change media type\n" if $media ne $oldmedia;
5029
5030         if (!drive_is_cdrom($old_drive)) {
5031
5032             if ($drive->{file} ne $old_drive->{file}) {
5033
5034                 die "skip\n" if !$hotplug;
5035
5036                 # unplug and register as unused
5037                 vm_deviceunplug($vmid, $conf, $opt);
5038                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5039
5040             } else {
5041                 # update existing disk
5042
5043                 # skip non hotpluggable value
5044                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5045                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5046                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5047                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
5048                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
5049                     die "skip\n";
5050                 }
5051
5052                 # apply throttle
5053                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5054                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5055                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5056                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5057                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5058                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5059                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5060                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5061                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5062                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5063                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5064                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5065                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5066                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5067                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5068                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5069                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5070                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5071
5072                     qemu_block_set_io_throttle(
5073                         $vmid,"drive-$opt",
5074                         ($drive->{mbps} || 0)*1024*1024,
5075                         ($drive->{mbps_rd} || 0)*1024*1024,
5076                         ($drive->{mbps_wr} || 0)*1024*1024,
5077                          $drive->{iops} || 0,
5078                          $drive->{iops_rd} || 0,
5079                          $drive->{iops_wr} || 0,
5080                         ($drive->{mbps_max} || 0)*1024*1024,
5081                         ($drive->{mbps_rd_max} || 0)*1024*1024,
5082                         ($drive->{mbps_wr_max} || 0)*1024*1024,
5083                          $drive->{iops_max} || 0,
5084                          $drive->{iops_rd_max} || 0,
5085                          $drive->{iops_wr_max} || 0,
5086                          $drive->{bps_max_length} || 1,
5087                          $drive->{bps_rd_max_length} || 1,
5088                          $drive->{bps_wr_max_length} || 1,
5089                          $drive->{iops_max_length} || 1,
5090                          $drive->{iops_rd_max_length} || 1,
5091                          $drive->{iops_wr_max_length} || 1,
5092                     );
5093
5094                 }
5095
5096                 return 1;
5097             }
5098
5099         } else { # cdrom
5100
5101             if ($drive->{file} eq 'none') {
5102                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5103                 if (drive_is_cloudinit($old_drive)) {
5104                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5105                 }
5106             } else {
5107                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5108
5109                 # force eject if locked
5110                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
5111
5112                 if ($path) {
5113                     mon_cmd($vmid, "blockdev-change-medium",
5114                         id => "$opt", filename => "$path");
5115                 }
5116             }
5117
5118             return 1;
5119         }
5120     }
5121
5122     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5123     # hotplug new disks
5124     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5125     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5126 }
5127
5128 # called in locked context by incoming migration
5129 sub vm_migrate_get_nbd_disks {
5130     my ($storecfg, $conf, $replicated_volumes) = @_;
5131
5132     my $local_volumes = {};
5133     PVE::QemuConfig->foreach_volume($conf, sub {
5134         my ($ds, $drive) = @_;
5135
5136         return if drive_is_cdrom($drive);
5137
5138         my $volid = $drive->{file};
5139
5140         return if !$volid;
5141
5142         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5143
5144         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5145         return if $scfg->{shared};
5146
5147         # replicated disks re-use existing state via bitmap
5148         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
5149         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
5150     });
5151     return $local_volumes;
5152 }
5153
5154 # called in locked context by incoming migration
5155 sub vm_migrate_alloc_nbd_disks {
5156     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
5157
5158     my $format = undef;
5159
5160     my $nbd = {};
5161     foreach my $opt (sort keys %$source_volumes) {
5162         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
5163
5164         if ($use_existing) {
5165             $nbd->{$opt}->{drivestr} = print_drive($drive);
5166             $nbd->{$opt}->{volid} = $volid;
5167             $nbd->{$opt}->{replicated} = 1;
5168             next;
5169         }
5170
5171         # If a remote storage is specified and the format of the original
5172         # volume is not available there, fall back to the default format.
5173         # Otherwise use the same format as the original.
5174         if (!$storagemap->{identity}) {
5175             $storeid = map_storage($storagemap, $storeid);
5176             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5177             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5178             my $fileFormat = qemu_img_format($scfg, $volname);
5179             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
5180         } else {
5181             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5182             $format = qemu_img_format($scfg, $volname);
5183         }
5184
5185         my $size = $drive->{size} / 1024;
5186         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
5187         my $newdrive = $drive;
5188         $newdrive->{format} = $format;
5189         $newdrive->{file} = $newvolid;
5190         my $drivestr = print_drive($newdrive);
5191         $nbd->{$opt}->{drivestr} = $drivestr;
5192         $nbd->{$opt}->{volid} = $newvolid;
5193     }
5194
5195     return $nbd;
5196 }
5197
5198 # see vm_start_nolock for parameters, additionally:
5199 # migrate_opts:
5200 #   storagemap = parsed storage map for allocating NBD disks
5201 sub vm_start {
5202     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
5203
5204     return PVE::QemuConfig->lock_config($vmid, sub {
5205         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
5206
5207         die "you can't start a vm if it's a template\n"
5208             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
5209
5210         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
5211         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
5212
5213         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
5214
5215         if ($has_backup_lock && $running) {
5216             # a backup is currently running, attempt to start the guest in the
5217             # existing QEMU instance
5218             return vm_resume($vmid);
5219         }
5220
5221         PVE::QemuConfig->check_lock($conf)
5222             if !($params->{skiplock} || $has_suspended_lock);
5223
5224         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
5225
5226         die "VM $vmid already running\n" if $running;
5227
5228         if (my $storagemap = $migrate_opts->{storagemap}) {
5229             my $replicated = $migrate_opts->{replicated_volumes};
5230             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
5231             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
5232
5233             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
5234                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
5235             }
5236         }
5237
5238         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
5239     });
5240 }
5241
5242
5243 # params:
5244 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
5245 #   skiplock => 0/1, skip checking for config lock
5246 #   skiptemplate => 0/1, skip checking whether VM is template
5247 #   forcemachine => to force Qemu machine (rollback/migration)
5248 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
5249 #   timeout => in seconds
5250 #   paused => start VM in paused state (backup)
5251 #   resume => resume from hibernation
5252 #   pbs-backing => {
5253 #      sata0 => {
5254 #         repository
5255 #         snapshot
5256 #         keyfile
5257 #         archive
5258 #      },
5259 #      virtio2 => ...
5260 #   }
5261 # migrate_opts:
5262 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
5263 #   migratedfrom => source node
5264 #   spice_ticket => used for spice migration, passed via tunnel/stdin
5265 #   network => CIDR of migration network
5266 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5267 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5268 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5269 sub vm_start_nolock {
5270     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5271
5272     my $statefile = $params->{statefile};
5273     my $resume = $params->{resume};
5274
5275     my $migratedfrom = $migrate_opts->{migratedfrom};
5276     my $migration_type = $migrate_opts->{type};
5277
5278     my $res = {};
5279
5280     # clean up leftover reboot request files
5281     eval { clear_reboot_request($vmid); };
5282     warn $@ if $@;
5283
5284     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5285         vmconfig_apply_pending($vmid, $conf, $storecfg);
5286         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5287     }
5288
5289     # don't regenerate the ISO if the VM is started as part of a live migration
5290     # this way we can reuse the old ISO with the correct config
5291     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid) if !$migratedfrom;
5292
5293     my $defaults = load_defaults();
5294
5295     # set environment variable useful inside network script
5296     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5297
5298     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5299
5300     my $forcemachine = $params->{forcemachine};
5301     my $forcecpu = $params->{forcecpu};
5302     if ($resume) {
5303         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5304         $forcemachine = $conf->{runningmachine};
5305         $forcecpu = $conf->{runningcpu};
5306         print "Resuming suspended VM\n";
5307     }
5308
5309     my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid,
5310         $conf, $defaults, $forcemachine, $forcecpu, $params->{'pbs-backing'});
5311
5312     my $migration_ip;
5313     my $get_migration_ip = sub {
5314         my ($nodename) = @_;
5315
5316         return $migration_ip if defined($migration_ip);
5317
5318         my $cidr = $migrate_opts->{network};
5319
5320         if (!defined($cidr)) {
5321             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5322             $cidr = $dc_conf->{migration}->{network};
5323         }
5324
5325         if (defined($cidr)) {
5326             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5327
5328             die "could not get IP: no address configured on local " .
5329                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5330
5331             die "could not get IP: multiple addresses configured on local " .
5332                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5333
5334             $migration_ip = @$ips[0];
5335         }
5336
5337         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5338             if !defined($migration_ip);
5339
5340         return $migration_ip;
5341     };
5342
5343     my $migrate_uri;
5344     if ($statefile) {
5345         if ($statefile eq 'tcp') {
5346             my $localip = "localhost";
5347             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5348             my $nodename = nodename();
5349
5350             if (!defined($migration_type)) {
5351                 if (defined($datacenterconf->{migration}->{type})) {
5352                     $migration_type = $datacenterconf->{migration}->{type};
5353                 } else {
5354                     $migration_type = 'secure';
5355                 }
5356             }
5357
5358             if ($migration_type eq 'insecure') {
5359                 $localip = $get_migration_ip->($nodename);
5360                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5361             }
5362
5363             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5364             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5365             $migrate_uri = "tcp:${localip}:${migrate_port}";
5366             push @$cmd, '-incoming', $migrate_uri;
5367             push @$cmd, '-S';
5368
5369         } elsif ($statefile eq 'unix') {
5370             # should be default for secure migrations as a ssh TCP forward
5371             # tunnel is not deterministic reliable ready and fails regurarly
5372             # to set up in time, so use UNIX socket forwards
5373             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5374             unlink $socket_addr;
5375
5376             $migrate_uri = "unix:$socket_addr";
5377
5378             push @$cmd, '-incoming', $migrate_uri;
5379             push @$cmd, '-S';
5380
5381         } elsif (-e $statefile) {
5382             push @$cmd, '-loadstate', $statefile;
5383         } else {
5384             my $statepath = PVE::Storage::path($storecfg, $statefile);
5385             push @$vollist, $statefile;
5386             push @$cmd, '-loadstate', $statepath;
5387         }
5388     } elsif ($params->{paused}) {
5389         push @$cmd, '-S';
5390     }
5391
5392     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5393
5394     my $pci_devices = {}; # host pci devices
5395     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5396         my $dev = $conf->{"hostpci$i"} or next;
5397         $pci_devices->{$i} = parse_hostpci($dev);
5398     }
5399
5400     my $pci_id_list = [ map { $_->{id} } map { $_->{pciid}->@* } values $pci_devices->%* ];
5401     # reserve all PCI IDs before actually doing anything with them
5402     PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, $start_timeout);
5403
5404     eval {
5405         for my $id (sort keys %$pci_devices) {
5406             my $d = $pci_devices->{$id};
5407             for my $dev ($d->{pciid}->@*) {
5408                 PVE::QemuServer::PCI::prepare_pci_device($vmid, $dev->{id}, $id, $d->{mdev});
5409             }
5410         }
5411     };
5412     if (my $err = $@) {
5413         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5414         warn $@ if $@;
5415         die $err;
5416     }
5417
5418     PVE::Storage::activate_volumes($storecfg, $vollist);
5419
5420     eval {
5421         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5422             outfunc => sub {}, errfunc => sub {});
5423     };
5424     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5425     # timeout should be more than enough here...
5426     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5427
5428     my $cpuunits = get_cpuunits($conf);
5429
5430     my %run_params = (
5431         timeout => $statefile ? undef : $start_timeout,
5432         umask => 0077,
5433         noerr => 1,
5434     );
5435
5436     # when migrating, prefix QEMU output so other side can pick up any
5437     # errors that might occur and show the user
5438     if ($migratedfrom) {
5439         $run_params{quiet} = 1;
5440         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5441     }
5442
5443     my %systemd_properties = (
5444         Slice => 'qemu.slice',
5445         KillMode => 'process',
5446         SendSIGKILL => 0,
5447         TimeoutStopUSec => ULONG_MAX, # infinity
5448     );
5449
5450     if (PVE::CGroup::cgroup_mode() == 2) {
5451         $cpuunits = 10000 if $cpuunits >= 10000; # else we get an error
5452         $systemd_properties{CPUWeight} = $cpuunits;
5453     } else {
5454         $systemd_properties{CPUShares} = $cpuunits;
5455     }
5456
5457     if (my $cpulimit = $conf->{cpulimit}) {
5458         $systemd_properties{CPUQuota} = int($cpulimit * 100);
5459     }
5460     $systemd_properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5461
5462     my $run_qemu = sub {
5463         PVE::Tools::run_fork sub {
5464             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %systemd_properties);
5465
5466             my $tpmpid;
5467             if (my $tpm = $conf->{tpmstate0}) {
5468                 # start the TPM emulator so QEMU can connect on start
5469                 $tpmpid = start_swtpm($storecfg, $vmid, $tpm, $migratedfrom);
5470             }
5471
5472             my $exitcode = run_command($cmd, %run_params);
5473             if ($exitcode) {
5474                 warn "stopping swtpm instance (pid $tpmpid) due to QEMU startup error\n";
5475                 kill 'TERM', $tpmpid if $tpmpid;
5476                 die "QEMU exited with code $exitcode\n";
5477             }
5478         };
5479     };
5480
5481     if ($conf->{hugepages}) {
5482
5483         my $code = sub {
5484             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5485             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5486
5487             PVE::QemuServer::Memory::hugepages_mount();
5488             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5489
5490             eval { $run_qemu->() };
5491             if (my $err = $@) {
5492                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5493                     if !$conf->{keephugepages};
5494                 die $err;
5495             }
5496
5497             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5498                 if !$conf->{keephugepages};
5499         };
5500         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5501
5502     } else {
5503         eval { $run_qemu->() };
5504     }
5505
5506     if (my $err = $@) {
5507         # deactivate volumes if start fails
5508         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5509         eval { PVE::QemuServer::PCI::remove_pci_reservation($pci_id_list) };
5510
5511         die "start failed: $err";
5512     }
5513
5514     # re-reserve all PCI IDs now that we can know the actual VM PID
5515     my $pid = PVE::QemuServer::Helpers::vm_running_locally($vmid);
5516     eval { PVE::QemuServer::PCI::reserve_pci_usage($pci_id_list, $vmid, undef, $pid) };
5517     warn $@ if $@;
5518
5519     print "migration listens on $migrate_uri\n" if $migrate_uri;
5520     $res->{migrate_uri} = $migrate_uri;
5521
5522     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5523         eval { mon_cmd($vmid, "cont"); };
5524         warn $@ if $@;
5525     }
5526
5527     #start nbd server for storage migration
5528     if (my $nbd = $migrate_opts->{nbd}) {
5529         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5530
5531         my $migrate_storage_uri;
5532         # nbd_protocol_version > 0 for unix socket support
5533         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5534             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5535             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5536             $migrate_storage_uri = "nbd:unix:$socket_path";
5537         } else {
5538             my $nodename = nodename();
5539             my $localip = $get_migration_ip->($nodename);
5540             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5541             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5542
5543             mon_cmd($vmid, "nbd-server-start", addr => {
5544                 type => 'inet',
5545                 data => {
5546                     host => "${localip}",
5547                     port => "${storage_migrate_port}",
5548                 },
5549             });
5550             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5551             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5552         }
5553
5554         $res->{migrate_storage_uri} = $migrate_storage_uri;
5555
5556         foreach my $opt (sort keys %$nbd) {
5557             my $drivestr = $nbd->{$opt}->{drivestr};
5558             my $volid = $nbd->{$opt}->{volid};
5559             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5560             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5561             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5562             print "re-using replicated volume: $opt - $volid\n"
5563                 if $nbd->{$opt}->{replicated};
5564
5565             $res->{drives}->{$opt} = $nbd->{$opt};
5566             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5567         }
5568     }
5569
5570     if ($migratedfrom) {
5571         eval {
5572             set_migration_caps($vmid);
5573         };
5574         warn $@ if $@;
5575
5576         if ($spice_port) {
5577             print "spice listens on port $spice_port\n";
5578             $res->{spice_port} = $spice_port;
5579             if ($migrate_opts->{spice_ticket}) {
5580                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5581                     $migrate_opts->{spice_ticket});
5582                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5583             }
5584         }
5585
5586     } else {
5587         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5588             if !$statefile && $conf->{balloon};
5589
5590         foreach my $opt (keys %$conf) {
5591             next if $opt !~  m/^net\d+$/;
5592             my $nicconf = parse_net($conf->{$opt});
5593             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5594         }
5595     }
5596
5597     mon_cmd($vmid, 'qom-set',
5598                 path => "machine/peripheral/balloon0",
5599                 property => "guest-stats-polling-interval",
5600                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5601
5602     if ($resume) {
5603         print "Resumed VM, removing state\n";
5604         if (my $vmstate = $conf->{vmstate}) {
5605             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5606             PVE::Storage::vdisk_free($storecfg, $vmstate);
5607         }
5608         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5609         PVE::QemuConfig->write_config($vmid, $conf);
5610     }
5611
5612     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5613
5614     return $res;
5615 }
5616
5617 sub vm_commandline {
5618     my ($storecfg, $vmid, $snapname) = @_;
5619
5620     my $conf = PVE::QemuConfig->load_config($vmid);
5621     my $forcemachine;
5622     my $forcecpu;
5623
5624     if ($snapname) {
5625         my $snapshot = $conf->{snapshots}->{$snapname};
5626         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5627
5628         # check for machine or CPU overrides in snapshot
5629         $forcemachine = $snapshot->{runningmachine};
5630         $forcecpu = $snapshot->{runningcpu};
5631
5632         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5633
5634         $conf = $snapshot;
5635     }
5636
5637     my $defaults = load_defaults();
5638
5639     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5640         $forcemachine, $forcecpu);
5641
5642     return PVE::Tools::cmd2string($cmd);
5643 }
5644
5645 sub vm_reset {
5646     my ($vmid, $skiplock) = @_;
5647
5648     PVE::QemuConfig->lock_config($vmid, sub {
5649
5650         my $conf = PVE::QemuConfig->load_config($vmid);
5651
5652         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5653
5654         mon_cmd($vmid, "system_reset");
5655     });
5656 }
5657
5658 sub get_vm_volumes {
5659     my ($conf) = @_;
5660
5661     my $vollist = [];
5662     foreach_volid($conf, sub {
5663         my ($volid, $attr) = @_;
5664
5665         return if $volid =~ m|^/|;
5666
5667         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5668         return if !$sid;
5669
5670         push @$vollist, $volid;
5671     });
5672
5673     return $vollist;
5674 }
5675
5676 sub vm_stop_cleanup {
5677     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5678
5679     eval {
5680
5681         if (!$keepActive) {
5682             my $vollist = get_vm_volumes($conf);
5683             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5684
5685             if (my $tpmdrive = $conf->{tpmstate0}) {
5686                 my $tpm = parse_drive("tpmstate0", $tpmdrive);
5687                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($tpm->{file}, 1);
5688                 if ($storeid) {
5689                     PVE::Storage::unmap_volume($storecfg, $tpm->{file});
5690                 }
5691             }
5692         }
5693
5694         foreach my $ext (qw(mon qmp pid vnc qga)) {
5695             unlink "/var/run/qemu-server/${vmid}.$ext";
5696         }
5697
5698         if ($conf->{ivshmem}) {
5699             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5700             # just delete it for now, VMs which have this already open do not
5701             # are affected, but new VMs will get a separated one. If this
5702             # becomes an issue we either add some sort of ref-counting or just
5703             # add a "don't delete on stop" flag to the ivshmem format.
5704             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5705         }
5706
5707         my $ids = [];
5708         foreach my $key (keys %$conf) {
5709             next if $key !~ m/^hostpci(\d+)$/;
5710             my $hostpciindex = $1;
5711             my $d = parse_hostpci($conf->{$key});
5712             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5713
5714             foreach my $pci (@{$d->{pciid}}) {
5715                 my $pciid = $pci->{id};
5716                 push @$ids, $pci->{id};
5717                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5718             }
5719         }
5720         PVE::QemuServer::PCI::remove_pci_reservation($ids);
5721
5722         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5723     };
5724     warn $@ if $@; # avoid errors - just warn
5725 }
5726
5727 # call only in locked context
5728 sub _do_vm_stop {
5729     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5730
5731     my $pid = check_running($vmid, $nocheck);
5732     return if !$pid;
5733
5734     my $conf;
5735     if (!$nocheck) {
5736         $conf = PVE::QemuConfig->load_config($vmid);
5737         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5738         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5739             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5740             $timeout = $opts->{down} if $opts->{down};
5741         }
5742         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5743     }
5744
5745     eval {
5746         if ($shutdown) {
5747             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5748                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5749             } else {
5750                 mon_cmd($vmid, "system_powerdown");
5751             }
5752         } else {
5753             mon_cmd($vmid, "quit");
5754         }
5755     };
5756     my $err = $@;
5757
5758     if (!$err) {
5759         $timeout = 60 if !defined($timeout);
5760
5761         my $count = 0;
5762         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5763             $count++;
5764             sleep 1;
5765         }
5766
5767         if ($count >= $timeout) {
5768             if ($force) {
5769                 warn "VM still running - terminating now with SIGTERM\n";
5770                 kill 15, $pid;
5771             } else {
5772                 die "VM quit/powerdown failed - got timeout\n";
5773             }
5774         } else {
5775             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5776             return;
5777         }
5778     } else {
5779        if (!check_running($vmid, $nocheck)) {
5780             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5781             return;
5782        }
5783        if ($force) {
5784             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5785             kill 15, $pid;
5786         } else {
5787             die "VM quit/powerdown failed\n";
5788         }
5789     }
5790
5791     # wait again
5792     $timeout = 10;
5793
5794     my $count = 0;
5795     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5796         $count++;
5797         sleep 1;
5798     }
5799
5800     if ($count >= $timeout) {
5801         warn "VM still running - terminating now with SIGKILL\n";
5802         kill 9, $pid;
5803         sleep 1;
5804     }
5805
5806     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5807 }
5808
5809 # Note: use $nocheck to skip tests if VM configuration file exists.
5810 # We need that when migration VMs to other nodes (files already moved)
5811 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5812 sub vm_stop {
5813     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5814
5815     $force = 1 if !defined($force) && !$shutdown;
5816
5817     if ($migratedfrom){
5818         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5819         kill 15, $pid if $pid;
5820         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5821         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5822         return;
5823     }
5824
5825     PVE::QemuConfig->lock_config($vmid, sub {
5826         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5827    });
5828 }
5829
5830 sub vm_reboot {
5831     my ($vmid, $timeout) = @_;
5832
5833     PVE::QemuConfig->lock_config($vmid, sub {
5834         eval {
5835
5836             # only reboot if running, as qmeventd starts it again on a stop event
5837             return if !check_running($vmid);
5838
5839             create_reboot_request($vmid);
5840
5841             my $storecfg = PVE::Storage::config();
5842             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5843
5844         };
5845         if (my $err = $@) {
5846             # avoid that the next normal shutdown will be confused for a reboot
5847             clear_reboot_request($vmid);
5848             die $err;
5849         }
5850    });
5851 }
5852
5853 # note: if using the statestorage parameter, the caller has to check privileges
5854 sub vm_suspend {
5855     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5856
5857     my $conf;
5858     my $path;
5859     my $storecfg;
5860     my $vmstate;
5861
5862     PVE::QemuConfig->lock_config($vmid, sub {
5863
5864         $conf = PVE::QemuConfig->load_config($vmid);
5865
5866         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5867         PVE::QemuConfig->check_lock($conf)
5868             if !($skiplock || $is_backing_up);
5869
5870         die "cannot suspend to disk during backup\n"
5871             if $is_backing_up && $includestate;
5872
5873         if ($includestate) {
5874             $conf->{lock} = 'suspending';
5875             my $date = strftime("%Y-%m-%d", localtime(time()));
5876             $storecfg = PVE::Storage::config();
5877             if (!$statestorage) {
5878                 $statestorage = find_vmstate_storage($conf, $storecfg);
5879                 # check permissions for the storage
5880                 my $rpcenv = PVE::RPCEnvironment::get();
5881                 if ($rpcenv->{type} ne 'cli') {
5882                     my $authuser = $rpcenv->get_user();
5883                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5884                 }
5885             }
5886
5887
5888             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5889                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5890             $path = PVE::Storage::path($storecfg, $vmstate);
5891             PVE::QemuConfig->write_config($vmid, $conf);
5892         } else {
5893             mon_cmd($vmid, "stop");
5894         }
5895     });
5896
5897     if ($includestate) {
5898         # save vm state
5899         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5900
5901         eval {
5902             set_migration_caps($vmid, 1);
5903             mon_cmd($vmid, "savevm-start", statefile => $path);
5904             for(;;) {
5905                 my $state = mon_cmd($vmid, "query-savevm");
5906                 if (!$state->{status}) {
5907                     die "savevm not active\n";
5908                 } elsif ($state->{status} eq 'active') {
5909                     sleep(1);
5910                     next;
5911                 } elsif ($state->{status} eq 'completed') {
5912                     print "State saved, quitting\n";
5913                     last;
5914                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5915                     die "query-savevm failed with error '$state->{error}'\n"
5916                 } else {
5917                     die "query-savevm returned status '$state->{status}'\n";
5918                 }
5919             }
5920         };
5921         my $err = $@;
5922
5923         PVE::QemuConfig->lock_config($vmid, sub {
5924             $conf = PVE::QemuConfig->load_config($vmid);
5925             if ($err) {
5926                 # cleanup, but leave suspending lock, to indicate something went wrong
5927                 eval {
5928                     mon_cmd($vmid, "savevm-end");
5929                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5930                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5931                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5932                     PVE::QemuConfig->write_config($vmid, $conf);
5933                 };
5934                 warn $@ if $@;
5935                 die $err;
5936             }
5937
5938             die "lock changed unexpectedly\n"
5939                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5940
5941             mon_cmd($vmid, "quit");
5942             $conf->{lock} = 'suspended';
5943             PVE::QemuConfig->write_config($vmid, $conf);
5944         });
5945     }
5946 }
5947
5948 sub vm_resume {
5949     my ($vmid, $skiplock, $nocheck) = @_;
5950
5951     PVE::QemuConfig->lock_config($vmid, sub {
5952         my $res = mon_cmd($vmid, 'query-status');
5953         my $resume_cmd = 'cont';
5954         my $reset = 0;
5955
5956         if ($res->{status}) {
5957             return if $res->{status} eq 'running'; # job done, go home
5958             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5959             $reset = 1 if $res->{status} eq 'shutdown';
5960         }
5961
5962         if (!$nocheck) {
5963
5964             my $conf = PVE::QemuConfig->load_config($vmid);
5965
5966             PVE::QemuConfig->check_lock($conf)
5967                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5968         }
5969
5970         if ($reset) {
5971             # required if a VM shuts down during a backup and we get a resume
5972             # request before the backup finishes for example
5973             mon_cmd($vmid, "system_reset");
5974         }
5975         mon_cmd($vmid, $resume_cmd);
5976     });
5977 }
5978
5979 sub vm_sendkey {
5980     my ($vmid, $skiplock, $key) = @_;
5981
5982     PVE::QemuConfig->lock_config($vmid, sub {
5983
5984         my $conf = PVE::QemuConfig->load_config($vmid);
5985
5986         # there is no qmp command, so we use the human monitor command
5987         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5988         die $res if $res ne '';
5989     });
5990 }
5991
5992 # vzdump restore implementaion
5993
5994 sub tar_archive_read_firstfile {
5995     my $archive = shift;
5996
5997     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5998
5999     # try to detect archive type first
6000     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6001         die "unable to open file '$archive'\n";
6002     my $firstfile = <$fh>;
6003     kill 15, $pid;
6004     close $fh;
6005
6006     die "ERROR: archive contaions no data\n" if !$firstfile;
6007     chomp $firstfile;
6008
6009     return $firstfile;
6010 }
6011
6012 sub tar_restore_cleanup {
6013     my ($storecfg, $statfile) = @_;
6014
6015     print STDERR "starting cleanup\n";
6016
6017     if (my $fd = IO::File->new($statfile, "r")) {
6018         while (defined(my $line = <$fd>)) {
6019             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6020                 my $volid = $2;
6021                 eval {
6022                     if ($volid =~ m|^/|) {
6023                         unlink $volid || die 'unlink failed\n';
6024                     } else {
6025                         PVE::Storage::vdisk_free($storecfg, $volid);
6026                     }
6027                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6028                 };
6029                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6030             } else {
6031                 print STDERR "unable to parse line in statfile - $line";
6032             }
6033         }
6034         $fd->close();
6035     }
6036 }
6037
6038 sub restore_file_archive {
6039     my ($archive, $vmid, $user, $opts) = @_;
6040
6041     return restore_vma_archive($archive, $vmid, $user, $opts)
6042         if $archive eq '-';
6043
6044     my $info = PVE::Storage::archive_info($archive);
6045     my $format = $opts->{format} // $info->{format};
6046     my $comp = $info->{compression};
6047
6048     # try to detect archive format
6049     if ($format eq 'tar') {
6050         return restore_tar_archive($archive, $vmid, $user, $opts);
6051     } else {
6052         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6053     }
6054 }
6055
6056 # hepler to remove disks that will not be used after restore
6057 my $restore_cleanup_oldconf = sub {
6058     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
6059
6060     PVE::QemuConfig->foreach_volume($oldconf, sub {
6061         my ($ds, $drive) = @_;
6062
6063         return if drive_is_cdrom($drive, 1);
6064
6065         my $volid = $drive->{file};
6066         return if !$volid || $volid =~ m|^/|;
6067
6068         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
6069         return if !$path || !$owner || ($owner != $vmid);
6070
6071         # Note: only delete disk we want to restore
6072         # other volumes will become unused
6073         if ($virtdev_hash->{$ds}) {
6074             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
6075             if (my $err = $@) {
6076                 warn $err;
6077             }
6078         }
6079     });
6080
6081     # delete vmstate files, after the restore we have no snapshots anymore
6082     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6083         my $snap = $oldconf->{snapshots}->{$snapname};
6084         if ($snap->{vmstate}) {
6085             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
6086             if (my $err = $@) {
6087                 warn $err;
6088             }
6089         }
6090     }
6091 };
6092
6093 # Helper to parse vzdump backup device hints
6094 #
6095 # $rpcenv: Environment, used to ckeck storage permissions
6096 # $user: User ID, to check storage permissions
6097 # $storecfg: Storage configuration
6098 # $fh: the file handle for reading the configuration
6099 # $devinfo: should contain device sizes for all backu-up'ed devices
6100 # $options: backup options (pool, default storage)
6101 #
6102 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
6103 my $parse_backup_hints = sub {
6104     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
6105
6106     my $virtdev_hash = {};
6107
6108     while (defined(my $line = <$fh>)) {
6109         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6110             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6111             die "archive does not contain data for drive '$virtdev'\n"
6112                 if !$devinfo->{$devname};
6113
6114             if (defined($options->{storage})) {
6115                 $storeid = $options->{storage} || 'local';
6116             } elsif (!$storeid) {
6117                 $storeid = 'local';
6118             }
6119             $format = 'raw' if !$format;
6120             $devinfo->{$devname}->{devname} = $devname;
6121             $devinfo->{$devname}->{virtdev} = $virtdev;
6122             $devinfo->{$devname}->{format} = $format;
6123             $devinfo->{$devname}->{storeid} = $storeid;
6124
6125             # check permission on storage
6126             my $pool = $options->{pool}; # todo: do we need that?
6127             if ($user ne 'root@pam') {
6128                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6129             }
6130
6131             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6132         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6133             my $virtdev = $1;
6134             my $drive = parse_drive($virtdev, $2);
6135             if (drive_is_cloudinit($drive)) {
6136                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6137                 $storeid = $options->{storage} if defined ($options->{storage});
6138                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6139                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6140
6141                 $virtdev_hash->{$virtdev} = {
6142                     format => $format,
6143                     storeid => $storeid,
6144                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6145                     is_cloudinit => 1,
6146                 };
6147             }
6148         }
6149     }
6150
6151     return $virtdev_hash;
6152 };
6153
6154 # Helper to allocate and activate all volumes required for a restore
6155 #
6156 # $storecfg: Storage configuration
6157 # $virtdev_hash: as returned by parse_backup_hints()
6158 #
6159 # Returns: { $virtdev => $volid }
6160 my $restore_allocate_devices = sub {
6161     my ($storecfg, $virtdev_hash, $vmid) = @_;
6162
6163     my $map = {};
6164     foreach my $virtdev (sort keys %$virtdev_hash) {
6165         my $d = $virtdev_hash->{$virtdev};
6166         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6167         my $storeid = $d->{storeid};
6168         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6169
6170         # test if requested format is supported
6171         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6172         my $supported = grep { $_ eq $d->{format} } @$validFormats;
6173         $d->{format} = $defFormat if !$supported;
6174
6175         my $name;
6176         if ($d->{is_cloudinit}) {
6177             $name = "vm-$vmid-cloudinit";
6178             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6179             if ($scfg->{path}) {
6180                 $name .= ".$d->{format}";
6181             }
6182         }
6183
6184         my $volid = PVE::Storage::vdisk_alloc(
6185             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6186
6187         print STDERR "new volume ID is '$volid'\n";
6188         $d->{volid} = $volid;
6189
6190         PVE::Storage::activate_volumes($storecfg, [$volid]);
6191
6192         $map->{$virtdev} = $volid;
6193     }
6194
6195     return $map;
6196 };
6197
6198 sub restore_update_config_line {
6199     my ($cookie, $map, $line, $unique) = @_;
6200
6201     return '' if $line =~ m/^\#qmdump\#/;
6202     return '' if $line =~ m/^\#vzdump\#/;
6203     return '' if $line =~ m/^lock:/;
6204     return '' if $line =~ m/^unused\d+:/;
6205     return '' if $line =~ m/^parent:/;
6206
6207     my $res = '';
6208
6209     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6210     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6211         # try to convert old 1.X settings
6212         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6213         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6214             my ($model, $macaddr) = split(/\=/, $devconfig);
6215             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6216             my $net = {
6217                 model => $model,
6218                 bridge => "vmbr$ind",
6219                 macaddr => $macaddr,
6220             };
6221             my $netstr = print_net($net);
6222
6223             $res .= "net$cookie->{netcount}: $netstr\n";
6224             $cookie->{netcount}++;
6225         }
6226     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6227         my ($id, $netstr) = ($1, $2);
6228         my $net = parse_net($netstr);
6229         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6230         $netstr = print_net($net);
6231         $res .= "$id: $netstr\n";
6232     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk|tpmstate)\d+):\s*(\S+)\s*$/) {
6233         my $virtdev = $1;
6234         my $value = $3;
6235         my $di = parse_drive($virtdev, $value);
6236         if (defined($di->{backup}) && !$di->{backup}) {
6237             $res .= "#$line";
6238         } elsif ($map->{$virtdev}) {
6239             delete $di->{format}; # format can change on restore
6240             $di->{file} = $map->{$virtdev};
6241             $value = print_drive($di);
6242             $res .= "$virtdev: $value\n";
6243         } else {
6244             $res .= $line;
6245         }
6246     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6247         my $vmgenid = $1;
6248         if ($vmgenid ne '0') {
6249             # always generate a new vmgenid if there was a valid one setup
6250             $vmgenid = generate_uuid();
6251         }
6252         $res .= "vmgenid: $vmgenid\n";
6253     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6254         my ($uuid, $uuid_str);
6255         UUID::generate($uuid);
6256         UUID::unparse($uuid, $uuid_str);
6257         my $smbios1 = parse_smbios1($2);
6258         $smbios1->{uuid} = $uuid_str;
6259         $res .= $1.print_smbios1($smbios1)."\n";
6260     } else {
6261         $res .= $line;
6262     }
6263
6264     return $res;
6265 }
6266
6267 my $restore_deactivate_volumes = sub {
6268     my ($storecfg, $devinfo) = @_;
6269
6270     my $vollist = [];
6271     foreach my $devname (keys %$devinfo) {
6272         my $volid = $devinfo->{$devname}->{volid};
6273         push @$vollist, $volid if $volid;
6274     }
6275
6276     PVE::Storage::deactivate_volumes($storecfg, $vollist);
6277 };
6278
6279 my $restore_destroy_volumes = sub {
6280     my ($storecfg, $devinfo) = @_;
6281
6282     foreach my $devname (keys %$devinfo) {
6283         my $volid = $devinfo->{$devname}->{volid};
6284         next if !$volid;
6285         eval {
6286             if ($volid =~ m|^/|) {
6287                 unlink $volid || die 'unlink failed\n';
6288             } else {
6289                 PVE::Storage::vdisk_free($storecfg, $volid);
6290             }
6291             print STDERR "temporary volume '$volid' sucessfuly removed\n";
6292         };
6293         print STDERR "unable to cleanup '$volid' - $@" if $@;
6294     }
6295 };
6296
6297 sub scan_volids {
6298     my ($cfg, $vmid) = @_;
6299
6300     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid, undef, 'images');
6301
6302     my $volid_hash = {};
6303     foreach my $storeid (keys %$info) {
6304         foreach my $item (@{$info->{$storeid}}) {
6305             next if !($item->{volid} && $item->{size});
6306             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6307             $volid_hash->{$item->{volid}} = $item;
6308         }
6309     }
6310
6311     return $volid_hash;
6312 }
6313
6314 sub update_disk_config {
6315     my ($vmid, $conf, $volid_hash) = @_;
6316
6317     my $changes;
6318     my $prefix = "VM $vmid";
6319
6320     # used and unused disks
6321     my $referenced = {};
6322
6323     # Note: it is allowed to define multiple storages with same path (alias), so
6324     # we need to check both 'volid' and real 'path' (two different volid can point
6325     # to the same path).
6326
6327     my $referencedpath = {};
6328
6329     # update size info
6330     PVE::QemuConfig->foreach_volume($conf, sub {
6331         my ($opt, $drive) = @_;
6332
6333         my $volid = $drive->{file};
6334         return if !$volid;
6335         my $volume = $volid_hash->{$volid};
6336
6337         # mark volid as "in-use" for next step
6338         $referenced->{$volid} = 1;
6339         if ($volume && (my $path = $volume->{path})) {
6340             $referencedpath->{$path} = 1;
6341         }
6342
6343         return if drive_is_cdrom($drive);
6344         return if !$volume;
6345
6346         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6347         if (defined($updated)) {
6348             $changes = 1;
6349             $conf->{$opt} = print_drive($updated);
6350             print "$prefix ($opt): $msg\n";
6351         }
6352     });
6353
6354     # remove 'unusedX' entry if volume is used
6355     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6356         my ($opt, $drive) = @_;
6357
6358         my $volid = $drive->{file};
6359         return if !$volid;
6360
6361         my $path;
6362         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6363         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6364             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6365             $changes = 1;
6366             delete $conf->{$opt};
6367         }
6368
6369         $referenced->{$volid} = 1;
6370         $referencedpath->{$path} = 1 if $path;
6371     });
6372
6373     foreach my $volid (sort keys %$volid_hash) {
6374         next if $volid =~ m/vm-$vmid-state-/;
6375         next if $referenced->{$volid};
6376         my $path = $volid_hash->{$volid}->{path};
6377         next if !$path; # just to be sure
6378         next if $referencedpath->{$path};
6379         $changes = 1;
6380         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6381         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6382         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6383     }
6384
6385     return $changes;
6386 }
6387
6388 sub rescan {
6389     my ($vmid, $nolock, $dryrun) = @_;
6390
6391     my $cfg = PVE::Storage::config();
6392
6393     print "rescan volumes...\n";
6394     my $volid_hash = scan_volids($cfg, $vmid);
6395
6396     my $updatefn =  sub {
6397         my ($vmid) = @_;
6398
6399         my $conf = PVE::QemuConfig->load_config($vmid);
6400
6401         PVE::QemuConfig->check_lock($conf);
6402
6403         my $vm_volids = {};
6404         foreach my $volid (keys %$volid_hash) {
6405             my $info = $volid_hash->{$volid};
6406             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6407         }
6408
6409         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6410
6411         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6412     };
6413
6414     if (defined($vmid)) {
6415         if ($nolock) {
6416             &$updatefn($vmid);
6417         } else {
6418             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6419         }
6420     } else {
6421         my $vmlist = config_list();
6422         foreach my $vmid (keys %$vmlist) {
6423             if ($nolock) {
6424                 &$updatefn($vmid);
6425             } else {
6426                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6427             }
6428         }
6429     }
6430 }
6431
6432 sub restore_proxmox_backup_archive {
6433     my ($archive, $vmid, $user, $options) = @_;
6434
6435     my $storecfg = PVE::Storage::config();
6436
6437     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6438     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6439
6440     my $fingerprint = $scfg->{fingerprint};
6441     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6442
6443     my $repo = PVE::PBSClient::get_repository($scfg);
6444
6445     # This is only used for `pbs-restore` and the QEMU PBS driver (live-restore)
6446     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6447     local $ENV{PBS_PASSWORD} = $password;
6448     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6449
6450     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6451         PVE::Storage::parse_volname($storecfg, $archive);
6452
6453     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6454
6455     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6456
6457     my $tmpdir = "/var/tmp/vzdumptmp$$";
6458     rmtree $tmpdir;
6459     mkpath $tmpdir;
6460
6461     my $conffile = PVE::QemuConfig->config_file($vmid);
6462      # disable interrupts (always do cleanups)
6463     local $SIG{INT} =
6464         local $SIG{TERM} =
6465         local $SIG{QUIT} =
6466         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6467
6468     # Note: $oldconf is undef if VM does not exists
6469     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6470     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6471     my $new_conf_raw = '';
6472
6473     my $rpcenv = PVE::RPCEnvironment::get();
6474     my $devinfo = {};
6475
6476     eval {
6477         # enable interrupts
6478         local $SIG{INT} =
6479             local $SIG{TERM} =
6480             local $SIG{QUIT} =
6481             local $SIG{HUP} =
6482             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6483
6484         my $cfgfn = "$tmpdir/qemu-server.conf";
6485         my $firewall_config_fn = "$tmpdir/fw.conf";
6486         my $index_fn = "$tmpdir/index.json";
6487
6488         my $cmd = "restore";
6489
6490         my $param = [$pbs_backup_name, "index.json", $index_fn];
6491         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6492         my $index = PVE::Tools::file_get_contents($index_fn);
6493         $index = decode_json($index);
6494
6495         # print Dumper($index);
6496         foreach my $info (@{$index->{files}}) {
6497             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6498                 my $devname = $1;
6499                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6500                     $devinfo->{$devname}->{size} = $1;
6501                 } else {
6502                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6503                 }
6504             }
6505         }
6506
6507         my $is_qemu_server_backup = scalar(
6508             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6509         );
6510         if (!$is_qemu_server_backup) {
6511             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6512         }
6513         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6514
6515         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6516         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6517
6518         if ($has_firewall_config) {
6519             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6520             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6521
6522             my $pve_firewall_dir = '/etc/pve/firewall';
6523             mkdir $pve_firewall_dir; # make sure the dir exists
6524             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6525         }
6526
6527         my $fh = IO::File->new($cfgfn, "r") ||
6528             die "unable to read qemu-server.conf - $!\n";
6529
6530         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6531
6532         # fixme: rate limit?
6533
6534         # create empty/temp config
6535         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6536
6537         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6538
6539         # allocate volumes
6540         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6541
6542         foreach my $virtdev (sort keys %$virtdev_hash) {
6543             my $d = $virtdev_hash->{$virtdev};
6544             next if $d->{is_cloudinit}; # no need to restore cloudinit
6545
6546             # this fails if storage is unavailable
6547             my $volid = $d->{volid};
6548             my $path = PVE::Storage::path($storecfg, $volid);
6549
6550             # for live-restore we only want to preload the efidisk and TPM state
6551             next if $options->{live} && $virtdev ne 'efidisk0' && $virtdev ne 'tpmstate0';
6552
6553             my $pbs_restore_cmd = [
6554                 '/usr/bin/pbs-restore',
6555                 '--repository', $repo,
6556                 $pbs_backup_name,
6557                 "$d->{devname}.img.fidx",
6558                 $path,
6559                 '--verbose',
6560                 ];
6561
6562             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6563             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6564
6565             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6566                 push @$pbs_restore_cmd, '--skip-zero';
6567             }
6568
6569             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6570             print "restore proxmox backup image: $dbg_cmdstring\n";
6571             run_command($pbs_restore_cmd);
6572         }
6573
6574         $fh->seek(0, 0) || die "seek failed - $!\n";
6575
6576         my $cookie = { netcount => 0 };
6577         while (defined(my $line = <$fh>)) {
6578             $new_conf_raw .= restore_update_config_line(
6579                 $cookie,
6580                 $map,
6581                 $line,
6582                 $options->{unique},
6583             );
6584         }
6585
6586         $fh->close();
6587     };
6588     my $err = $@;
6589
6590     if ($err || !$options->{live}) {
6591         $restore_deactivate_volumes->($storecfg, $devinfo);
6592     }
6593
6594     rmtree $tmpdir;
6595
6596     if ($err) {
6597         $restore_destroy_volumes->($storecfg, $devinfo);
6598         die $err;
6599     }
6600
6601     if ($options->{live}) {
6602         # keep lock during live-restore
6603         $new_conf_raw .= "\nlock: create";
6604     }
6605
6606     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6607
6608     PVE::Cluster::cfs_update(); # make sure we read new file
6609
6610     eval { rescan($vmid, 1); };
6611     warn $@ if $@;
6612
6613     PVE::AccessControl::add_vm_to_pool($vmid, $options->{pool}) if $options->{pool};
6614
6615     if ($options->{live}) {
6616         # enable interrupts
6617         local $SIG{INT} =
6618             local $SIG{TERM} =
6619             local $SIG{QUIT} =
6620             local $SIG{HUP} =
6621             local $SIG{PIPE} = sub { die "got signal ($!) - abort\n"; };
6622
6623         my $conf = PVE::QemuConfig->load_config($vmid);
6624         die "cannot do live-restore for template\n" if PVE::QemuConfig->is_template($conf);
6625
6626         # these special drives are already restored before start
6627         delete $devinfo->{'drive-efidisk0'};
6628         delete $devinfo->{'drive-tpmstate0-backup'};
6629         pbs_live_restore($vmid, $conf, $storecfg, $devinfo, $repo, $keyfile, $pbs_backup_name);
6630
6631         PVE::QemuConfig->remove_lock($vmid, "create");
6632     }
6633 }
6634
6635 sub pbs_live_restore {
6636     my ($vmid, $conf, $storecfg, $restored_disks, $repo, $keyfile, $snap) = @_;
6637
6638     print "starting VM for live-restore\n";
6639     print "repository: '$repo', snapshot: '$snap'\n";
6640
6641     my $pbs_backing = {};
6642     for my $ds (keys %$restored_disks) {
6643         $ds =~ m/^drive-(.*)$/;
6644         my $confname = $1;
6645         $pbs_backing->{$confname} = {
6646             repository => $repo,
6647             snapshot => $snap,
6648             archive => "$ds.img.fidx",
6649         };
6650         $pbs_backing->{$confname}->{keyfile} = $keyfile if -e $keyfile;
6651
6652         my $drive = parse_drive($confname, $conf->{$confname});
6653         print "restoring '$ds' to '$drive->{file}'\n";
6654     }
6655
6656     my $drives_streamed = 0;
6657     eval {
6658         # make sure HA doesn't interrupt our restore by stopping the VM
6659         if (PVE::HA::Config::vm_is_ha_managed($vmid)) {
6660             run_command(['ha-manager', 'set',  "vm:$vmid", '--state', 'started']);
6661         }
6662
6663         # start VM with backing chain pointing to PBS backup, environment vars for PBS driver
6664         # in QEMU (PBS_PASSWORD and PBS_FINGERPRINT) are already set by our caller
6665         vm_start_nolock($storecfg, $vmid, $conf, {paused => 1, 'pbs-backing' => $pbs_backing}, {});
6666
6667         my $qmeventd_fd = register_qmeventd_handle($vmid);
6668
6669         # begin streaming, i.e. data copy from PBS to target disk for every vol,
6670         # this will effectively collapse the backing image chain consisting of
6671         # [target <- alloc-track -> PBS snapshot] to just [target] (alloc-track
6672         # removes itself once all backing images vanish with 'auto-remove=on')
6673         my $jobs = {};
6674         for my $ds (sort keys %$restored_disks) {
6675             my $job_id = "restore-$ds";
6676             mon_cmd($vmid, 'block-stream',
6677                 'job-id' => $job_id,
6678                 device => "$ds",
6679             );
6680             $jobs->{$job_id} = {};
6681         }
6682
6683         mon_cmd($vmid, 'cont');
6684         qemu_drive_mirror_monitor($vmid, undef, $jobs, 'auto', 0, 'stream');
6685
6686         print "restore-drive jobs finished successfully, removing all tracking block devices"
6687             ." to disconnect from Proxmox Backup Server\n";
6688
6689         for my $ds (sort keys %$restored_disks) {
6690             mon_cmd($vmid, 'blockdev-del', 'node-name' => "$ds-pbs");
6691         }
6692
6693         close($qmeventd_fd);
6694     };
6695
6696     my $err = $@;
6697
6698     if ($err) {
6699         warn "An error occured during live-restore: $err\n";
6700         _do_vm_stop($storecfg, $vmid, 1, 1, 10, 0, 1);
6701         die "live-restore failed\n";
6702     }
6703 }
6704
6705 sub restore_vma_archive {
6706     my ($archive, $vmid, $user, $opts, $comp) = @_;
6707
6708     my $readfrom = $archive;
6709
6710     my $cfg = PVE::Storage::config();
6711     my $commands = [];
6712     my $bwlimit = $opts->{bwlimit};
6713
6714     my $dbg_cmdstring = '';
6715     my $add_pipe = sub {
6716         my ($cmd) = @_;
6717         push @$commands, $cmd;
6718         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6719         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6720         $readfrom = '-';
6721     };
6722
6723     my $input = undef;
6724     if ($archive eq '-') {
6725         $input = '<&STDIN';
6726     } else {
6727         # If we use a backup from a PVE defined storage we also consider that
6728         # storage's rate limit:
6729         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6730         if (defined($volid)) {
6731             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6732             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6733             if ($readlimit) {
6734                 print STDERR "applying read rate limit: $readlimit\n";
6735                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6736                 $add_pipe->($cstream);
6737             }
6738         }
6739     }
6740
6741     if ($comp) {
6742         my $info = PVE::Storage::decompressor_info('vma', $comp);
6743         my $cmd = $info->{decompressor};
6744         push @$cmd, $readfrom;
6745         $add_pipe->($cmd);
6746     }
6747
6748     my $tmpdir = "/var/tmp/vzdumptmp$$";
6749     rmtree $tmpdir;
6750
6751     # disable interrupts (always do cleanups)
6752     local $SIG{INT} =
6753         local $SIG{TERM} =
6754         local $SIG{QUIT} =
6755         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6756
6757     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6758     POSIX::mkfifo($mapfifo, 0600);
6759     my $fifofh;
6760     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6761
6762     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6763
6764     my $oldtimeout;
6765     my $timeout = 5;
6766
6767     my $devinfo = {};
6768
6769     my $rpcenv = PVE::RPCEnvironment::get();
6770
6771     my $conffile = PVE::QemuConfig->config_file($vmid);
6772
6773     # Note: $oldconf is undef if VM does not exist
6774     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6775     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6776     my $new_conf_raw = '';
6777
6778     my %storage_limits;
6779
6780     my $print_devmap = sub {
6781         my $cfgfn = "$tmpdir/qemu-server.conf";
6782
6783         # we can read the config - that is already extracted
6784         my $fh = IO::File->new($cfgfn, "r") ||
6785             die "unable to read qemu-server.conf - $!\n";
6786
6787         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6788         if (-f $fwcfgfn) {
6789             my $pve_firewall_dir = '/etc/pve/firewall';
6790             mkdir $pve_firewall_dir; # make sure the dir exists
6791             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6792         }
6793
6794         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6795
6796         foreach my $info (values %{$virtdev_hash}) {
6797             my $storeid = $info->{storeid};
6798             next if defined($storage_limits{$storeid});
6799
6800             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6801             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6802             $storage_limits{$storeid} = $limit * 1024;
6803         }
6804
6805         foreach my $devname (keys %$devinfo) {
6806             die "found no device mapping information for device '$devname'\n"
6807                 if !$devinfo->{$devname}->{virtdev};
6808         }
6809
6810         # create empty/temp config
6811         if ($oldconf) {
6812             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6813             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6814         }
6815
6816         # allocate volumes
6817         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6818
6819         # print restore information to $fifofh
6820         foreach my $virtdev (sort keys %$virtdev_hash) {
6821             my $d = $virtdev_hash->{$virtdev};
6822             next if $d->{is_cloudinit}; # no need to restore cloudinit
6823
6824             my $storeid = $d->{storeid};
6825             my $volid = $d->{volid};
6826
6827             my $map_opts = '';
6828             if (my $limit = $storage_limits{$storeid}) {
6829                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6830             }
6831
6832             my $write_zeros = 1;
6833             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6834                 $write_zeros = 0;
6835             }
6836
6837             my $path = PVE::Storage::path($cfg, $volid);
6838
6839             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6840
6841             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6842         }
6843
6844         $fh->seek(0, 0) || die "seek failed - $!\n";
6845
6846         my $cookie = { netcount => 0 };
6847         while (defined(my $line = <$fh>)) {
6848             $new_conf_raw .= restore_update_config_line(
6849                 $cookie,
6850                 $map,
6851                 $line,
6852                 $opts->{unique},
6853             );
6854         }
6855
6856         $fh->close();
6857     };
6858
6859     eval {
6860         # enable interrupts
6861         local $SIG{INT} =
6862             local $SIG{TERM} =
6863             local $SIG{QUIT} =
6864             local $SIG{HUP} =
6865             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6866         local $SIG{ALRM} = sub { die "got timeout\n"; };
6867
6868         $oldtimeout = alarm($timeout);
6869
6870         my $parser = sub {
6871             my $line = shift;
6872
6873             print "$line\n";
6874
6875             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6876                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6877                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6878             } elsif ($line =~ m/^CTIME: /) {
6879                 # we correctly received the vma config, so we can disable
6880                 # the timeout now for disk allocation (set to 10 minutes, so
6881                 # that we always timeout if something goes wrong)
6882                 alarm(600);
6883                 &$print_devmap();
6884                 print $fifofh "done\n";
6885                 my $tmp = $oldtimeout || 0;
6886                 $oldtimeout = undef;
6887                 alarm($tmp);
6888                 close($fifofh);
6889                 $fifofh = undef;
6890             }
6891         };
6892
6893         print "restore vma archive: $dbg_cmdstring\n";
6894         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6895     };
6896     my $err = $@;
6897
6898     alarm($oldtimeout) if $oldtimeout;
6899
6900     $restore_deactivate_volumes->($cfg, $devinfo);
6901
6902     close($fifofh) if $fifofh;
6903     unlink $mapfifo;
6904     rmtree $tmpdir;
6905
6906     if ($err) {
6907         $restore_destroy_volumes->($cfg, $devinfo);
6908         die $err;
6909     }
6910
6911     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6912
6913     PVE::Cluster::cfs_update(); # make sure we read new file
6914
6915     eval { rescan($vmid, 1); };
6916     warn $@ if $@;
6917
6918     PVE::AccessControl::add_vm_to_pool($vmid, $opts->{pool}) if $opts->{pool};
6919 }
6920
6921 sub restore_tar_archive {
6922     my ($archive, $vmid, $user, $opts) = @_;
6923
6924     if ($archive ne '-') {
6925         my $firstfile = tar_archive_read_firstfile($archive);
6926         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6927             if $firstfile ne 'qemu-server.conf';
6928     }
6929
6930     my $storecfg = PVE::Storage::config();
6931
6932     # avoid zombie disks when restoring over an existing VM -> cleanup first
6933     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6934     # skiplock=1 because qmrestore has set the 'create' lock itself already
6935     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6936     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6937
6938     my $tocmd = "/usr/lib/qemu-server/qmextract";
6939
6940     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6941     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6942     $tocmd .= ' --prealloc' if $opts->{prealloc};
6943     $tocmd .= ' --info' if $opts->{info};
6944
6945     # tar option "xf" does not autodetect compression when read from STDIN,
6946     # so we pipe to zcat
6947     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6948         PVE::Tools::shellquote("--to-command=$tocmd");
6949
6950     my $tmpdir = "/var/tmp/vzdumptmp$$";
6951     mkpath $tmpdir;
6952
6953     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6954     local $ENV{VZDUMP_VMID} = $vmid;
6955     local $ENV{VZDUMP_USER} = $user;
6956
6957     my $conffile = PVE::QemuConfig->config_file($vmid);
6958     my $new_conf_raw = '';
6959
6960     # disable interrupts (always do cleanups)
6961     local $SIG{INT} =
6962         local $SIG{TERM} =
6963         local $SIG{QUIT} =
6964         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6965
6966     eval {
6967         # enable interrupts
6968         local $SIG{INT} =
6969             local $SIG{TERM} =
6970             local $SIG{QUIT} =
6971             local $SIG{HUP} =
6972             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6973
6974         if ($archive eq '-') {
6975             print "extracting archive from STDIN\n";
6976             run_command($cmd, input => "<&STDIN");
6977         } else {
6978             print "extracting archive '$archive'\n";
6979             run_command($cmd);
6980         }
6981
6982         return if $opts->{info};
6983
6984         # read new mapping
6985         my $map = {};
6986         my $statfile = "$tmpdir/qmrestore.stat";
6987         if (my $fd = IO::File->new($statfile, "r")) {
6988             while (defined (my $line = <$fd>)) {
6989                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6990                     $map->{$1} = $2 if $1;
6991                 } else {
6992                     print STDERR "unable to parse line in statfile - $line\n";
6993                 }
6994             }
6995             $fd->close();
6996         }
6997
6998         my $confsrc = "$tmpdir/qemu-server.conf";
6999
7000         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
7001
7002         my $cookie = { netcount => 0 };
7003         while (defined (my $line = <$srcfd>)) {
7004             $new_conf_raw .= restore_update_config_line(
7005                 $cookie,
7006                 $map,
7007                 $line,
7008                 $opts->{unique},
7009             );
7010         }
7011
7012         $srcfd->close();
7013     };
7014     if (my $err = $@) {
7015         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
7016         die $err;
7017     }
7018
7019     rmtree $tmpdir;
7020
7021     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
7022
7023     PVE::Cluster::cfs_update(); # make sure we read new file
7024
7025     eval { rescan($vmid, 1); };
7026     warn $@ if $@;
7027 };
7028
7029 sub foreach_storage_used_by_vm {
7030     my ($conf, $func) = @_;
7031
7032     my $sidhash = {};
7033
7034     PVE::QemuConfig->foreach_volume($conf, sub {
7035         my ($ds, $drive) = @_;
7036         return if drive_is_cdrom($drive);
7037
7038         my $volid = $drive->{file};
7039
7040         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
7041         $sidhash->{$sid} = $sid if $sid;
7042     });
7043
7044     foreach my $sid (sort keys %$sidhash) {
7045         &$func($sid);
7046     }
7047 }
7048
7049 my $qemu_snap_storage = {
7050     rbd => 1,
7051 };
7052 sub do_snapshots_with_qemu {
7053     my ($storecfg, $volid, $deviceid) = @_;
7054
7055     return if $deviceid =~ m/tpmstate0/;
7056
7057     my $storage_name = PVE::Storage::parse_volume_id($volid);
7058     my $scfg = $storecfg->{ids}->{$storage_name};
7059     die "could not find storage '$storage_name'\n" if !defined($scfg);
7060
7061     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
7062         return 1;
7063     }
7064
7065     if ($volid =~ m/\.(qcow2|qed)$/){
7066         return 1;
7067     }
7068
7069     return;
7070 }
7071
7072 sub qga_check_running {
7073     my ($vmid, $nowarn) = @_;
7074
7075     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
7076     if ($@) {
7077         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
7078         return 0;
7079     }
7080     return 1;
7081 }
7082
7083 sub template_create {
7084     my ($vmid, $conf, $disk) = @_;
7085
7086     my $storecfg = PVE::Storage::config();
7087
7088     PVE::QemuConfig->foreach_volume($conf, sub {
7089         my ($ds, $drive) = @_;
7090
7091         return if drive_is_cdrom($drive);
7092         return if $disk && $ds ne $disk;
7093
7094         my $volid = $drive->{file};
7095         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
7096
7097         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
7098         $drive->{file} = $voliddst;
7099         $conf->{$ds} = print_drive($drive);
7100         PVE::QemuConfig->write_config($vmid, $conf);
7101     });
7102 }
7103
7104 sub convert_iscsi_path {
7105     my ($path) = @_;
7106
7107     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
7108         my $portal = $1;
7109         my $target = $2;
7110         my $lun = $3;
7111
7112         my $initiator_name = get_initiator_name();
7113
7114         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
7115                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
7116     }
7117
7118     die "cannot convert iscsi path '$path', unkown format\n";
7119 }
7120
7121 sub qemu_img_convert {
7122     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
7123
7124     my $storecfg = PVE::Storage::config();
7125     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
7126     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
7127
7128     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
7129
7130     my $cachemode;
7131     my $src_path;
7132     my $src_is_iscsi = 0;
7133     my $src_format;
7134
7135     if ($src_storeid) {
7136         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
7137         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
7138         $src_format = qemu_img_format($src_scfg, $src_volname);
7139         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
7140         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
7141         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
7142     } elsif (-f $src_volid) {
7143         $src_path = $src_volid;
7144         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7145             $src_format = $1;
7146         }
7147     }
7148
7149     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
7150
7151     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7152     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
7153     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7154     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
7155
7156     my $cmd = [];
7157     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
7158     push @$cmd, '-l', "snapshot.name=$snapname"
7159         if $snapname && $src_format && $src_format eq "qcow2";
7160     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
7161     push @$cmd, '-T', $cachemode if defined($cachemode);
7162
7163     if ($src_is_iscsi) {
7164         push @$cmd, '--image-opts';
7165         $src_path = convert_iscsi_path($src_path);
7166     } elsif ($src_format) {
7167         push @$cmd, '-f', $src_format;
7168     }
7169
7170     if ($dst_is_iscsi) {
7171         push @$cmd, '--target-image-opts';
7172         $dst_path = convert_iscsi_path($dst_path);
7173     } else {
7174         push @$cmd, '-O', $dst_format;
7175     }
7176
7177     push @$cmd, $src_path;
7178
7179     if (!$dst_is_iscsi && $is_zero_initialized) {
7180         push @$cmd, "zeroinit:$dst_path";
7181     } else {
7182         push @$cmd, $dst_path;
7183     }
7184
7185     my $parser = sub {
7186         my $line = shift;
7187         if($line =~ m/\((\S+)\/100\%\)/){
7188             my $percent = $1;
7189             my $transferred = int($size * $percent / 100);
7190             my $total_h = render_bytes($size, 1);
7191             my $transferred_h = render_bytes($transferred, 1);
7192
7193             print "transferred $transferred_h of $total_h ($percent%)\n";
7194         }
7195
7196     };
7197
7198     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
7199     my $err = $@;
7200     die "copy failed: $err" if $err;
7201 }
7202
7203 sub qemu_img_format {
7204     my ($scfg, $volname) = @_;
7205
7206     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
7207         return $1;
7208     } else {
7209         return "raw";
7210     }
7211 }
7212
7213 sub qemu_drive_mirror {
7214     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
7215
7216     $jobs = {} if !$jobs;
7217
7218     my $qemu_target;
7219     my $format;
7220     $jobs->{"drive-$drive"} = {};
7221
7222     if ($dst_volid =~ /^nbd:/) {
7223         $qemu_target = $dst_volid;
7224         $format = "nbd";
7225     } else {
7226         my $storecfg = PVE::Storage::config();
7227         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
7228
7229         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
7230
7231         $format = qemu_img_format($dst_scfg, $dst_volname);
7232
7233         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
7234
7235         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
7236     }
7237
7238     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
7239     $opts->{format} = $format if $format;
7240
7241     if (defined($src_bitmap)) {
7242         $opts->{sync} = 'incremental';
7243         $opts->{bitmap} = $src_bitmap;
7244         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
7245     }
7246
7247     if (defined($bwlimit)) {
7248         $opts->{speed} = $bwlimit * 1024;
7249         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
7250     } else {
7251         print "drive mirror is starting for drive-$drive\n";
7252     }
7253
7254     # if a job already runs for this device we get an error, catch it for cleanup
7255     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
7256     if (my $err = $@) {
7257         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7258         warn "$@\n" if $@;
7259         die "mirroring error: $err\n";
7260     }
7261
7262     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
7263 }
7264
7265 # $completion can be either
7266 # 'complete': wait until all jobs are ready, block-job-complete them (default)
7267 # 'cancel': wait until all jobs are ready, block-job-cancel them
7268 # 'skip': wait until all jobs are ready, return with block jobs in ready state
7269 # 'auto': wait until all jobs disappear, only use for jobs which complete automatically
7270 sub qemu_drive_mirror_monitor {
7271     my ($vmid, $vmiddst, $jobs, $completion, $qga, $op) = @_;
7272
7273     $completion //= 'complete';
7274     $op //= "mirror";
7275
7276     eval {
7277         my $err_complete = 0;
7278
7279         my $starttime = time ();
7280         while (1) {
7281             die "block job ('$op') timed out\n" if $err_complete > 300;
7282
7283             my $stats = mon_cmd($vmid, "query-block-jobs");
7284             my $ctime = time();
7285
7286             my $running_jobs = {};
7287             for my $stat (@$stats) {
7288                 next if $stat->{type} ne $op;
7289                 $running_jobs->{$stat->{device}} = $stat;
7290             }
7291
7292             my $readycounter = 0;
7293
7294             for my $job_id (sort keys %$jobs) {
7295                 my $job = $running_jobs->{$job_id};
7296
7297                 my $vanished = !defined($job);
7298                 my $complete = defined($jobs->{$job_id}->{complete}) && $vanished;
7299                 if($complete || ($vanished && $completion eq 'auto')) {
7300                     print "$job_id: $op-job finished\n";
7301                     delete $jobs->{$job_id};
7302                     next;
7303                 }
7304
7305                 die "$job_id: '$op' has been cancelled\n" if !defined($job);
7306
7307                 my $busy = $job->{busy};
7308                 my $ready = $job->{ready};
7309                 if (my $total = $job->{len}) {
7310                     my $transferred = $job->{offset} || 0;
7311                     my $remaining = $total - $transferred;
7312                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7313
7314                     my $duration = $ctime - $starttime;
7315                     my $total_h = render_bytes($total, 1);
7316                     my $transferred_h = render_bytes($transferred, 1);
7317
7318                     my $status = sprintf(
7319                         "transferred $transferred_h of $total_h ($percent%%) in %s",
7320                         render_duration($duration),
7321                     );
7322
7323                     if ($ready) {
7324                         if ($busy) {
7325                             $status .= ", still busy"; # shouldn't even happen? but mirror is weird
7326                         } else {
7327                             $status .= ", ready";
7328                         }
7329                     }
7330                     print "$job_id: $status\n" if !$jobs->{$job_id}->{ready};
7331                     $jobs->{$job_id}->{ready} = $ready;
7332                 }
7333
7334                 $readycounter++ if $job->{ready};
7335             }
7336
7337             last if scalar(keys %$jobs) == 0;
7338
7339             if ($readycounter == scalar(keys %$jobs)) {
7340                 print "all '$op' jobs are ready\n";
7341
7342                 # do the complete later (or has already been done)
7343                 last if $completion eq 'skip' || $completion eq 'auto';
7344
7345                 if ($vmiddst && $vmiddst != $vmid) {
7346                     my $agent_running = $qga && qga_check_running($vmid);
7347                     if ($agent_running) {
7348                         print "freeze filesystem\n";
7349                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7350                     } else {
7351                         print "suspend vm\n";
7352                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7353                     }
7354
7355                     # if we clone a disk for a new target vm, we don't switch the disk
7356                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7357
7358                     if ($agent_running) {
7359                         print "unfreeze filesystem\n";
7360                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7361                     } else {
7362                         print "resume vm\n";
7363                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7364                     }
7365
7366                     last;
7367                 } else {
7368
7369                     for my $job_id (sort keys %$jobs) {
7370                         # try to switch the disk if source and destination are on the same guest
7371                         print "$job_id: Completing block job_id...\n";
7372
7373                         my $op;
7374                         if ($completion eq 'complete') {
7375                             $op = 'block-job-complete';
7376                         } elsif ($completion eq 'cancel') {
7377                             $op = 'block-job-cancel';
7378                         } else {
7379                             die "invalid completion value: $completion\n";
7380                         }
7381                         eval { mon_cmd($vmid, $op, device => $job_id) };
7382                         if ($@ =~ m/cannot be completed/) {
7383                             print "$job_id: block job cannot be completed, trying again.\n";
7384                             $err_complete++;
7385                         }else {
7386                             print "$job_id: Completed successfully.\n";
7387                             $jobs->{$job_id}->{complete} = 1;
7388                         }
7389                     }
7390                 }
7391             }
7392             sleep 1;
7393         }
7394     };
7395     my $err = $@;
7396
7397     if ($err) {
7398         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7399         die "block job ($op) error: $err";
7400     }
7401 }
7402
7403 sub qemu_blockjobs_cancel {
7404     my ($vmid, $jobs) = @_;
7405
7406     foreach my $job (keys %$jobs) {
7407         print "$job: Cancelling block job\n";
7408         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
7409         $jobs->{$job}->{cancel} = 1;
7410     }
7411
7412     while (1) {
7413         my $stats = mon_cmd($vmid, "query-block-jobs");
7414
7415         my $running_jobs = {};
7416         foreach my $stat (@$stats) {
7417             $running_jobs->{$stat->{device}} = $stat;
7418         }
7419
7420         foreach my $job (keys %$jobs) {
7421
7422             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7423                 print "$job: Done.\n";
7424                 delete $jobs->{$job};
7425             }
7426         }
7427
7428         last if scalar(keys %$jobs) == 0;
7429
7430         sleep 1;
7431     }
7432 }
7433
7434 sub clone_disk {
7435     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7436         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7437
7438     my $newvolid;
7439
7440     if (!$full) {
7441         print "create linked clone of drive $drivename ($drive->{file})\n";
7442         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7443         push @$newvollist, $newvolid;
7444     } else {
7445
7446         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7447         $storeid = $storage if $storage;
7448
7449         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7450
7451         print "create full clone of drive $drivename ($drive->{file})\n";
7452         my $name = undef;
7453         my $size = undef;
7454         if (drive_is_cloudinit($drive)) {
7455             $name = "vm-$newvmid-cloudinit";
7456             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7457             if ($scfg->{path}) {
7458                 $name .= ".$dst_format";
7459             }
7460             $snapname = undef;
7461             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7462         } elsif ($drivename eq 'efidisk0') {
7463             $size = get_efivars_size($conf);
7464         } elsif ($drivename eq 'tpmstate0') {
7465             $size = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7466         } else {
7467             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7468         }
7469         $newvolid = PVE::Storage::vdisk_alloc(
7470             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7471         );
7472         push @$newvollist, $newvolid;
7473
7474         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7475
7476         if (drive_is_cloudinit($drive)) {
7477             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7478             # if this is the case, we have to complete any block-jobs still there from
7479             # previous drive-mirrors
7480             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7481                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7482             }
7483             goto no_data_clone;
7484         }
7485
7486         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7487         if (!$running || $snapname) {
7488             # TODO: handle bwlimits
7489             if ($drivename eq 'efidisk0') {
7490                 # the relevant data on the efidisk may be smaller than the source
7491                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7492                 # that is given by the OVMF_VARS.fd
7493                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7494                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7495
7496                 # better for Ceph if block size is not too small, see bug #3324
7497                 my $bs = 1024*1024;
7498
7499                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7500                     "if=$src_path", "of=$dst_path"]);
7501             } else {
7502                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7503             }
7504         } else {
7505
7506             die "cannot move TPM state while VM is running\n" if $drivename eq 'tpmstate0';
7507
7508             my $kvmver = get_running_qemu_version ($vmid);
7509             if (!min_version($kvmver, 2, 7)) {
7510                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7511                     if $drive->{iothread};
7512             }
7513
7514             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7515                 $completion, $qga, $bwlimit);
7516         }
7517     }
7518
7519 no_data_clone:
7520     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7521
7522     my $disk = $drive;
7523     $disk->{format} = undef;
7524     $disk->{file} = $newvolid;
7525     $disk->{size} = $size if defined($size);
7526
7527     return $disk;
7528 }
7529
7530 sub get_running_qemu_version {
7531     my ($vmid) = @_;
7532     my $res = mon_cmd($vmid, "query-version");
7533     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7534 }
7535
7536 sub qemu_use_old_bios_files {
7537     my ($machine_type) = @_;
7538
7539     return if !$machine_type;
7540
7541     my $use_old_bios_files = undef;
7542
7543     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7544         $machine_type = $1;
7545         $use_old_bios_files = 1;
7546     } else {
7547         my $version = extract_version($machine_type, kvm_user_version());
7548         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7549         # load new efi bios files on migration. So this hack is required to allow
7550         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7551         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7552         $use_old_bios_files = !min_version($version, 2, 4);
7553     }
7554
7555     return ($use_old_bios_files, $machine_type);
7556 }
7557
7558 sub get_efivars_size {
7559     my ($conf) = @_;
7560     my $arch = get_vm_arch($conf);
7561     my $efidisk = $conf->{efidisk0} ? parse_drive('efidisk0', $conf->{efidisk0}) : undef;
7562     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk);
7563     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7564     return -s $ovmf_vars;
7565 }
7566
7567 sub update_efidisk_size {
7568     my ($conf) = @_;
7569
7570     return if !defined($conf->{efidisk0});
7571
7572     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7573     $disk->{size} = get_efivars_size($conf);
7574     $conf->{efidisk0} = print_drive($disk);
7575
7576     return;
7577 }
7578
7579 sub update_tpmstate_size {
7580     my ($conf) = @_;
7581
7582     my $disk = PVE::QemuServer::parse_drive('tpmstate0', $conf->{tpmstate0});
7583     $disk->{size} = PVE::QemuServer::Drive::TPMSTATE_DISK_SIZE;
7584     $conf->{tpmstate0} = print_drive($disk);
7585 }
7586
7587 sub create_efidisk($$$$$$) {
7588     my ($storecfg, $storeid, $vmid, $fmt, $arch, $efidisk) = @_;
7589
7590     my (undef, $ovmf_vars) = get_ovmf_files($arch, $efidisk);
7591     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7592
7593     my $vars_size_b = -s $ovmf_vars;
7594     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7595     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7596     PVE::Storage::activate_volumes($storecfg, [$volid]);
7597
7598     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7599     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7600
7601     return ($volid, $size/1024);
7602 }
7603
7604 sub vm_iothreads_list {
7605     my ($vmid) = @_;
7606
7607     my $res = mon_cmd($vmid, 'query-iothreads');
7608
7609     my $iothreads = {};
7610     foreach my $iothread (@$res) {
7611         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7612     }
7613
7614     return $iothreads;
7615 }
7616
7617 sub scsihw_infos {
7618     my ($conf, $drive) = @_;
7619
7620     my $maxdev = 0;
7621
7622     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7623         $maxdev = 7;
7624     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7625         $maxdev = 1;
7626     } else {
7627         $maxdev = 256;
7628     }
7629
7630     my $controller = int($drive->{index} / $maxdev);
7631     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7632         ? "virtioscsi"
7633         : "scsihw";
7634
7635     return ($maxdev, $controller, $controller_prefix);
7636 }
7637
7638 sub windows_version {
7639     my ($ostype) = @_;
7640
7641     return 0 if !$ostype;
7642
7643     my $winversion = 0;
7644
7645     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7646         $winversion = 5;
7647     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7648         $winversion = 6;
7649     } elsif ($ostype =~ m/^win(\d+)$/) {
7650         $winversion = $1;
7651     }
7652
7653     return $winversion;
7654 }
7655
7656 sub resolve_dst_disk_format {
7657         my ($storecfg, $storeid, $src_volname, $format) = @_;
7658         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7659
7660         if (!$format) {
7661             # if no target format is specified, use the source disk format as hint
7662             if ($src_volname) {
7663                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7664                 $format = qemu_img_format($scfg, $src_volname);
7665             } else {
7666                 return $defFormat;
7667             }
7668         }
7669
7670         # test if requested format is supported - else use default
7671         my $supported = grep { $_ eq $format } @$validFormats;
7672         $format = $defFormat if !$supported;
7673         return $format;
7674 }
7675
7676 # NOTE: if this logic changes, please update docs & possibly gui logic
7677 sub find_vmstate_storage {
7678     my ($conf, $storecfg) = @_;
7679
7680     # first, return storage from conf if set
7681     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7682
7683     my ($target, $shared, $local);
7684
7685     foreach_storage_used_by_vm($conf, sub {
7686         my ($sid) = @_;
7687         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7688         my $dst = $scfg->{shared} ? \$shared : \$local;
7689         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7690     });
7691
7692     # second, use shared storage where VM has at least one disk
7693     # third, use local storage where VM has at least one disk
7694     # fall back to local storage
7695     $target = $shared // $local // 'local';
7696
7697     return $target;
7698 }
7699
7700 sub generate_uuid {
7701     my ($uuid, $uuid_str);
7702     UUID::generate($uuid);
7703     UUID::unparse($uuid, $uuid_str);
7704     return $uuid_str;
7705 }
7706
7707 sub generate_smbios1_uuid {
7708     return "uuid=".generate_uuid();
7709 }
7710
7711 sub nbd_stop {
7712     my ($vmid) = @_;
7713
7714     mon_cmd($vmid, 'nbd-server-stop');
7715 }
7716
7717 sub create_reboot_request {
7718     my ($vmid) = @_;
7719     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7720         or die "failed to create reboot trigger file: $!\n";
7721     close($fh);
7722 }
7723
7724 sub clear_reboot_request {
7725     my ($vmid) = @_;
7726     my $path = "/run/qemu-server/$vmid.reboot";
7727     my $res = 0;
7728
7729     $res = unlink($path);
7730     die "could not remove reboot request for $vmid: $!"
7731         if !$res && $! != POSIX::ENOENT;
7732
7733     return $res;
7734 }
7735
7736 sub bootorder_from_legacy {
7737     my ($conf, $bootcfg) = @_;
7738
7739     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7740     my $bootindex_hash = {};
7741     my $i = 1;
7742     foreach my $o (split(//, $boot)) {
7743         $bootindex_hash->{$o} = $i*100;
7744         $i++;
7745     }
7746
7747     my $bootorder = {};
7748
7749     PVE::QemuConfig->foreach_volume($conf, sub {
7750         my ($ds, $drive) = @_;
7751
7752         if (drive_is_cdrom ($drive, 1)) {
7753             if ($bootindex_hash->{d}) {
7754                 $bootorder->{$ds} = $bootindex_hash->{d};
7755                 $bootindex_hash->{d} += 1;
7756             }
7757         } elsif ($bootindex_hash->{c}) {
7758             $bootorder->{$ds} = $bootindex_hash->{c}
7759                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7760             $bootindex_hash->{c} += 1;
7761         }
7762     });
7763
7764     if ($bootindex_hash->{n}) {
7765         for (my $i = 0; $i < $MAX_NETS; $i++) {
7766             my $netname = "net$i";
7767             next if !$conf->{$netname};
7768             $bootorder->{$netname} = $bootindex_hash->{n};
7769             $bootindex_hash->{n} += 1;
7770         }
7771     }
7772
7773     return $bootorder;
7774 }
7775
7776 # Generate default device list for 'boot: order=' property. Matches legacy
7777 # default boot order, but with explicit device names. This is important, since
7778 # the fallback for when neither 'order' nor the old format is specified relies
7779 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7780 sub get_default_bootdevices {
7781     my ($conf) = @_;
7782
7783     my @ret = ();
7784
7785     # harddisk
7786     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7787     push @ret, $first if $first;
7788
7789     # cdrom
7790     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7791     push @ret, $first if $first;
7792
7793     # network
7794     for (my $i = 0; $i < $MAX_NETS; $i++) {
7795         my $netname = "net$i";
7796         next if !$conf->{$netname};
7797         push @ret, $netname;
7798         last;
7799     }
7800
7801     return \@ret;
7802 }
7803
7804 sub device_bootorder {
7805     my ($conf) = @_;
7806
7807     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7808
7809     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7810
7811     my $bootorder = {};
7812     if (!defined($boot) || $boot->{legacy}) {
7813         $bootorder = bootorder_from_legacy($conf, $boot);
7814     } elsif ($boot->{order}) {
7815         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7816         for my $dev (PVE::Tools::split_list($boot->{order})) {
7817             $bootorder->{$dev} = $i++;
7818         }
7819     }
7820
7821     return $bootorder;
7822 }
7823
7824 sub register_qmeventd_handle {
7825     my ($vmid) = @_;
7826
7827     my $fh;
7828     my $peer = "/var/run/qmeventd.sock";
7829     my $count = 0;
7830
7831     for (;;) {
7832         $count++;
7833         $fh = IO::Socket::UNIX->new(Peer => $peer, Blocking => 0, Timeout => 1);
7834         last if $fh;
7835         if ($! != EINTR && $! != EAGAIN) {
7836             die "unable to connect to qmeventd socket (vmid: $vmid) - $!\n";
7837         }
7838         if ($count > 4) {
7839             die "unable to connect to qmeventd socket (vmid: $vmid) - timeout "
7840               . "after $count retries\n";
7841         }
7842         usleep(25000);
7843     }
7844
7845     # send handshake to mark VM as backing up
7846     print $fh to_json({vzdump => {vmid => "$vmid"}});
7847
7848     # return handle to be closed later when inhibit is no longer required
7849     return $fh;
7850 }
7851
7852 # bash completion helper
7853
7854 sub complete_backup_archives {
7855     my ($cmdname, $pname, $cvalue) = @_;
7856
7857     my $cfg = PVE::Storage::config();
7858
7859     my $storeid;
7860
7861     if ($cvalue =~ m/^([^:]+):/) {
7862         $storeid = $1;
7863     }
7864
7865     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7866
7867     my $res = [];
7868     foreach my $id (keys %$data) {
7869         foreach my $item (@{$data->{$id}}) {
7870             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7871             push @$res, $item->{volid} if defined($item->{volid});
7872         }
7873     }
7874
7875     return $res;
7876 }
7877
7878 my $complete_vmid_full = sub {
7879     my ($running) = @_;
7880
7881     my $idlist = vmstatus();
7882
7883     my $res = [];
7884
7885     foreach my $id (keys %$idlist) {
7886         my $d = $idlist->{$id};
7887         if (defined($running)) {
7888             next if $d->{template};
7889             next if $running && $d->{status} ne 'running';
7890             next if !$running && $d->{status} eq 'running';
7891         }
7892         push @$res, $id;
7893
7894     }
7895     return $res;
7896 };
7897
7898 sub complete_vmid {
7899     return &$complete_vmid_full();
7900 }
7901
7902 sub complete_vmid_stopped {
7903     return &$complete_vmid_full(0);
7904 }
7905
7906 sub complete_vmid_running {
7907     return &$complete_vmid_full(1);
7908 }
7909
7910 sub complete_storage {
7911
7912     my $cfg = PVE::Storage::config();
7913     my $ids = $cfg->{ids};
7914
7915     my $res = [];
7916     foreach my $sid (keys %$ids) {
7917         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7918         next if !$ids->{$sid}->{content}->{images};
7919         push @$res, $sid;
7920     }
7921
7922     return $res;
7923 }
7924
7925 sub complete_migration_storage {
7926     my ($cmd, $param, $current_value, $all_args) = @_;
7927
7928     my $targetnode = @$all_args[1];
7929
7930     my $cfg = PVE::Storage::config();
7931     my $ids = $cfg->{ids};
7932
7933     my $res = [];
7934     foreach my $sid (keys %$ids) {
7935         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7936         next if !$ids->{$sid}->{content}->{images};
7937         push @$res, $sid;
7938     }
7939
7940     return $res;
7941 }
7942
7943 sub vm_is_paused {
7944     my ($vmid) = @_;
7945     my $qmpstatus = eval {
7946         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7947         mon_cmd($vmid, "query-status");
7948     };
7949     warn "$@\n" if $@;
7950     return $qmpstatus && $qmpstatus->{status} eq "paused";
7951 }
7952
7953 sub check_volume_storage_type {
7954     my ($storecfg, $vol) = @_;
7955
7956     my ($storeid, $volname) = PVE::Storage::parse_volume_id($vol);
7957     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7958     my ($vtype) = PVE::Storage::parse_volname($storecfg, $vol);
7959
7960     die "storage '$storeid' does not support content-type '$vtype'\n"
7961         if !$scfg->{content}->{$vtype};
7962
7963     return 1;
7964 }
7965
7966 1;