PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016/2019
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 970 dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $usb_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string', format => 'pve-qm-usb-device',
1005         format_description => 'HOSTUSBDEVICE|spice',
1006         description => <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1008
1009  'bus-port(.port)*' (decimal numbers) or
1010  'vendor_id:product_id' (hexadeciaml numbers) or
1011  'spice'
1012
1013 You can use the 'lsusb -t' command to list existing usb devices.
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1017
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1019 EODESCR
1020     },
1021     usb3 => {
1022         optional => 1,
1023         type => 'boolean',
1024         description => "Specifies whether if given host option is a USB3 device or port.",
1025         default => 0,
1026     },
1027 };
1028
1029 my $usbdesc = {
1030     optional => 1,
1031     type => 'string', format => $usb_fmt,
1032     description => "Configure an USB device (n is 0 to 4).",
1033 };
1034 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1035
1036 my $serialdesc = {
1037         optional => 1,
1038         type => 'string',
1039         pattern => '(/dev/.+|socket)',
1040         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1041         verbose_description =>  <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1045
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052
1053 my $paralleldesc= {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1057         description =>  "Map host parallel devices (n is 0 to 2).",
1058         verbose_description =>  <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1078 }
1079
1080 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1081     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1082 }
1083
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1085     $confdesc->{"usb$i"} = $usbdesc;
1086 }
1087
1088 my $boot_fmt = {
1089     legacy => {
1090         optional => 1,
1091         default_key => 1,
1092         type => 'string',
1093         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094                      . " Deprecated, use 'order=' instead.",
1095         pattern => '[acdn]{1,4}',
1096         format_description => "[acdn]{1,4}",
1097
1098         # note: this is also the fallback if boot: is not given at all
1099         default => 'cdn',
1100     },
1101     order => {
1102         optional => 1,
1103         type => 'string',
1104         format => 'pve-qm-bootdev-list',
1105         format_description => "device[;device...]",
1106         description => <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1108
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1112
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1116
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1118 EODESC
1119     },
1120 };
1121 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1122
1123 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1124 sub verify_bootdev {
1125     my ($dev, $noerr) = @_;
1126
1127     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1128
1129     my $check = sub {
1130         my ($base) = @_;
1131         return 0 if $dev !~ m/^$base\d+$/;
1132         return 0 if !$confdesc->{$dev};
1133         return 1;
1134     };
1135
1136     return $dev if $check->("net");
1137     return $dev if $check->("usb");
1138     return $dev if $check->("hostpci");
1139
1140     return if $noerr;
1141     die "invalid boot device '$dev'\n";
1142 }
1143
1144 sub print_bootorder {
1145     my ($devs) = @_;
1146     return "" if !@$devs;
1147     my $data = { order => join(';', @$devs) };
1148     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1149 }
1150
1151 my $kvm_api_version = 0;
1152
1153 sub kvm_version {
1154     return $kvm_api_version if $kvm_api_version;
1155
1156     open my $fh, '<', '/dev/kvm' or return;
1157
1158     # 0xae00 => KVM_GET_API_VERSION
1159     $kvm_api_version = ioctl($fh, 0xae00, 0);
1160     close($fh);
1161
1162     return $kvm_api_version;
1163 }
1164
1165 my $kvm_user_version = {};
1166 my $kvm_mtime = {};
1167
1168 sub kvm_user_version {
1169     my ($binary) = @_;
1170
1171     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1172     my $st = stat($binary);
1173
1174     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176         $cachedmtime == $st->mtime;
1177
1178     $kvm_user_version->{$binary} = 'unknown';
1179     $kvm_mtime->{$binary} = $st->mtime;
1180
1181     my $code = sub {
1182         my $line = shift;
1183         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184             $kvm_user_version->{$binary} = $2;
1185         }
1186     };
1187
1188     eval { run_command([$binary, '--version'], outfunc => $code); };
1189     warn $@ if $@;
1190
1191     return $kvm_user_version->{$binary};
1192
1193 }
1194 my sub extract_version {
1195     my ($machine_type, $version) = @_;
1196     $version = kvm_user_version() if !defined($version);
1197     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1198 }
1199
1200 sub kernel_has_vhost_net {
1201     return -c '/dev/vhost-net';
1202 }
1203
1204 sub option_exists {
1205     my $key = shift;
1206     return defined($confdesc->{$key});
1207 }
1208
1209 my $cdrom_path;
1210 sub get_cdrom_path {
1211
1212     return  $cdrom_path if $cdrom_path;
1213
1214     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1215     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1216     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1217 }
1218
1219 sub get_iso_path {
1220     my ($storecfg, $vmid, $cdrom) = @_;
1221
1222     if ($cdrom eq 'cdrom') {
1223         return get_cdrom_path();
1224     } elsif ($cdrom eq 'none') {
1225         return '';
1226     } elsif ($cdrom =~ m|^/|) {
1227         return $cdrom;
1228     } else {
1229         return PVE::Storage::path($storecfg, $cdrom);
1230     }
1231 }
1232
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id {
1235     my ($vmid, $file, $media) = @_;
1236
1237      if (!($file eq 'none' || $file eq 'cdrom' ||
1238           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1239
1240         return if $file =~ m|/|;
1241
1242         if ($media && $media eq 'cdrom') {
1243             $file = "local:iso/$file";
1244         } else {
1245             $file = "local:$vmid/$file";
1246         }
1247     }
1248
1249     return $file;
1250 }
1251
1252 sub verify_media_type {
1253     my ($opt, $vtype, $media) = @_;
1254
1255     return if !$media;
1256
1257     my $etype;
1258     if ($media eq 'disk') {
1259         $etype = 'images';
1260     } elsif ($media eq 'cdrom') {
1261         $etype = 'iso';
1262     } else {
1263         die "internal error";
1264     }
1265
1266     return if ($vtype eq $etype);
1267
1268     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1269 }
1270
1271 sub cleanup_drive_path {
1272     my ($opt, $storecfg, $drive) = @_;
1273
1274     # try to convert filesystem paths to volume IDs
1275
1276     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1277         ($drive->{file} !~ m|^/dev/.+|) &&
1278         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1279         ($drive->{file} !~ m/^\d+$/)) {
1280         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1281         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1282             if !$vtype;
1283         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1284         verify_media_type($opt, $vtype, $drive->{media});
1285         $drive->{file} = $volid;
1286     }
1287
1288     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1289 }
1290
1291 sub parse_hotplug_features {
1292     my ($data) = @_;
1293
1294     my $res = {};
1295
1296     return $res if $data eq '0';
1297
1298     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1299
1300     foreach my $feature (PVE::Tools::split_list($data)) {
1301         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1302             $res->{$1} = 1;
1303         } else {
1304             die "invalid hotplug feature '$feature'\n";
1305         }
1306     }
1307     return $res;
1308 }
1309
1310 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1311 sub pve_verify_hotplug_features {
1312     my ($value, $noerr) = @_;
1313
1314     return $value if parse_hotplug_features($value);
1315
1316     return if $noerr;
1317
1318     die "unable to parse hotplug option\n";
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub print_tabletdevice_full {
1383     my ($conf, $arch) = @_;
1384
1385     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1386
1387     # we use uhci for old VMs because tablet driver was buggy in older qemu
1388     my $usbbus;
1389     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1390         $usbbus = 'ehci';
1391     } else {
1392         $usbbus = 'uhci';
1393     }
1394
1395     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1396 }
1397
1398 sub print_keyboarddevice_full {
1399     my ($conf, $arch, $machine) = @_;
1400
1401     return if $arch ne 'aarch64';
1402
1403     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1404 }
1405
1406 sub print_drivedevice_full {
1407     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1408
1409     my $device = '';
1410     my $maxdev = 0;
1411
1412     my $drive_id = "$drive->{interface}$drive->{index}";
1413     if ($drive->{interface} eq 'virtio') {
1414         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1415         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1417     } elsif ($drive->{interface} eq 'scsi') {
1418
1419         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1420         my $unit = $drive->{index} % $maxdev;
1421         my $devicetype = 'hd';
1422         my $path = '';
1423         if (drive_is_cdrom($drive)) {
1424             $devicetype = 'cd';
1425         } else {
1426             if ($drive->{file} =~ m|^/|) {
1427                 $path = $drive->{file};
1428                 if (my $info = path_is_scsi($path)) {
1429                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1430                         $devicetype = 'block';
1431                     } elsif ($info->{type} == 1) { # tape
1432                         $devicetype = 'generic';
1433                     }
1434                 }
1435             } else {
1436                  $path = PVE::Storage::path($storecfg, $drive->{file});
1437             }
1438
1439             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440             my $version = extract_version($machine_type, kvm_user_version());
1441             if ($path =~ m/^iscsi\:\/\// &&
1442                !min_version($version, 4, 1)) {
1443                 $devicetype = 'generic';
1444             }
1445         }
1446
1447         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1448             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1449         } else {
1450             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451                 .",lun=$drive->{index}";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456             $device .= ",rotation_rate=1";
1457         }
1458         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1459
1460     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1461         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1462         my $controller = int($drive->{index} / $maxdev);
1463         my $unit = $drive->{index} % $maxdev;
1464         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1465
1466         $device = "ide-$devicetype";
1467         if ($drive->{interface} eq 'ide') {
1468             $device .= ",bus=ide.$controller,unit=$unit";
1469         } else {
1470             $device .= ",bus=ahci$controller.$unit";
1471         }
1472         $device .= ",drive=drive-$drive_id,id=$drive_id";
1473
1474         if ($devicetype eq 'hd') {
1475             if (my $model = $drive->{model}) {
1476                 $model = URI::Escape::uri_unescape($model);
1477                 $device .= ",model=$model";
1478             }
1479             if ($drive->{ssd}) {
1480                 $device .= ",rotation_rate=1";
1481             }
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484     } elsif ($drive->{interface} eq 'usb') {
1485         die "implement me";
1486         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1487     } else {
1488         die "unsupported interface type";
1489     }
1490
1491     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1492
1493     if (my $serial = $drive->{serial}) {
1494         $serial = URI::Escape::uri_unescape($serial);
1495         $device .= ",serial=$serial";
1496     }
1497
1498
1499     return $device;
1500 }
1501
1502 sub get_initiator_name {
1503     my $initiator;
1504
1505     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1506     while (defined(my $line = <$fh>)) {
1507         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1508         $initiator = $1;
1509         last;
1510     }
1511     $fh->close();
1512
1513     return $initiator;
1514 }
1515
1516 sub print_drive_commandline_full {
1517     my ($storecfg, $vmid, $drive) = @_;
1518
1519     my $path;
1520     my $volid = $drive->{file};
1521     my $format;
1522
1523     if (drive_is_cdrom($drive)) {
1524         $path = get_iso_path($storecfg, $vmid, $volid);
1525     } else {
1526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1527         if ($storeid) {
1528             $path = PVE::Storage::path($storecfg, $volid);
1529             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1530             $format = qemu_img_format($scfg, $volname);
1531         } else {
1532             $path = $volid;
1533             $format = "raw";
1534         }
1535    }
1536
1537     my $opts = '';
1538     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539     foreach my $o (@qemu_drive_options) {
1540         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1541     }
1542
1543     # snapshot only accepts on|off
1544     if (defined($drive->{snapshot})) {
1545         my $v = $drive->{snapshot} ? 'on' : 'off';
1546         $opts .= ",snapshot=$v";
1547     }
1548
1549     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1550         my ($dir, $qmpname) = @$type;
1551         if (my $v = $drive->{"mbps$dir"}) {
1552             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1553         }
1554         if (my $v = $drive->{"mbps${dir}_max"}) {
1555             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1556         }
1557         if (my $v = $drive->{"bps${dir}_max_length"}) {
1558             $opts .= ",throttling.bps$qmpname-max-length=$v";
1559         }
1560         if (my $v = $drive->{"iops${dir}"}) {
1561             $opts .= ",throttling.iops$qmpname=$v";
1562         }
1563         if (my $v = $drive->{"iops${dir}_max"}) {
1564             $opts .= ",throttling.iops$qmpname-max=$v";
1565         }
1566         if (my $v = $drive->{"iops${dir}_max_length"}) {
1567             $opts .= ",throttling.iops$qmpname-max-length=$v";
1568         }
1569     }
1570
1571     $opts .= ",format=$format" if $format && !$drive->{format};
1572
1573     my $cache_direct = 0;
1574
1575     if (my $cache = $drive->{cache}) {
1576         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577     } elsif (!drive_is_cdrom($drive)) {
1578         $opts .= ",cache=none";
1579         $cache_direct = 1;
1580     }
1581
1582     # aio native works only with O_DIRECT
1583     if (!$drive->{aio}) {
1584         if($cache_direct) {
1585             $opts .= ",aio=native";
1586         } else {
1587             $opts .= ",aio=threads";
1588         }
1589     }
1590
1591     if (!drive_is_cdrom($drive)) {
1592         my $detectzeroes;
1593         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1594             $detectzeroes = 'off';
1595         } elsif ($drive->{discard}) {
1596             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1597         } else {
1598             # This used to be our default with discard not being specified:
1599             $detectzeroes = 'on';
1600         }
1601         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1602     }
1603
1604     my $pathinfo = $path ? "file=$path," : '';
1605
1606     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1607 }
1608
1609 sub print_netdevice_full {
1610     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1611
1612     my $device = $net->{model};
1613     if ($net->{model} eq 'virtio') {
1614          $device = 'virtio-net-pci';
1615      };
1616
1617     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1618     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1620         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621         # and out of each queue plus one config interrupt and control vector queue
1622         my $vectors = $net->{queues} * 2 + 2;
1623         $tmpstr .= ",vectors=$vectors,mq=on";
1624     }
1625     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1626
1627     if (my $mtu = $net->{mtu}) {
1628         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1629             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1630             if ($mtu == 1) {
1631                  $mtu = $bridge_mtu;
1632             } elsif ($mtu < 576) {
1633                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634             } elsif ($mtu > $bridge_mtu) {
1635                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1636             }
1637             $tmpstr .= ",host_mtu=$mtu";
1638         } else {
1639             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1640         }
1641     }
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     if (is_native($arch)) {
1680         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1681     }
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1691     } else {
1692         $netdev = "type=user,id=$netid,hostname=$vmname";
1693     }
1694
1695     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1696
1697     return $netdev;
1698 }
1699
1700 my $vga_map = {
1701     'cirrus' => 'cirrus-vga',
1702     'std' => 'VGA',
1703     'vmware' => 'vmware-svga',
1704     'virtio' => 'virtio-vga',
1705 };
1706
1707 sub print_vga_device {
1708     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1709
1710     my $type = $vga_map->{$vga->{type}};
1711     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712         $type = 'virtio-gpu';
1713     }
1714     my $vgamem_mb = $vga->{memory};
1715
1716     my $max_outputs = '';
1717     if ($qxlnum) {
1718         $type = $id ? 'qxl' : 'qxl-vga';
1719
1720         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1721             # set max outputs so linux can have up to 4 qxl displays with one device
1722             if (min_version($machine_version, 4, 1)) {
1723                 $max_outputs = ",max_outputs=4";
1724             }
1725         }
1726     }
1727
1728     die "no devicetype for $vga->{type}\n" if !$type;
1729
1730     my $memory = "";
1731     if ($vgamem_mb) {
1732         if ($vga->{type} eq 'virtio') {
1733             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1734             $memory = ",max_hostmem=$bytes";
1735         } elsif ($qxlnum) {
1736             # from https://www.spice-space.org/multiple-monitors.html
1737             $memory = ",vgamem_mb=$vga->{memory}";
1738             my $ram = $vgamem_mb * 4;
1739             my $vram = $vgamem_mb * 2;
1740             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1741         } else {
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743         }
1744     } elsif ($qxlnum && $id) {
1745         $memory = ",ram_size=67108864,vram_size=33554432";
1746     }
1747
1748     my $edidoff = "";
1749     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1750         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1751     }
1752
1753     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1754     my $vgaid = "vga" . ($id // '');
1755     my $pciaddr;
1756
1757     if ($q35 && $vgaid eq 'vga') {
1758         # the first display uses pcie.0 bus on q35 machines
1759         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1760     } else {
1761         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1762     }
1763
1764     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1765 }
1766
1767 sub parse_number_sets {
1768     my ($set) = @_;
1769     my $res = [];
1770     foreach my $part (split(/;/, $set)) {
1771         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773             push @$res, [ $1, $2 ];
1774         } else {
1775             die "invalid range: $part\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 sub parse_numa {
1782     my ($data) = @_;
1783
1784     my $res = parse_property_string($numa_fmt, $data);
1785     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1786     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1787     return $res;
1788 }
1789
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1791 sub parse_net {
1792     my ($data) = @_;
1793
1794     my $res = eval { parse_property_string($net_fmt, $data) };
1795     if ($@) {
1796         warn $@;
1797         return;
1798     }
1799     if (!defined($res->{macaddr})) {
1800         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1801         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1802     }
1803     return $res;
1804 }
1805
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig {
1808     my ($data) = @_;
1809
1810     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1811     if ($@) {
1812         warn $@;
1813         return;
1814     }
1815
1816     if ($res->{gw} && !$res->{ip}) {
1817         warn 'gateway specified without specifying an IP address';
1818         return;
1819     }
1820     if ($res->{gw6} && !$res->{ip6}) {
1821         warn 'IPv6 gateway specified without specifying an IPv6 address';
1822         return;
1823     }
1824     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1825         warn 'gateway specified together with DHCP';
1826         return;
1827     }
1828     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1829         # gw6 + auto/dhcp
1830         warn "IPv6 gateway specified together with $res->{ip6} address";
1831         return;
1832     }
1833
1834     if (!$res->{ip} && !$res->{ip6}) {
1835         return { ip => 'dhcp', ip6 => 'dhcp' };
1836     }
1837
1838     return $res;
1839 }
1840
1841 sub print_net {
1842     my $net = shift;
1843
1844     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1845 }
1846
1847 sub add_random_macs {
1848     my ($settings) = @_;
1849
1850     foreach my $opt (keys %$settings) {
1851         next if $opt !~ m/^net(\d+)$/;
1852         my $net = parse_net($settings->{$opt});
1853         next if !$net;
1854         $settings->{$opt} = print_net($net);
1855     }
1856 }
1857
1858 sub vm_is_volid_owner {
1859     my ($storecfg, $vmid, $volid) = @_;
1860
1861     if ($volid !~  m|^/|) {
1862         my ($path, $owner);
1863         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1864         if ($owner && ($owner == $vmid)) {
1865             return 1;
1866         }
1867     }
1868
1869     return;
1870 }
1871
1872 sub vmconfig_register_unused_drive {
1873     my ($storecfg, $vmid, $conf, $drive) = @_;
1874
1875     if (drive_is_cloudinit($drive)) {
1876         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1877         warn $@ if $@;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         my $volid = $drive->{file};
1880         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1881             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1882         }
1883     }
1884 }
1885
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1887 my $smbios1_fmt = {
1888     uuid => {
1889         type => 'string',
1890         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891         format_description => 'UUID',
1892         description => "Set SMBIOS1 UUID.",
1893         optional => 1,
1894     },
1895     version => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 version.",
1900         optional => 1,
1901     },
1902     serial => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 serial number.",
1907         optional => 1,
1908     },
1909     manufacturer => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 manufacturer.",
1914         optional => 1,
1915     },
1916     product => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 product ID.",
1921         optional => 1,
1922     },
1923     sku => {
1924         type => 'string',
1925         pattern => '[A-Za-z0-9+\/]+={0,2}',
1926         format_description => 'Base64 encoded string',
1927         description => "Set SMBIOS1 SKU string.",
1928         optional => 1,
1929     },
1930     family => {
1931         type => 'string',
1932         pattern => '[A-Za-z0-9+\/]+={0,2}',
1933         format_description => 'Base64 encoded string',
1934         description => "Set SMBIOS1 family string.",
1935         optional => 1,
1936     },
1937     base64 => {
1938         type => 'boolean',
1939         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1940         optional => 1,
1941     },
1942 };
1943
1944 sub parse_smbios1 {
1945     my ($data) = @_;
1946
1947     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1948     warn $@ if $@;
1949     return $res;
1950 }
1951
1952 sub print_smbios1 {
1953     my ($smbios1) = @_;
1954     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1958
1959 sub parse_watchdog {
1960     my ($value) = @_;
1961
1962     return if !$value;
1963
1964     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1965     warn $@ if $@;
1966     return $res;
1967 }
1968
1969 sub parse_guest_agent {
1970     my ($conf) = @_;
1971
1972     return {} if !defined($conf->{agent});
1973
1974     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1975     warn $@ if $@;
1976
1977     # if the agent is disabled ignore the other potentially set properties
1978     return {} if !$res->{enabled};
1979     return $res;
1980 }
1981
1982 sub get_qga_key {
1983     my ($conf, $key) = @_;
1984     return undef if !defined($conf->{agent});
1985
1986     my $agent = parse_guest_agent($conf);
1987     return $agent->{$key};
1988 }
1989
1990 sub parse_vga {
1991     my ($value) = @_;
1992
1993     return {} if !$value;
1994     my $res = eval { parse_property_string($vga_fmt, $value) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub parse_rng {
2000     my ($value) = @_;
2001
2002     return if !$value;
2003
2004     my $res = eval { parse_property_string($rng_fmt, $value) };
2005     warn $@ if $@;
2006     return $res;
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2010 sub verify_usb_device {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if parse_usb_device($value);
2014
2015     return if $noerr;
2016
2017     die "unable to parse usb device\n";
2018 }
2019
2020 # add JSON properties for create and set function
2021 sub json_config_properties {
2022     my $prop = shift;
2023
2024     foreach my $opt (keys %$confdesc) {
2025         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027         $prop->{$opt} = $confdesc->{$opt};
2028     }
2029
2030     return $prop;
2031 }
2032
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties {
2035
2036     return dclone($confdesc_cloudinit);
2037 }
2038
2039 sub check_type {
2040     my ($key, $value) = @_;
2041
2042     die "unknown setting '$key'\n" if !$confdesc->{$key};
2043
2044     my $type = $confdesc->{$key}->{type};
2045
2046     if (!defined($value)) {
2047         die "got undefined value\n";
2048     }
2049
2050     if ($value =~ m/[\n\r]/) {
2051         die "property contains a line feed\n";
2052     }
2053
2054     if ($type eq 'boolean') {
2055         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057         die "type check ('boolean') failed - got '$value'\n";
2058     } elsif ($type eq 'integer') {
2059         return int($1) if $value =~ m/^(\d+)$/;
2060         die "type check ('integer') failed - got '$value'\n";
2061     } elsif ($type eq 'number') {
2062         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063         die "type check ('number') failed - got '$value'\n";
2064     } elsif ($type eq 'string') {
2065         if (my $fmt = $confdesc->{$key}->{format}) {
2066             PVE::JSONSchema::check_format($fmt, $value);
2067             return $value;
2068         }
2069         $value =~ s/^\"(.*)\"$/$1/;
2070         return $value;
2071     } else {
2072         die "internal error"
2073     }
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     if ($conf->{template}) {
2084         # check if any base image is still used by a linked clone
2085         PVE::QemuConfig->foreach_volume($conf, sub {
2086                 my ($ds, $drive) = @_;
2087                 return if drive_is_cdrom($drive);
2088
2089                 my $volid = $drive->{file};
2090                 return if !$volid || $volid =~ m|^/|;
2091
2092                 die "base volume '$volid' is still in use by linked cloned\n"
2093                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2094
2095         });
2096     }
2097
2098     # only remove disks owned by this VM (referenced in the config)
2099     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2100         my ($ds, $drive) = @_;
2101         return if drive_is_cdrom($drive, 1);
2102
2103         my $volid = $drive->{file};
2104         return if !$volid || $volid =~ m|^/|;
2105
2106         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2107         return if !$path || !$owner || ($owner != $vmid);
2108
2109         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2110         warn "Could not remove disk '$volid', check manually: $@" if $@;
2111     });
2112
2113     if ($purge_unreferenced) { # also remove unreferenced disk
2114         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2115         PVE::Storage::foreach_volid($vmdisks, sub {
2116             my ($volid, $sid, $volname, $d) = @_;
2117             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2118             warn $@ if $@;
2119         });
2120     }
2121
2122     if (defined $replacement_conf) {
2123         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2124     } else {
2125         PVE::QemuConfig->destroy_config($vmid);
2126     }
2127 }
2128
2129 sub parse_vm_config {
2130     my ($filename, $raw) = @_;
2131
2132     return if !defined($raw);
2133
2134     my $res = {
2135         digest => Digest::SHA::sha1_hex($raw),
2136         snapshots => {},
2137         pending => {},
2138     };
2139
2140     $filename =~ m|/qemu-server/(\d+)\.conf$|
2141         || die "got strange filename '$filename'";
2142
2143     my $vmid = $1;
2144
2145     my $conf = $res;
2146     my $descr;
2147     my $section = '';
2148
2149     my @lines = split(/\n/, $raw);
2150     foreach my $line (@lines) {
2151         next if $line =~ m/^\s*$/;
2152
2153         if ($line =~ m/^\[PENDING\]\s*$/i) {
2154             $section = 'pending';
2155             if (defined($descr)) {
2156                 $descr =~ s/\s+$//;
2157                 $conf->{description} = $descr;
2158             }
2159             $descr = undef;
2160             $conf = $res->{$section} = {};
2161             next;
2162
2163         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2164             $section = $1;
2165             if (defined($descr)) {
2166                 $descr =~ s/\s+$//;
2167                 $conf->{description} = $descr;
2168             }
2169             $descr = undef;
2170             $conf = $res->{snapshots}->{$section} = {};
2171             next;
2172         }
2173
2174         if ($line =~ m/^\#(.*)\s*$/) {
2175             $descr = '' if !defined($descr);
2176             $descr .= PVE::Tools::decode_text($1) . "\n";
2177             next;
2178         }
2179
2180         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181             $descr = '' if !defined($descr);
2182             $descr .= PVE::Tools::decode_text($2);
2183         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184             $conf->{snapstate} = $1;
2185         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2186             my $key = $1;
2187             my $value = $2;
2188             $conf->{$key} = $value;
2189         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2190             my $value = $1;
2191             if ($section eq 'pending') {
2192                 $conf->{delete} = $value; # we parse this later
2193             } else {
2194                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2195             }
2196         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2197             my $key = $1;
2198             my $value = $2;
2199             eval { $value = check_type($key, $value); };
2200             if ($@) {
2201                 warn "vm $vmid - unable to parse value of '$key' - $@";
2202             } else {
2203                 $key = 'ide2' if $key eq 'cdrom';
2204                 my $fmt = $confdesc->{$key}->{format};
2205                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206                     my $v = parse_drive($key, $value);
2207                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2208                         $v->{file} = $volid;
2209                         $value = print_drive($v);
2210                     } else {
2211                         warn "vm $vmid - unable to parse value of '$key'\n";
2212                         next;
2213                     }
2214                 }
2215
2216                 $conf->{$key} = $value;
2217             }
2218         } else {
2219             warn "vm $vmid - unable to parse config: $line\n";
2220         }
2221     }
2222
2223     if (defined($descr)) {
2224         $descr =~ s/\s+$//;
2225         $conf->{description} = $descr;
2226     }
2227     delete $res->{snapstate}; # just to be sure
2228
2229     return $res;
2230 }
2231
2232 sub write_vm_config {
2233     my ($filename, $conf) = @_;
2234
2235     delete $conf->{snapstate}; # just to be sure
2236
2237     if ($conf->{cdrom}) {
2238         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2239         $conf->{ide2} = $conf->{cdrom};
2240         delete $conf->{cdrom};
2241     }
2242
2243     # we do not use 'smp' any longer
2244     if ($conf->{sockets}) {
2245         delete $conf->{smp};
2246     } elsif ($conf->{smp}) {
2247         $conf->{sockets} = $conf->{smp};
2248         delete $conf->{cores};
2249         delete $conf->{smp};
2250     }
2251
2252     my $used_volids = {};
2253
2254     my $cleanup_config = sub {
2255         my ($cref, $pending, $snapname) = @_;
2256
2257         foreach my $key (keys %$cref) {
2258             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2259                 $key eq 'snapstate' || $key eq 'pending';
2260             my $value = $cref->{$key};
2261             if ($key eq 'delete') {
2262                 die "propertry 'delete' is only allowed in [PENDING]\n"
2263                     if !$pending;
2264                 # fixme: check syntax?
2265                 next;
2266             }
2267             eval { $value = check_type($key, $value); };
2268             die "unable to parse value of '$key' - $@" if $@;
2269
2270             $cref->{$key} = $value;
2271
2272             if (!$snapname && is_valid_drivename($key)) {
2273                 my $drive = parse_drive($key, $value);
2274                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2275             }
2276         }
2277     };
2278
2279     &$cleanup_config($conf);
2280
2281     &$cleanup_config($conf->{pending}, 1);
2282
2283     foreach my $snapname (keys %{$conf->{snapshots}}) {
2284         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2285         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2286     }
2287
2288     # remove 'unusedX' settings if we re-add a volume
2289     foreach my $key (keys %$conf) {
2290         my $value = $conf->{$key};
2291         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2292             delete $conf->{$key};
2293         }
2294     }
2295
2296     my $generate_raw_config = sub {
2297         my ($conf, $pending) = @_;
2298
2299         my $raw = '';
2300
2301         # add description as comment to top of file
2302         if (defined(my $descr = $conf->{description})) {
2303             if ($descr) {
2304                 foreach my $cl (split(/\n/, $descr)) {
2305                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2306                 }
2307             } else {
2308                 $raw .= "#\n" if $pending;
2309             }
2310         }
2311
2312         foreach my $key (sort keys %$conf) {
2313             next if $key =~ /^(digest|description|pending|snapshots)$/;
2314             $raw .= "$key: $conf->{$key}\n";
2315         }
2316         return $raw;
2317     };
2318
2319     my $raw = &$generate_raw_config($conf);
2320
2321     if (scalar(keys %{$conf->{pending}})){
2322         $raw .= "\n[PENDING]\n";
2323         $raw .= &$generate_raw_config($conf->{pending}, 1);
2324     }
2325
2326     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2327         $raw .= "\n[$snapname]\n";
2328         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2329     }
2330
2331     return $raw;
2332 }
2333
2334 sub load_defaults {
2335
2336     my $res = {};
2337
2338     # we use static defaults from our JSON schema configuration
2339     foreach my $key (keys %$confdesc) {
2340         if (defined(my $default = $confdesc->{$key}->{default})) {
2341             $res->{$key} = $default;
2342         }
2343     }
2344
2345     return $res;
2346 }
2347
2348 sub config_list {
2349     my $vmlist = PVE::Cluster::get_vmlist();
2350     my $res = {};
2351     return $res if !$vmlist || !$vmlist->{ids};
2352     my $ids = $vmlist->{ids};
2353     my $nodename = nodename();
2354
2355     foreach my $vmid (keys %$ids) {
2356         my $d = $ids->{$vmid};
2357         next if !$d->{node} || $d->{node} ne $nodename;
2358         next if !$d->{type} || $d->{type} ne 'qemu';
2359         $res->{$vmid}->{exists} = 1;
2360     }
2361     return $res;
2362 }
2363
2364 # test if VM uses local resources (to prevent migration)
2365 sub check_local_resources {
2366     my ($conf, $noerr) = @_;
2367
2368     my @loc_res = ();
2369
2370     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2371     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2372
2373     push @loc_res, "ivshmem" if $conf->{ivshmem};
2374
2375     foreach my $k (keys %$conf) {
2376         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2377         # sockets are safe: they will recreated be on the target side post-migrate
2378         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2379         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2380     }
2381
2382     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2383
2384     return \@loc_res;
2385 }
2386
2387 # check if used storages are available on all nodes (use by migrate)
2388 sub check_storage_availability {
2389     my ($storecfg, $conf, $node) = @_;
2390
2391     PVE::QemuConfig->foreach_volume($conf, sub {
2392         my ($ds, $drive) = @_;
2393
2394         my $volid = $drive->{file};
2395         return if !$volid;
2396
2397         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2398         return if !$sid;
2399
2400         # check if storage is available on both nodes
2401         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2402         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2403    });
2404 }
2405
2406 # list nodes where all VM images are available (used by has_feature API)
2407 sub shared_nodes {
2408     my ($conf, $storecfg) = @_;
2409
2410     my $nodelist = PVE::Cluster::get_nodelist();
2411     my $nodehash = { map { $_ => 1 } @$nodelist };
2412     my $nodename = nodename();
2413
2414     PVE::QemuConfig->foreach_volume($conf, sub {
2415         my ($ds, $drive) = @_;
2416
2417         my $volid = $drive->{file};
2418         return if !$volid;
2419
2420         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2421         if ($storeid) {
2422             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2423             if ($scfg->{disable}) {
2424                 $nodehash = {};
2425             } elsif (my $avail = $scfg->{nodes}) {
2426                 foreach my $node (keys %$nodehash) {
2427                     delete $nodehash->{$node} if !$avail->{$node};
2428                 }
2429             } elsif (!$scfg->{shared}) {
2430                 foreach my $node (keys %$nodehash) {
2431                     delete $nodehash->{$node} if $node ne $nodename
2432                 }
2433             }
2434         }
2435     });
2436
2437     return $nodehash
2438 }
2439
2440 sub check_local_storage_availability {
2441     my ($conf, $storecfg) = @_;
2442
2443     my $nodelist = PVE::Cluster::get_nodelist();
2444     my $nodehash = { map { $_ => {} } @$nodelist };
2445
2446     PVE::QemuConfig->foreach_volume($conf, sub {
2447         my ($ds, $drive) = @_;
2448
2449         my $volid = $drive->{file};
2450         return if !$volid;
2451
2452         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2453         if ($storeid) {
2454             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2455
2456             if ($scfg->{disable}) {
2457                 foreach my $node (keys %$nodehash) {
2458                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2459                 }
2460             } elsif (my $avail = $scfg->{nodes}) {
2461                 foreach my $node (keys %$nodehash) {
2462                     if (!$avail->{$node}) {
2463                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2464                     }
2465                 }
2466             }
2467         }
2468     });
2469
2470     foreach my $node (values %$nodehash) {
2471         if (my $unavail = $node->{unavailable_storages}) {
2472             $node->{unavailable_storages} = [ sort keys %$unavail ];
2473         }
2474     }
2475
2476     return $nodehash
2477 }
2478
2479 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2480 sub check_running {
2481     my ($vmid, $nocheck, $node) = @_;
2482
2483     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2484     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2485 }
2486
2487 sub vzlist {
2488
2489     my $vzlist = config_list();
2490
2491     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2492
2493     while (defined(my $de = $fd->read)) {
2494         next if $de !~ m/^(\d+)\.pid$/;
2495         my $vmid = $1;
2496         next if !defined($vzlist->{$vmid});
2497         if (my $pid = check_running($vmid)) {
2498             $vzlist->{$vmid}->{pid} = $pid;
2499         }
2500     }
2501
2502     return $vzlist;
2503 }
2504
2505 our $vmstatus_return_properties = {
2506     vmid => get_standard_option('pve-vmid'),
2507     status => {
2508         description => "Qemu process status.",
2509         type => 'string',
2510         enum => ['stopped', 'running'],
2511     },
2512     maxmem => {
2513         description => "Maximum memory in bytes.",
2514         type => 'integer',
2515         optional => 1,
2516         renderer => 'bytes',
2517     },
2518     maxdisk => {
2519         description => "Root disk size in bytes.",
2520         type => 'integer',
2521         optional => 1,
2522         renderer => 'bytes',
2523     },
2524     name => {
2525         description => "VM name.",
2526         type => 'string',
2527         optional => 1,
2528     },
2529     qmpstatus => {
2530         description => "Qemu QMP agent status.",
2531         type => 'string',
2532         optional => 1,
2533     },
2534     pid => {
2535         description => "PID of running qemu process.",
2536         type => 'integer',
2537         optional => 1,
2538     },
2539     uptime => {
2540         description => "Uptime.",
2541         type => 'integer',
2542         optional => 1,
2543         renderer => 'duration',
2544     },
2545     cpus => {
2546         description => "Maximum usable CPUs.",
2547         type => 'number',
2548         optional => 1,
2549     },
2550     lock => {
2551         description => "The current config lock, if any.",
2552         type => 'string',
2553         optional => 1,
2554     },
2555     tags => {
2556         description  => "The current configured tags, if any",
2557         type => 'string',
2558         optional => 1,
2559     },
2560     'running-machine' => {
2561         description => "The currently running machine type (if running).",
2562         type => 'string',
2563         optional => 1,
2564     },
2565     'running-qemu' => {
2566         description => "The currently running QEMU version (if running).",
2567         type => 'string',
2568         optional => 1,
2569     },
2570 };
2571
2572 my $last_proc_pid_stat;
2573
2574 # get VM status information
2575 # This must be fast and should not block ($full == false)
2576 # We only query KVM using QMP if $full == true (this can be slow)
2577 sub vmstatus {
2578     my ($opt_vmid, $full) = @_;
2579
2580     my $res = {};
2581
2582     my $storecfg = PVE::Storage::config();
2583
2584     my $list = vzlist();
2585     my $defaults = load_defaults();
2586
2587     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2588
2589     my $cpucount = $cpuinfo->{cpus} || 1;
2590
2591     foreach my $vmid (keys %$list) {
2592         next if $opt_vmid && ($vmid ne $opt_vmid);
2593
2594         my $conf = PVE::QemuConfig->load_config($vmid);
2595
2596         my $d = { vmid => $vmid };
2597         $d->{pid} = $list->{$vmid}->{pid};
2598
2599         # fixme: better status?
2600         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2601
2602         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2603         if (defined($size)) {
2604             $d->{disk} = 0; # no info available
2605             $d->{maxdisk} = $size;
2606         } else {
2607             $d->{disk} = 0;
2608             $d->{maxdisk} = 0;
2609         }
2610
2611         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2612             * ($conf->{cores} || $defaults->{cores});
2613         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2614         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2615
2616         $d->{name} = $conf->{name} || "VM $vmid";
2617         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2618             : $defaults->{memory}*(1024*1024);
2619
2620         if ($conf->{balloon}) {
2621             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2622             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2623                 : $defaults->{shares};
2624         }
2625
2626         $d->{uptime} = 0;
2627         $d->{cpu} = 0;
2628         $d->{mem} = 0;
2629
2630         $d->{netout} = 0;
2631         $d->{netin} = 0;
2632
2633         $d->{diskread} = 0;
2634         $d->{diskwrite} = 0;
2635
2636         $d->{template} = PVE::QemuConfig->is_template($conf);
2637
2638         $d->{serial} = 1 if conf_has_serial($conf);
2639         $d->{lock} = $conf->{lock} if $conf->{lock};
2640         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2641
2642         $res->{$vmid} = $d;
2643     }
2644
2645     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2646     foreach my $dev (keys %$netdev) {
2647         next if $dev !~ m/^tap([1-9]\d*)i/;
2648         my $vmid = $1;
2649         my $d = $res->{$vmid};
2650         next if !$d;
2651
2652         $d->{netout} += $netdev->{$dev}->{receive};
2653         $d->{netin} += $netdev->{$dev}->{transmit};
2654
2655         if ($full) {
2656             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2657             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2658         }
2659
2660     }
2661
2662     my $ctime = gettimeofday;
2663
2664     foreach my $vmid (keys %$list) {
2665
2666         my $d = $res->{$vmid};
2667         my $pid = $d->{pid};
2668         next if !$pid;
2669
2670         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2671         next if !$pstat; # not running
2672
2673         my $used = $pstat->{utime} + $pstat->{stime};
2674
2675         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2676
2677         if ($pstat->{vsize}) {
2678             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2679         }
2680
2681         my $old = $last_proc_pid_stat->{$pid};
2682         if (!$old) {
2683             $last_proc_pid_stat->{$pid} = {
2684                 time => $ctime,
2685                 used => $used,
2686                 cpu => 0,
2687             };
2688             next;
2689         }
2690
2691         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2692
2693         if ($dtime > 1000) {
2694             my $dutime = $used -  $old->{used};
2695
2696             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2697             $last_proc_pid_stat->{$pid} = {
2698                 time => $ctime,
2699                 used => $used,
2700                 cpu => $d->{cpu},
2701             };
2702         } else {
2703             $d->{cpu} = $old->{cpu};
2704         }
2705     }
2706
2707     return $res if !$full;
2708
2709     my $qmpclient = PVE::QMPClient->new();
2710
2711     my $ballooncb = sub {
2712         my ($vmid, $resp) = @_;
2713
2714         my $info = $resp->{'return'};
2715         return if !$info->{max_mem};
2716
2717         my $d = $res->{$vmid};
2718
2719         # use memory assigned to VM
2720         $d->{maxmem} = $info->{max_mem};
2721         $d->{balloon} = $info->{actual};
2722
2723         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2724             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2725             $d->{freemem} = $info->{free_mem};
2726         }
2727
2728         $d->{ballooninfo} = $info;
2729     };
2730
2731     my $blockstatscb = sub {
2732         my ($vmid, $resp) = @_;
2733         my $data = $resp->{'return'} || [];
2734         my $totalrdbytes = 0;
2735         my $totalwrbytes = 0;
2736
2737         for my $blockstat (@$data) {
2738             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2739             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2740
2741             $blockstat->{device} =~ s/drive-//;
2742             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2743         }
2744         $res->{$vmid}->{diskread} = $totalrdbytes;
2745         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2746     };
2747
2748     my $machinecb = sub {
2749         my ($vmid, $resp) = @_;
2750         my $data = $resp->{'return'} || [];
2751
2752         $res->{$vmid}->{'running-machine'} =
2753             PVE::QemuServer::Machine::current_from_query_machines($data);
2754     };
2755
2756     my $versioncb = sub {
2757         my ($vmid, $resp) = @_;
2758         my $data = $resp->{'return'} // {};
2759         my $version = 'unknown';
2760
2761         if (my $v = $data->{qemu}) {
2762             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2763         }
2764
2765         $res->{$vmid}->{'running-qemu'} = $version;
2766     };
2767
2768     my $statuscb = sub {
2769         my ($vmid, $resp) = @_;
2770
2771         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2772         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2773         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2774         # this fails if ballon driver is not loaded, so this must be
2775         # the last commnand (following command are aborted if this fails).
2776         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2777
2778         my $status = 'unknown';
2779         if (!defined($status = $resp->{'return'}->{status})) {
2780             warn "unable to get VM status\n";
2781             return;
2782         }
2783
2784         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2785     };
2786
2787     foreach my $vmid (keys %$list) {
2788         next if $opt_vmid && ($vmid ne $opt_vmid);
2789         next if !$res->{$vmid}->{pid}; # not running
2790         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2791     }
2792
2793     $qmpclient->queue_execute(undef, 2);
2794
2795     foreach my $vmid (keys %$list) {
2796         next if $opt_vmid && ($vmid ne $opt_vmid);
2797         next if !$res->{$vmid}->{pid}; #not running
2798
2799         # we can't use the $qmpclient since it might have already aborted on
2800         # 'query-balloon', but this might also fail for older versions...
2801         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2802         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2803     }
2804
2805     foreach my $vmid (keys %$list) {
2806         next if $opt_vmid && ($vmid ne $opt_vmid);
2807         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2808     }
2809
2810     return $res;
2811 }
2812
2813 sub conf_has_serial {
2814     my ($conf) = @_;
2815
2816     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2817         if ($conf->{"serial$i"}) {
2818             return 1;
2819         }
2820     }
2821
2822     return 0;
2823 }
2824
2825 sub conf_has_audio {
2826     my ($conf, $id) = @_;
2827
2828     $id //= 0;
2829     my $audio = $conf->{"audio$id"};
2830     return if !defined($audio);
2831
2832     my $audioproperties = parse_property_string($audio_fmt, $audio);
2833     my $audiodriver = $audioproperties->{driver} // 'spice';
2834
2835     return {
2836         dev => $audioproperties->{device},
2837         dev_id => "audiodev$id",
2838         backend => $audiodriver,
2839         backend_id => "$audiodriver-backend${id}",
2840     };
2841 }
2842
2843 sub audio_devs {
2844     my ($audio, $audiopciaddr, $machine_version) = @_;
2845
2846     my $devs = [];
2847
2848     my $id = $audio->{dev_id};
2849     my $audiodev = "";
2850     if (min_version($machine_version, 4, 2)) {
2851         $audiodev = ",audiodev=$audio->{backend_id}";
2852     }
2853
2854     if ($audio->{dev} eq 'AC97') {
2855         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2856     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2857         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2858         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2859         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2860     } else {
2861         die "unkown audio device '$audio->{dev}', implement me!";
2862     }
2863
2864     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2865
2866     return $devs;
2867 }
2868
2869 sub vga_conf_has_spice {
2870     my ($vga) = @_;
2871
2872     my $vgaconf = parse_vga($vga);
2873     my $vgatype = $vgaconf->{type};
2874     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2875
2876     return $1 || 1;
2877 }
2878
2879 sub is_native($) {
2880     my ($arch) = @_;
2881     return get_host_arch() eq $arch;
2882 }
2883
2884 sub get_vm_arch {
2885     my ($conf) = @_;
2886     return $conf->{arch} // get_host_arch();
2887 }
2888
2889 my $default_machines = {
2890     x86_64 => 'pc',
2891     aarch64 => 'virt',
2892 };
2893
2894 sub get_installed_machine_version {
2895     my ($kvmversion) = @_;
2896     $kvmversion = kvm_user_version() if !defined($kvmversion);
2897     $kvmversion =~ m/^(\d+\.\d+)/;
2898     return $1;
2899 }
2900
2901 sub windows_get_pinned_machine_version {
2902     my ($machine, $base_version, $kvmversion) = @_;
2903
2904     my $pin_version = $base_version;
2905     if (!defined($base_version) ||
2906         !PVE::QemuServer::Machine::can_run_pve_machine_version($base_version, $kvmversion)
2907     ) {
2908         $pin_version = get_installed_machine_version($kvmversion);
2909     }
2910     if (!$machine || $machine eq 'pc') {
2911         $machine = "pc-i440fx-$pin_version";
2912     } elsif ($machine eq 'q35') {
2913         $machine = "pc-q35-$pin_version";
2914     } elsif ($machine eq 'virt') {
2915         $machine = "virt-$pin_version";
2916     } else {
2917         warn "unknown machine type '$machine', not touching that!\n";
2918     }
2919
2920     return $machine;
2921 }
2922
2923 sub get_vm_machine {
2924     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2925
2926     my $machine = $forcemachine || $conf->{machine};
2927
2928     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2929         $kvmversion //= kvm_user_version();
2930         # we must pin Windows VMs without a specific version to 5.1, as 5.2 fixed a bug in ACPI
2931         # layout which confuses windows quite a bit and may result in various regressions..
2932         # see: https://lists.gnu.org/archive/html/qemu-devel/2021-02/msg08484.html
2933         if (windows_version($conf->{ostype})) {
2934             $machine = windows_get_pinned_machine_version($machine, '5.1', $kvmversion);
2935         }
2936         $arch //= 'x86_64';
2937         $machine ||= $default_machines->{$arch};
2938         if ($add_pve_version) {
2939             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2940             $machine .= "+pve$pvever";
2941         }
2942     }
2943
2944     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2945         # for version-pinned machines that do not include a pve-version (e.g.
2946         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2947         $machine .= '+pve0';
2948     }
2949
2950     return $machine;
2951 }
2952
2953 sub get_ovmf_files($) {
2954     my ($arch) = @_;
2955
2956     my $ovmf = $OVMF->{$arch}
2957         or die "no OVMF images known for architecture '$arch'\n";
2958
2959     return @$ovmf;
2960 }
2961
2962 my $Arch2Qemu = {
2963     aarch64 => '/usr/bin/qemu-system-aarch64',
2964     x86_64 => '/usr/bin/qemu-system-x86_64',
2965 };
2966 sub get_command_for_arch($) {
2967     my ($arch) = @_;
2968     return '/usr/bin/kvm' if is_native($arch);
2969
2970     my $cmd = $Arch2Qemu->{$arch}
2971         or die "don't know how to emulate architecture '$arch'\n";
2972     return $cmd;
2973 }
2974
2975 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2976 # to use in a QEMU command line (-cpu element), first array_intersect the result
2977 # of query_supported_ with query_understood_. This is necessary because:
2978 #
2979 # a) query_understood_ returns flags the host cannot use and
2980 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2981 #    flags, but CPU settings - with most of them being flags. Those settings
2982 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2983 #
2984 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2985 # expensive. If you need the value returned from this, you can get it much
2986 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2987 # $accel being 'kvm' or 'tcg'.
2988 #
2989 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2990 # changes, automatically populating pmxcfs.
2991 #
2992 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2993 # since kvm and tcg machines support different flags
2994 #
2995 sub query_supported_cpu_flags {
2996     my ($arch) = @_;
2997
2998     $arch //= get_host_arch();
2999     my $default_machine = $default_machines->{$arch};
3000
3001     my $flags = {};
3002
3003     # FIXME: Once this is merged, the code below should work for ARM as well:
3004     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
3005     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
3006         $arch eq "aarch64";
3007
3008     my $kvm_supported = defined(kvm_version());
3009     my $qemu_cmd = get_command_for_arch($arch);
3010     my $fakevmid = -1;
3011     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
3012
3013     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
3014     my $query_supported_run_qemu = sub {
3015         my ($kvm) = @_;
3016
3017         my $flags = {};
3018         my $cmd = [
3019             $qemu_cmd,
3020             '-machine', $default_machine,
3021             '-display', 'none',
3022             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
3023             '-mon', 'chardev=qmp,mode=control',
3024             '-pidfile', $pidfile,
3025             '-S', '-daemonize'
3026         ];
3027
3028         if (!$kvm) {
3029             push @$cmd, '-accel', 'tcg';
3030         }
3031
3032         my $rc = run_command($cmd, noerr => 1, quiet => 0);
3033         die "QEMU flag querying VM exited with code " . $rc if $rc;
3034
3035         eval {
3036             my $cmd_result = mon_cmd(
3037                 $fakevmid,
3038                 'query-cpu-model-expansion',
3039                 type => 'full',
3040                 model => { name => 'host' }
3041             );
3042
3043             my $props = $cmd_result->{model}->{props};
3044             foreach my $prop (keys %$props) {
3045                 next if $props->{$prop} ne '1';
3046                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3047                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3048                 # We only keep those with underscores, to match /proc/cpuinfo
3049                 $prop =~ s/\.|-/_/g;
3050                 $flags->{$prop} = 1;
3051             }
3052         };
3053         my $err = $@;
3054
3055         # force stop with 10 sec timeout and 'nocheck'
3056         # always stop, even if QMP failed
3057         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3058
3059         die $err if $err;
3060
3061         return [ sort keys %$flags ];
3062     };
3063
3064     # We need to query QEMU twice, since KVM and TCG have different supported flags
3065     PVE::QemuConfig->lock_config($fakevmid, sub {
3066         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3067         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3068
3069         if ($kvm_supported) {
3070             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3071             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3072         }
3073     });
3074
3075     return $flags;
3076 }
3077
3078 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3079 my $understood_cpu_flag_dir = "/usr/share/kvm";
3080 sub query_understood_cpu_flags {
3081     my $arch = get_host_arch();
3082     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3083
3084     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3085         if ! -e $filepath;
3086
3087     my $raw = file_get_contents($filepath);
3088     $raw =~ s/^\s+|\s+$//g;
3089     my @flags = split(/\s+/, $raw);
3090
3091     return \@flags;
3092 }
3093
3094 sub config_to_command {
3095     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3096
3097     my $cmd = [];
3098     my $globalFlags = [];
3099     my $machineFlags = [];
3100     my $rtcFlags = [];
3101     my $devices = [];
3102     my $pciaddr = '';
3103     my $bridges = {};
3104     my $ostype = $conf->{ostype};
3105     my $winversion = windows_version($ostype);
3106     my $kvm = $conf->{kvm};
3107     my $nodename = nodename();
3108
3109     my $arch = get_vm_arch($conf);
3110     my $kvm_binary = get_command_for_arch($arch);
3111     my $kvmver = kvm_user_version($kvm_binary);
3112
3113     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3114         $kvmver //= "undefined";
3115         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3116     }
3117
3118     my $add_pve_version = min_version($kvmver, 4, 1);
3119
3120     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3121     my $machine_version = extract_version($machine_type, $kvmver);
3122     $kvm //= 1 if is_native($arch);
3123
3124     $machine_version =~ m/(\d+)\.(\d+)/;
3125     my ($machine_major, $machine_minor) = ($1, $2);
3126
3127     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3128         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3129     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3130         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3131             ." please upgrade node '$nodename'\n"
3132     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3133         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3134         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3135             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3136             ." node '$nodename'\n";
3137     }
3138
3139     # if a specific +pve version is required for a feature, use $version_guard
3140     # instead of min_version to allow machines to be run with the minimum
3141     # required version
3142     my $required_pve_version = 0;
3143     my $version_guard = sub {
3144         my ($major, $minor, $pve) = @_;
3145         return 0 if !min_version($machine_version, $major, $minor, $pve);
3146         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3147         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3148         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3149         return 1;
3150     };
3151
3152     if ($kvm && !defined kvm_version()) {
3153         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3154             ." or enable in BIOS.\n";
3155     }
3156
3157     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3158     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3159     my $use_old_bios_files = undef;
3160     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3161
3162     my $cpuunits = defined($conf->{cpuunits}) ?
3163             $conf->{cpuunits} : $defaults->{cpuunits};
3164
3165     push @$cmd, $kvm_binary;
3166
3167     push @$cmd, '-id', $vmid;
3168
3169     my $vmname = $conf->{name} || "vm$vmid";
3170
3171     push @$cmd, '-name', $vmname;
3172
3173     push @$cmd, '-no-shutdown';
3174
3175     my $use_virtio = 0;
3176
3177     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3178     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3179     push @$cmd, '-mon', "chardev=qmp,mode=control";
3180
3181     if (min_version($machine_version, 2, 12)) {
3182         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3183         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3184     }
3185
3186     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3187
3188     push @$cmd, '-daemonize';
3189
3190     if ($conf->{smbios1}) {
3191         my $smbios_conf = parse_smbios1($conf->{smbios1});
3192         if ($smbios_conf->{base64}) {
3193             # Do not pass base64 flag to qemu
3194             delete $smbios_conf->{base64};
3195             my $smbios_string = "";
3196             foreach my $key (keys %$smbios_conf) {
3197                 my $value;
3198                 if ($key eq "uuid") {
3199                     $value = $smbios_conf->{uuid}
3200                 } else {
3201                     $value = decode_base64($smbios_conf->{$key});
3202                 }
3203                 # qemu accepts any binary data, only commas need escaping by double comma
3204                 $value =~ s/,/,,/g;
3205                 $smbios_string .= "," . $key . "=" . $value if $value;
3206             }
3207             push @$cmd, '-smbios', "type=1" . $smbios_string;
3208         } else {
3209             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3210         }
3211     }
3212
3213     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3214         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3215         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3216
3217         my ($path, $format);
3218         if (my $efidisk = $conf->{efidisk0}) {
3219             my $d = parse_drive('efidisk0', $efidisk);
3220             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3221             $format = $d->{format};
3222             if ($storeid) {
3223                 $path = PVE::Storage::path($storecfg, $d->{file});
3224                 if (!defined($format)) {
3225                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3226                     $format = qemu_img_format($scfg, $volname);
3227                 }
3228             } else {
3229                 $path = $d->{file};
3230                 die "efidisk format must be specified\n"
3231                     if !defined($format);
3232             }
3233         } else {
3234             warn "no efidisk configured! Using temporary efivars disk.\n";
3235             $path = "/tmp/$vmid-ovmf.fd";
3236             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3237             $format = 'raw';
3238         }
3239
3240         my $size_str = "";
3241
3242         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3243             $size_str = ",size=" . (-s $ovmf_vars);
3244         }
3245
3246         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3247         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3248     }
3249
3250     # load q35 config
3251     if ($q35) {
3252         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3253         if (min_version($machine_version, 4, 0)) {
3254             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3255         } else {
3256             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3257         }
3258     }
3259
3260     if ($conf->{vmgenid}) {
3261         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3262     }
3263
3264     # add usb controllers
3265     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3266         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3267     push @$devices, @usbcontrollers if @usbcontrollers;
3268     my $vga = parse_vga($conf->{vga});
3269
3270     my $qxlnum = vga_conf_has_spice($conf->{vga});
3271     $vga->{type} = 'qxl' if $qxlnum;
3272
3273     if (!$vga->{type}) {
3274         if ($arch eq 'aarch64') {
3275             $vga->{type} = 'virtio';
3276         } elsif (min_version($machine_version, 2, 9)) {
3277             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3278         } else {
3279             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3280         }
3281     }
3282
3283     # enable absolute mouse coordinates (needed by vnc)
3284     my $tablet;
3285     if (defined($conf->{tablet})) {
3286         $tablet = $conf->{tablet};
3287     } else {
3288         $tablet = $defaults->{tablet};
3289         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3290         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3291     }
3292
3293     if ($tablet) {
3294         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3295         my $kbd = print_keyboarddevice_full($conf, $arch);
3296         push @$devices, '-device', $kbd if defined($kbd);
3297     }
3298
3299     my $bootorder = device_bootorder($conf);
3300
3301     # host pci device passthrough
3302     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3303         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3304
3305     # usb devices
3306     my $usb_dev_features = {};
3307     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3308
3309     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3310         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3311     push @$devices, @usbdevices if @usbdevices;
3312
3313     # serial devices
3314     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3315         if (my $path = $conf->{"serial$i"}) {
3316             if ($path eq 'socket') {
3317                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3318                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3319                 # On aarch64, serial0 is the UART device. Qemu only allows
3320                 # connecting UART devices via the '-serial' command line, as
3321                 # the device has a fixed slot on the hardware...
3322                 if ($arch eq 'aarch64' && $i == 0) {
3323                     push @$devices, '-serial', "chardev:serial$i";
3324                 } else {
3325                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3326                 }
3327             } else {
3328                 die "no such serial device\n" if ! -c $path;
3329                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3330                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3331             }
3332         }
3333     }
3334
3335     # parallel devices
3336     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3337         if (my $path = $conf->{"parallel$i"}) {
3338             die "no such parallel device\n" if ! -c $path;
3339             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3340             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3341             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3342         }
3343     }
3344
3345     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3346         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3347         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3348         push @$devices, @$audio_devs;
3349     }
3350
3351     my $sockets = 1;
3352     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3353     $sockets = $conf->{sockets} if  $conf->{sockets};
3354
3355     my $cores = $conf->{cores} || 1;
3356
3357     my $maxcpus = $sockets * $cores;
3358
3359     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3360
3361     my $allowed_vcpus = $cpuinfo->{cpus};
3362
3363     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3364         if ($allowed_vcpus < $maxcpus);
3365
3366     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3367
3368         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3369         for (my $i = 2; $i <= $vcpus; $i++)  {
3370             my $cpustr = print_cpu_device($conf,$i);
3371             push @$cmd, '-device', $cpustr;
3372         }
3373
3374     } else {
3375
3376         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3377     }
3378     push @$cmd, '-nodefaults';
3379
3380     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3381
3382     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3383
3384     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3385
3386     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3387         push @$devices, '-device', print_vga_device(
3388             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3389         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3390         push @$cmd,  '-vnc', "unix:$socket,password";
3391     } else {
3392         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3393         push @$cmd, '-nographic';
3394     }
3395
3396     # time drift fix
3397     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3398     my $useLocaltime = $conf->{localtime};
3399
3400     if ($winversion >= 5) { # windows
3401         $useLocaltime = 1 if !defined($conf->{localtime});
3402
3403         # use time drift fix when acpi is enabled
3404         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3405             $tdf = 1 if !defined($conf->{tdf});
3406         }
3407     }
3408
3409     if ($winversion >= 6) {
3410         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3411         push @$cmd, '-no-hpet';
3412     }
3413
3414     push @$rtcFlags, 'driftfix=slew' if $tdf;
3415
3416     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3417         push @$rtcFlags, "base=$conf->{startdate}";
3418     } elsif ($useLocaltime) {
3419         push @$rtcFlags, 'base=localtime';
3420     }
3421
3422     if ($forcecpu) {
3423         push @$cmd, '-cpu', $forcecpu;
3424     } else {
3425         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3426     }
3427
3428     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3429
3430     push @$cmd, '-S' if $conf->{freeze};
3431
3432     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3433
3434     my $guest_agent = parse_guest_agent($conf);
3435
3436     if ($guest_agent->{enabled}) {
3437         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3438         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3439
3440         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3441             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3442             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3443             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3444         } elsif ($guest_agent->{type} eq 'isa') {
3445             push @$devices, '-device', "isa-serial,chardev=qga0";
3446         }
3447     }
3448
3449     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3450     if ($rng && $version_guard->(4, 1, 2)) {
3451         check_rng_source($rng->{source});
3452
3453         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3454         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3455         my $limiter_str = "";
3456         if ($max_bytes) {
3457             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3458         }
3459
3460         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3461         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3462         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3463     }
3464
3465     my $spice_port;
3466
3467     if ($qxlnum) {
3468         if ($qxlnum > 1) {
3469             if ($winversion){
3470                 for (my $i = 1; $i < $qxlnum; $i++){
3471                     push @$devices, '-device', print_vga_device(
3472                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3473                 }
3474             } else {
3475                 # assume other OS works like Linux
3476                 my ($ram, $vram) = ("134217728", "67108864");
3477                 if ($vga->{memory}) {
3478                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3479                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3480                 }
3481                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3482                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3483             }
3484         }
3485
3486         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3487
3488         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3489         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3490         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3491
3492         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3493         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3494         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3495
3496         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3497         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3498
3499         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3500         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3501         if ($spice_enhancement->{foldersharing}) {
3502             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3503             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3504         }
3505
3506         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3507         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3508             if $spice_enhancement->{videostreaming};
3509
3510         push @$devices, '-spice', "$spice_opts";
3511     }
3512
3513     # enable balloon by default, unless explicitly disabled
3514     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3515         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3516         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3517     }
3518
3519     if ($conf->{watchdog}) {
3520         my $wdopts = parse_watchdog($conf->{watchdog});
3521         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3522         my $watchdog = $wdopts->{model} || 'i6300esb';
3523         push @$devices, '-device', "$watchdog$pciaddr";
3524         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3525     }
3526
3527     my $vollist = [];
3528     my $scsicontroller = {};
3529     my $ahcicontroller = {};
3530     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3531
3532     # Add iscsi initiator name if available
3533     if (my $initiator = get_initiator_name()) {
3534         push @$devices, '-iscsi', "initiator-name=$initiator";
3535     }
3536
3537     PVE::QemuConfig->foreach_volume($conf, sub {
3538         my ($ds, $drive) = @_;
3539
3540         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3541             push @$vollist, $drive->{file};
3542         }
3543
3544         # ignore efidisk here, already added in bios/fw handling code above
3545         return if $drive->{interface} eq 'efidisk';
3546
3547         $use_virtio = 1 if $ds =~ m/^virtio/;
3548
3549         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3550
3551         if ($drive->{interface} eq 'virtio'){
3552            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3553         }
3554
3555         if ($drive->{interface} eq 'scsi') {
3556
3557             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3558
3559             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3560                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3561
3562             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3563             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3564
3565             my $iothread = '';
3566             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3567                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3568                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3569             } elsif ($drive->{iothread}) {
3570                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3571             }
3572
3573             my $queues = '';
3574             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3575                 $queues = ",num_queues=$drive->{queues}";
3576             }
3577
3578             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3579                 if !$scsicontroller->{$controller};
3580             $scsicontroller->{$controller}=1;
3581         }
3582
3583         if ($drive->{interface} eq 'sata') {
3584             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3585             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3586             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3587                 if !$ahcicontroller->{$controller};
3588             $ahcicontroller->{$controller}=1;
3589         }
3590
3591         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3592         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3593
3594         push @$devices, '-drive',$drive_cmd;
3595         push @$devices, '-device', print_drivedevice_full(
3596             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3597     });
3598
3599     for (my $i = 0; $i < $MAX_NETS; $i++) {
3600         my $netname = "net$i";
3601
3602         next if !$conf->{$netname};
3603         my $d = parse_net($conf->{$netname});
3604         next if !$d;
3605
3606         $use_virtio = 1 if $d->{model} eq 'virtio';
3607
3608         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3609
3610         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3611         push @$devices, '-netdev', $netdevfull;
3612
3613         my $netdevicefull = print_netdevice_full(
3614             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3615
3616         push @$devices, '-device', $netdevicefull;
3617     }
3618
3619     if ($conf->{ivshmem}) {
3620         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3621
3622         my $bus;
3623         if ($q35) {
3624             $bus = print_pcie_addr("ivshmem");
3625         } else {
3626             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3627         }
3628
3629         my $ivshmem_name = $ivshmem->{name} // $vmid;
3630         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3631
3632         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3633         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3634             .",size=$ivshmem->{size}M";
3635     }
3636
3637     # pci.4 is nested in pci.1
3638     $bridges->{1} = 1 if $bridges->{4};
3639
3640     if (!$q35) {
3641         # add pci bridges
3642         if (min_version($machine_version, 2, 3)) {
3643            $bridges->{1} = 1;
3644            $bridges->{2} = 1;
3645         }
3646
3647         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3648
3649     }
3650
3651     for my $k (sort {$b cmp $a} keys %$bridges) {
3652         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3653
3654         my $k_name = $k;
3655         if ($k == 2 && $legacy_igd) {
3656             $k_name = "$k-igd";
3657         }
3658         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3659
3660         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3661         if ($q35) {
3662             # add after -readconfig pve-q35.cfg
3663             splice @$devices, 2, 0, '-device', $devstr;
3664         } else {
3665             unshift @$devices, '-device', $devstr if $k > 0;
3666         }
3667     }
3668
3669     if (!$kvm) {
3670         push @$machineFlags, 'accel=tcg';
3671     }
3672
3673     my $machine_type_min = $machine_type;
3674     if ($add_pve_version) {
3675         $machine_type_min =~ s/\+pve\d+$//;
3676         $machine_type_min .= "+pve$required_pve_version";
3677     }
3678     push @$machineFlags, "type=${machine_type_min}";
3679
3680     push @$cmd, @$devices;
3681     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3682     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3683     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3684
3685     if (my $vmstate = $conf->{vmstate}) {
3686         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3687         push @$vollist, $vmstate;
3688         push @$cmd, '-loadstate', $statepath;
3689         print "activating and using '$vmstate' as vmstate\n";
3690     }
3691
3692     # add custom args
3693     if ($conf->{args}) {
3694         my $aa = PVE::Tools::split_args($conf->{args});
3695         push @$cmd, @$aa;
3696     }
3697
3698     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3699 }
3700
3701 sub check_rng_source {
3702     my ($source) = @_;
3703
3704     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3705     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3706         if ! -e $source;
3707
3708     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3709     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3710         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3711         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3712         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3713             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3714             ." to the host.\n";
3715     }
3716 }
3717
3718 sub spice_port {
3719     my ($vmid) = @_;
3720
3721     my $res = mon_cmd($vmid, 'query-spice');
3722
3723     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3724 }
3725
3726 sub vm_devices_list {
3727     my ($vmid) = @_;
3728
3729     my $res = mon_cmd($vmid, 'query-pci');
3730     my $devices_to_check = [];
3731     my $devices = {};
3732     foreach my $pcibus (@$res) {
3733         push @$devices_to_check, @{$pcibus->{devices}},
3734     }
3735
3736     while (@$devices_to_check) {
3737         my $to_check = [];
3738         for my $d (@$devices_to_check) {
3739             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3740             next if !$d->{'pci_bridge'};
3741
3742             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3743             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3744         }
3745         $devices_to_check = $to_check;
3746     }
3747
3748     my $resblock = mon_cmd($vmid, 'query-block');
3749     foreach my $block (@$resblock) {
3750         if($block->{device} =~ m/^drive-(\S+)/){
3751                 $devices->{$1} = 1;
3752         }
3753     }
3754
3755     my $resmice = mon_cmd($vmid, 'query-mice');
3756     foreach my $mice (@$resmice) {
3757         if ($mice->{name} eq 'QEMU HID Tablet') {
3758             $devices->{tablet} = 1;
3759             last;
3760         }
3761     }
3762
3763     # for usb devices there is no query-usb
3764     # but we can iterate over the entries in
3765     # qom-list path=/machine/peripheral
3766     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3767     foreach my $per (@$resperipheral) {
3768         if ($per->{name} =~ m/^usb\d+$/) {
3769             $devices->{$per->{name}} = 1;
3770         }
3771     }
3772
3773     return $devices;
3774 }
3775
3776 sub vm_deviceplug {
3777     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3778
3779     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3780
3781     my $devices_list = vm_devices_list($vmid);
3782     return 1 if defined($devices_list->{$deviceid});
3783
3784     # add PCI bridge if we need it for the device
3785     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3786
3787     if ($deviceid eq 'tablet') {
3788
3789         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3790
3791     } elsif ($deviceid eq 'keyboard') {
3792
3793         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3794
3795     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3796
3797         die "usb hotplug currently not reliable\n";
3798         # since we can't reliably hot unplug all added usb devices and usb
3799         # passthrough breaks live migration we disable usb hotplugging for now
3800         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3801
3802     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3803
3804         qemu_iothread_add($vmid, $deviceid, $device);
3805
3806         qemu_driveadd($storecfg, $vmid, $device);
3807         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3808
3809         qemu_deviceadd($vmid, $devicefull);
3810         eval { qemu_deviceaddverify($vmid, $deviceid); };
3811         if (my $err = $@) {
3812             eval { qemu_drivedel($vmid, $deviceid); };
3813             warn $@ if $@;
3814             die $err;
3815         }
3816
3817     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3818
3819
3820         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3821         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3822         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3823
3824         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3825
3826         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3827             qemu_iothread_add($vmid, $deviceid, $device);
3828             $devicefull .= ",iothread=iothread-$deviceid";
3829         }
3830
3831         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3832             $devicefull .= ",num_queues=$device->{queues}";
3833         }
3834
3835         qemu_deviceadd($vmid, $devicefull);
3836         qemu_deviceaddverify($vmid, $deviceid);
3837
3838     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3839
3840         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3841         qemu_driveadd($storecfg, $vmid, $device);
3842
3843         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3844         eval { qemu_deviceadd($vmid, $devicefull); };
3845         if (my $err = $@) {
3846             eval { qemu_drivedel($vmid, $deviceid); };
3847             warn $@ if $@;
3848             die $err;
3849         }
3850
3851     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3852
3853         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3854
3855         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3856         my $use_old_bios_files = undef;
3857         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3858
3859         my $netdevicefull = print_netdevice_full(
3860             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3861         qemu_deviceadd($vmid, $netdevicefull);
3862         eval {
3863             qemu_deviceaddverify($vmid, $deviceid);
3864             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3865         };
3866         if (my $err = $@) {
3867             eval { qemu_netdevdel($vmid, $deviceid); };
3868             warn $@ if $@;
3869             die $err;
3870         }
3871
3872     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3873
3874         my $bridgeid = $2;
3875         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3876         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3877
3878         qemu_deviceadd($vmid, $devicefull);
3879         qemu_deviceaddverify($vmid, $deviceid);
3880
3881     } else {
3882         die "can't hotplug device '$deviceid'\n";
3883     }
3884
3885     return 1;
3886 }
3887
3888 # fixme: this should raise exceptions on error!
3889 sub vm_deviceunplug {
3890     my ($vmid, $conf, $deviceid) = @_;
3891
3892     my $devices_list = vm_devices_list($vmid);
3893     return 1 if !defined($devices_list->{$deviceid});
3894
3895     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3896     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3897
3898     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3899
3900         qemu_devicedel($vmid, $deviceid);
3901
3902     } elsif ($deviceid =~ m/^usb\d+$/) {
3903
3904         die "usb hotplug currently not reliable\n";
3905         # when unplugging usb devices this way, there may be remaining usb
3906         # controllers/hubs so we disable it for now
3907         #qemu_devicedel($vmid, $deviceid);
3908         #qemu_devicedelverify($vmid, $deviceid);
3909
3910     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3911
3912         qemu_devicedel($vmid, $deviceid);
3913         qemu_devicedelverify($vmid, $deviceid);
3914         qemu_drivedel($vmid, $deviceid);
3915         qemu_iothread_del($conf, $vmid, $deviceid);
3916
3917     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3918
3919         qemu_devicedel($vmid, $deviceid);
3920         qemu_devicedelverify($vmid, $deviceid);
3921         qemu_iothread_del($conf, $vmid, $deviceid);
3922
3923     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3924
3925         qemu_devicedel($vmid, $deviceid);
3926         qemu_drivedel($vmid, $deviceid);
3927         qemu_deletescsihw($conf, $vmid, $deviceid);
3928
3929     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3930
3931         qemu_devicedel($vmid, $deviceid);
3932         qemu_devicedelverify($vmid, $deviceid);
3933         qemu_netdevdel($vmid, $deviceid);
3934
3935     } else {
3936         die "can't unplug device '$deviceid'\n";
3937     }
3938
3939     return 1;
3940 }
3941
3942 sub qemu_deviceadd {
3943     my ($vmid, $devicefull) = @_;
3944
3945     $devicefull = "driver=".$devicefull;
3946     my %options =  split(/[=,]/, $devicefull);
3947
3948     mon_cmd($vmid, "device_add" , %options);
3949 }
3950
3951 sub qemu_devicedel {
3952     my ($vmid, $deviceid) = @_;
3953
3954     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3955 }
3956
3957 sub qemu_iothread_add {
3958     my($vmid, $deviceid, $device) = @_;
3959
3960     if ($device->{iothread}) {
3961         my $iothreads = vm_iothreads_list($vmid);
3962         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3963     }
3964 }
3965
3966 sub qemu_iothread_del {
3967     my($conf, $vmid, $deviceid) = @_;
3968
3969     my $confid = $deviceid;
3970     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3971         $confid = 'scsi' . $1;
3972     }
3973     my $device = parse_drive($confid, $conf->{$confid});
3974     if ($device->{iothread}) {
3975         my $iothreads = vm_iothreads_list($vmid);
3976         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3977     }
3978 }
3979
3980 sub qemu_objectadd {
3981     my($vmid, $objectid, $qomtype) = @_;
3982
3983     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3984
3985     return 1;
3986 }
3987
3988 sub qemu_objectdel {
3989     my($vmid, $objectid) = @_;
3990
3991     mon_cmd($vmid, "object-del", id => $objectid);
3992
3993     return 1;
3994 }
3995
3996 sub qemu_driveadd {
3997     my ($storecfg, $vmid, $device) = @_;
3998
3999     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
4000     $drive =~ s/\\/\\\\/g;
4001     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
4002
4003     # If the command succeeds qemu prints: "OK"
4004     return 1 if $ret =~ m/OK/s;
4005
4006     die "adding drive failed: $ret\n";
4007 }
4008
4009 sub qemu_drivedel {
4010     my($vmid, $deviceid) = @_;
4011
4012     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
4013     $ret =~ s/^\s+//;
4014
4015     return 1 if $ret eq "";
4016
4017     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4018     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4019
4020     die "deleting drive $deviceid failed : $ret\n";
4021 }
4022
4023 sub qemu_deviceaddverify {
4024     my ($vmid, $deviceid) = @_;
4025
4026     for (my $i = 0; $i <= 5; $i++) {
4027          my $devices_list = vm_devices_list($vmid);
4028          return 1 if defined($devices_list->{$deviceid});
4029          sleep 1;
4030     }
4031
4032     die "error on hotplug device '$deviceid'\n";
4033 }
4034
4035
4036 sub qemu_devicedelverify {
4037     my ($vmid, $deviceid) = @_;
4038
4039     # need to verify that the device is correctly removed as device_del
4040     # is async and empty return is not reliable
4041
4042     for (my $i = 0; $i <= 5; $i++) {
4043          my $devices_list = vm_devices_list($vmid);
4044          return 1 if !defined($devices_list->{$deviceid});
4045          sleep 1;
4046     }
4047
4048     die "error on hot-unplugging device '$deviceid'\n";
4049 }
4050
4051 sub qemu_findorcreatescsihw {
4052     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4053
4054     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4055
4056     my $scsihwid="$controller_prefix$controller";
4057     my $devices_list = vm_devices_list($vmid);
4058
4059     if(!defined($devices_list->{$scsihwid})) {
4060         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4061     }
4062
4063     return 1;
4064 }
4065
4066 sub qemu_deletescsihw {
4067     my ($conf, $vmid, $opt) = @_;
4068
4069     my $device = parse_drive($opt, $conf->{$opt});
4070
4071     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4072         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4073         return 1;
4074     }
4075
4076     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4077
4078     my $devices_list = vm_devices_list($vmid);
4079     foreach my $opt (keys %{$devices_list}) {
4080         if (is_valid_drivename($opt)) {
4081             my $drive = parse_drive($opt, $conf->{$opt});
4082             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4083                 return 1;
4084             }
4085         }
4086     }
4087
4088     my $scsihwid="scsihw$controller";
4089
4090     vm_deviceunplug($vmid, $conf, $scsihwid);
4091
4092     return 1;
4093 }
4094
4095 sub qemu_add_pci_bridge {
4096     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4097
4098     my $bridges = {};
4099
4100     my $bridgeid;
4101
4102     print_pci_addr($device, $bridges, $arch, $machine_type);
4103
4104     while (my ($k, $v) = each %$bridges) {
4105         $bridgeid = $k;
4106     }
4107     return 1 if !defined($bridgeid) || $bridgeid < 1;
4108
4109     my $bridge = "pci.$bridgeid";
4110     my $devices_list = vm_devices_list($vmid);
4111
4112     if (!defined($devices_list->{$bridge})) {
4113         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4114     }
4115
4116     return 1;
4117 }
4118
4119 sub qemu_set_link_status {
4120     my ($vmid, $device, $up) = @_;
4121
4122     mon_cmd($vmid, "set_link", name => $device,
4123                up => $up ? JSON::true : JSON::false);
4124 }
4125
4126 sub qemu_netdevadd {
4127     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4128
4129     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4130     my %options =  split(/[=,]/, $netdev);
4131
4132     if (defined(my $vhost = $options{vhost})) {
4133         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4134     }
4135
4136     if (defined(my $queues = $options{queues})) {
4137         $options{queues} = $queues + 0;
4138     }
4139
4140     mon_cmd($vmid, "netdev_add",  %options);
4141     return 1;
4142 }
4143
4144 sub qemu_netdevdel {
4145     my ($vmid, $deviceid) = @_;
4146
4147     mon_cmd($vmid, "netdev_del", id => $deviceid);
4148 }
4149
4150 sub qemu_usb_hotplug {
4151     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4152
4153     return if !$device;
4154
4155     # remove the old one first
4156     vm_deviceunplug($vmid, $conf, $deviceid);
4157
4158     # check if xhci controller is necessary and available
4159     if ($device->{usb3}) {
4160
4161         my $devicelist = vm_devices_list($vmid);
4162
4163         if (!$devicelist->{xhci}) {
4164             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4165             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4166         }
4167     }
4168     my $d = parse_usb_device($device->{host});
4169     $d->{usb3} = $device->{usb3};
4170
4171     # add the new one
4172     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4173 }
4174
4175 sub qemu_cpu_hotplug {
4176     my ($vmid, $conf, $vcpus) = @_;
4177
4178     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4179
4180     my $sockets = 1;
4181     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4182     $sockets = $conf->{sockets} if  $conf->{sockets};
4183     my $cores = $conf->{cores} || 1;
4184     my $maxcpus = $sockets * $cores;
4185
4186     $vcpus = $maxcpus if !$vcpus;
4187
4188     die "you can't add more vcpus than maxcpus\n"
4189         if $vcpus > $maxcpus;
4190
4191     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4192
4193     if ($vcpus < $currentvcpus) {
4194
4195         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4196
4197             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4198                 qemu_devicedel($vmid, "cpu$i");
4199                 my $retry = 0;
4200                 my $currentrunningvcpus = undef;
4201                 while (1) {
4202                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4203                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4204                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4205                     $retry++;
4206                     sleep 1;
4207                 }
4208                 #update conf after each succesfull cpu unplug
4209                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4210                 PVE::QemuConfig->write_config($vmid, $conf);
4211             }
4212         } else {
4213             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4214         }
4215
4216         return;
4217     }
4218
4219     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4220     die "vcpus in running vm does not match its configuration\n"
4221         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4222
4223     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4224
4225         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4226             my $cpustr = print_cpu_device($conf, $i);
4227             qemu_deviceadd($vmid, $cpustr);
4228
4229             my $retry = 0;
4230             my $currentrunningvcpus = undef;
4231             while (1) {
4232                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4233                 last if scalar(@{$currentrunningvcpus}) == $i;
4234                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4235                 sleep 1;
4236                 $retry++;
4237             }
4238             #update conf after each succesfull cpu hotplug
4239             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4240             PVE::QemuConfig->write_config($vmid, $conf);
4241         }
4242     } else {
4243
4244         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4245             mon_cmd($vmid, "cpu-add", id => int($i));
4246         }
4247     }
4248 }
4249
4250 sub qemu_block_set_io_throttle {
4251     my ($vmid, $deviceid,
4252         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4253         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4254         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4255         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4256
4257     return if !check_running($vmid) ;
4258
4259     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4260         bps => int($bps),
4261         bps_rd => int($bps_rd),
4262         bps_wr => int($bps_wr),
4263         iops => int($iops),
4264         iops_rd => int($iops_rd),
4265         iops_wr => int($iops_wr),
4266         bps_max => int($bps_max),
4267         bps_rd_max => int($bps_rd_max),
4268         bps_wr_max => int($bps_wr_max),
4269         iops_max => int($iops_max),
4270         iops_rd_max => int($iops_rd_max),
4271         iops_wr_max => int($iops_wr_max),
4272         bps_max_length => int($bps_max_length),
4273         bps_rd_max_length => int($bps_rd_max_length),
4274         bps_wr_max_length => int($bps_wr_max_length),
4275         iops_max_length => int($iops_max_length),
4276         iops_rd_max_length => int($iops_rd_max_length),
4277         iops_wr_max_length => int($iops_wr_max_length),
4278     );
4279
4280 }
4281
4282 sub qemu_block_resize {
4283     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4284
4285     my $running = check_running($vmid);
4286
4287     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4288
4289     return if !$running;
4290
4291     my $padding = (1024 - $size % 1024) % 1024;
4292     $size = $size + $padding;
4293
4294     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4295
4296 }
4297
4298 sub qemu_volume_snapshot {
4299     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4300
4301     my $running = check_running($vmid);
4302
4303     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4304         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4305     } else {
4306         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4307     }
4308 }
4309
4310 sub qemu_volume_snapshot_delete {
4311     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4312
4313     my $running = check_running($vmid);
4314
4315     if($running) {
4316
4317         $running = undef;
4318         my $conf = PVE::QemuConfig->load_config($vmid);
4319         PVE::QemuConfig->foreach_volume($conf, sub {
4320             my ($ds, $drive) = @_;
4321             $running = 1 if $drive->{file} eq $volid;
4322         });
4323     }
4324
4325     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4326         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4327     } else {
4328         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4329     }
4330 }
4331
4332 sub set_migration_caps {
4333     my ($vmid) = @_;
4334
4335     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4336
4337     my $cap_ref = [];
4338
4339     my $enabled_cap = {
4340         "auto-converge" => 1,
4341         "xbzrle" => 1,
4342         "x-rdma-pin-all" => 0,
4343         "zero-blocks" => 0,
4344         "compress" => 0,
4345         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4346     };
4347
4348     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4349
4350     for my $supported_capability (@$supported_capabilities) {
4351         push @$cap_ref, {
4352             capability => $supported_capability->{capability},
4353             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4354         };
4355     }
4356
4357     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4358 }
4359
4360 sub foreach_volid {
4361     my ($conf, $func, @param) = @_;
4362
4363     my $volhash = {};
4364
4365     my $test_volid = sub {
4366         my ($key, $drive, $snapname) = @_;
4367
4368         my $volid = $drive->{file};
4369         return if !$volid;
4370
4371         $volhash->{$volid}->{cdrom} //= 1;
4372         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4373
4374         my $replicate = $drive->{replicate} // 1;
4375         $volhash->{$volid}->{replicate} //= 0;
4376         $volhash->{$volid}->{replicate} = 1 if $replicate;
4377
4378         $volhash->{$volid}->{shared} //= 0;
4379         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4380
4381         $volhash->{$volid}->{referenced_in_config} //= 0;
4382         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4383
4384         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4385             if defined($snapname);
4386
4387         my $size = $drive->{size};
4388         $volhash->{$volid}->{size} //= $size if $size;
4389
4390         $volhash->{$volid}->{is_vmstate} //= 0;
4391         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4392
4393         $volhash->{$volid}->{is_unused} //= 0;
4394         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4395     };
4396
4397     my $include_opts = {
4398         extra_keys => ['vmstate'],
4399         include_unused => 1,
4400     };
4401
4402     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4403     foreach my $snapname (keys %{$conf->{snapshots}}) {
4404         my $snap = $conf->{snapshots}->{$snapname};
4405         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4406     }
4407
4408     foreach my $volid (keys %$volhash) {
4409         &$func($volid, $volhash->{$volid}, @param);
4410     }
4411 }
4412
4413 my $fast_plug_option = {
4414     'lock' => 1,
4415     'name' => 1,
4416     'onboot' => 1,
4417     'shares' => 1,
4418     'startup' => 1,
4419     'description' => 1,
4420     'protection' => 1,
4421     'vmstatestorage' => 1,
4422     'hookscript' => 1,
4423     'tags' => 1,
4424 };
4425
4426 # hotplug changes in [PENDING]
4427 # $selection hash can be used to only apply specified options, for
4428 # example: { cores => 1 } (only apply changed 'cores')
4429 # $errors ref is used to return error messages
4430 sub vmconfig_hotplug_pending {
4431     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4432
4433     my $defaults = load_defaults();
4434     my $arch = get_vm_arch($conf);
4435     my $machine_type = get_vm_machine($conf, undef, $arch);
4436
4437     # commit values which do not have any impact on running VM first
4438     # Note: those option cannot raise errors, we we do not care about
4439     # $selection and always apply them.
4440
4441     my $add_error = sub {
4442         my ($opt, $msg) = @_;
4443         $errors->{$opt} = "hotplug problem - $msg";
4444     };
4445
4446     my $changes = 0;
4447     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4448         if ($fast_plug_option->{$opt}) {
4449             $conf->{$opt} = $conf->{pending}->{$opt};
4450             delete $conf->{pending}->{$opt};
4451             $changes = 1;
4452         }
4453     }
4454
4455     if ($changes) {
4456         PVE::QemuConfig->write_config($vmid, $conf);
4457     }
4458
4459     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4460
4461     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4462     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4463     foreach my $opt (sort keys %$pending_delete_hash) {
4464         next if $selection && !$selection->{$opt};
4465         my $force = $pending_delete_hash->{$opt}->{force};
4466         eval {
4467             if ($opt eq 'hotplug') {
4468                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4469             } elsif ($opt eq 'tablet') {
4470                 die "skip\n" if !$hotplug_features->{usb};
4471                 if ($defaults->{tablet}) {
4472                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4473                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4474                         if $arch eq 'aarch64';
4475                 } else {
4476                     vm_deviceunplug($vmid, $conf, 'tablet');
4477                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4478                 }
4479             } elsif ($opt =~ m/^usb\d+/) {
4480                 die "skip\n";
4481                 # since we cannot reliably hot unplug usb devices we are disabling it
4482                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4483                 #vm_deviceunplug($vmid, $conf, $opt);
4484             } elsif ($opt eq 'vcpus') {
4485                 die "skip\n" if !$hotplug_features->{cpu};
4486                 qemu_cpu_hotplug($vmid, $conf, undef);
4487             } elsif ($opt eq 'balloon') {
4488                 # enable balloon device is not hotpluggable
4489                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4490                 # here we reset the ballooning value to memory
4491                 my $balloon = $conf->{memory} || $defaults->{memory};
4492                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4493             } elsif ($fast_plug_option->{$opt}) {
4494                 # do nothing
4495             } elsif ($opt =~ m/^net(\d+)$/) {
4496                 die "skip\n" if !$hotplug_features->{network};
4497                 vm_deviceunplug($vmid, $conf, $opt);
4498             } elsif (is_valid_drivename($opt)) {
4499                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4500                 vm_deviceunplug($vmid, $conf, $opt);
4501                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4502             } elsif ($opt =~ m/^memory$/) {
4503                 die "skip\n" if !$hotplug_features->{memory};
4504                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4505             } elsif ($opt eq 'cpuunits') {
4506                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4507             } elsif ($opt eq 'cpulimit') {
4508                 $cgroup->change_cpu_quota(-1, 100000);
4509             } else {
4510                 die "skip\n";
4511             }
4512         };
4513         if (my $err = $@) {
4514             &$add_error($opt, $err) if $err ne "skip\n";
4515         } else {
4516             delete $conf->{$opt};
4517             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4518         }
4519     }
4520
4521     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4522     $apply_pending_cloudinit = sub {
4523         return if $apply_pending_cloudinit_done; # once is enough
4524         $apply_pending_cloudinit_done = 1; # once is enough
4525
4526         my ($key, $value) = @_;
4527
4528         my @cloudinit_opts = keys %$confdesc_cloudinit;
4529         foreach my $opt (keys %{$conf->{pending}}) {
4530             next if !grep { $_ eq $opt } @cloudinit_opts;
4531             $conf->{$opt} = delete $conf->{pending}->{$opt};
4532         }
4533
4534         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4535         foreach my $opt (sort keys %$pending_delete_hash) {
4536             next if !grep { $_ eq $opt } @cloudinit_opts;
4537             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4538             delete $conf->{$opt};
4539         }
4540
4541         my $new_conf = { %$conf };
4542         $new_conf->{$key} = $value;
4543         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4544     };
4545
4546     foreach my $opt (keys %{$conf->{pending}}) {
4547         next if $selection && !$selection->{$opt};
4548         my $value = $conf->{pending}->{$opt};
4549         eval {
4550             if ($opt eq 'hotplug') {
4551                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4552             } elsif ($opt eq 'tablet') {
4553                 die "skip\n" if !$hotplug_features->{usb};
4554                 if ($value == 1) {
4555                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4556                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4557                         if $arch eq 'aarch64';
4558                 } elsif ($value == 0) {
4559                     vm_deviceunplug($vmid, $conf, 'tablet');
4560                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4561                 }
4562             } elsif ($opt =~ m/^usb\d+$/) {
4563                 die "skip\n";
4564                 # since we cannot reliably hot unplug usb devices we disable it for now
4565                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4566                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4567                 #die "skip\n" if !$d;
4568                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4569             } elsif ($opt eq 'vcpus') {
4570                 die "skip\n" if !$hotplug_features->{cpu};
4571                 qemu_cpu_hotplug($vmid, $conf, $value);
4572             } elsif ($opt eq 'balloon') {
4573                 # enable/disable balloning device is not hotpluggable
4574                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4575                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4576                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4577
4578                 # allow manual ballooning if shares is set to zero
4579                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4580                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4581                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4582                 }
4583             } elsif ($opt =~ m/^net(\d+)$/) {
4584                 # some changes can be done without hotplug
4585                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4586                                     $vmid, $opt, $value, $arch, $machine_type);
4587             } elsif (is_valid_drivename($opt)) {
4588                 die "skip\n" if $opt eq 'efidisk0';
4589                 # some changes can be done without hotplug
4590                 my $drive = parse_drive($opt, $value);
4591                 if (drive_is_cloudinit($drive)) {
4592                     &$apply_pending_cloudinit($opt, $value);
4593                 }
4594                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4595                                      $vmid, $opt, $value, $arch, $machine_type);
4596             } elsif ($opt =~ m/^memory$/) { #dimms
4597                 die "skip\n" if !$hotplug_features->{memory};
4598                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4599             } elsif ($opt eq 'cpuunits') {
4600                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4601             } elsif ($opt eq 'cpulimit') {
4602                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4603                 $cgroup->change_cpu_quota($cpulimit, 100000);
4604             } else {
4605                 die "skip\n";  # skip non-hot-pluggable options
4606             }
4607         };
4608         if (my $err = $@) {
4609             &$add_error($opt, $err) if $err ne "skip\n";
4610         } else {
4611             $conf->{$opt} = $value;
4612             delete $conf->{pending}->{$opt};
4613         }
4614     }
4615
4616     PVE::QemuConfig->write_config($vmid, $conf);
4617 }
4618
4619 sub try_deallocate_drive {
4620     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4621
4622     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4623         my $volid = $drive->{file};
4624         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4625             my $sid = PVE::Storage::parse_volume_id($volid);
4626             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4627
4628             # check if the disk is really unused
4629             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4630                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4631             PVE::Storage::vdisk_free($storecfg, $volid);
4632             return 1;
4633         } else {
4634             # If vm is not owner of this disk remove from config
4635             return 1;
4636         }
4637     }
4638
4639     return;
4640 }
4641
4642 sub vmconfig_delete_or_detach_drive {
4643     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4644
4645     my $drive = parse_drive($opt, $conf->{$opt});
4646
4647     my $rpcenv = PVE::RPCEnvironment::get();
4648     my $authuser = $rpcenv->get_user();
4649
4650     if ($force) {
4651         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4652         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4653     } else {
4654         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4655     }
4656 }
4657
4658
4659
4660 sub vmconfig_apply_pending {
4661     my ($vmid, $conf, $storecfg, $errors) = @_;
4662
4663     my $add_apply_error = sub {
4664         my ($opt, $msg) = @_;
4665         my $err_msg = "unable to apply pending change $opt : $msg";
4666         $errors->{$opt} = $err_msg;
4667         warn $err_msg;
4668     };
4669
4670     # cold plug
4671
4672     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4673     foreach my $opt (sort keys %$pending_delete_hash) {
4674         my $force = $pending_delete_hash->{$opt}->{force};
4675         eval {
4676             if ($opt =~ m/^unused/) {
4677                 die "internal error";
4678             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4679                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4680             }
4681         };
4682         if (my $err = $@) {
4683             $add_apply_error->($opt, $err);
4684         } else {
4685             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4686             delete $conf->{$opt};
4687         }
4688     }
4689
4690     PVE::QemuConfig->cleanup_pending($conf);
4691
4692     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4693         next if $opt eq 'delete'; # just to be sure
4694         eval {
4695             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4696                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4697             }
4698         };
4699         if (my $err = $@) {
4700             $add_apply_error->($opt, $err);
4701         } else {
4702             $conf->{$opt} = delete $conf->{pending}->{$opt};
4703         }
4704     }
4705
4706     # write all changes at once to avoid unnecessary i/o
4707     PVE::QemuConfig->write_config($vmid, $conf);
4708 }
4709
4710 sub vmconfig_update_net {
4711     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4712
4713     my $newnet = parse_net($value);
4714
4715     if ($conf->{$opt}) {
4716         my $oldnet = parse_net($conf->{$opt});
4717
4718         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4719             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4720             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4721             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4722
4723             # for non online change, we try to hot-unplug
4724             die "skip\n" if !$hotplug;
4725             vm_deviceunplug($vmid, $conf, $opt);
4726         } else {
4727
4728             die "internal error" if $opt !~ m/net(\d+)/;
4729             my $iface = "tap${vmid}i$1";
4730
4731             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4732                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4733                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4734                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4735                 PVE::Network::tap_unplug($iface);
4736
4737                 if ($have_sdn) {
4738                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4739                 } else {
4740                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4741                 }
4742             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4743                 # Rate can be applied on its own but any change above needs to
4744                 # include the rate in tap_plug since OVS resets everything.
4745                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4746             }
4747
4748             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4749                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4750             }
4751
4752             return 1;
4753         }
4754     }
4755
4756     if ($hotplug) {
4757         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4758     } else {
4759         die "skip\n";
4760     }
4761 }
4762
4763 sub vmconfig_update_disk {
4764     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4765
4766     my $drive = parse_drive($opt, $value);
4767
4768     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4769         my $media = $drive->{media} || 'disk';
4770         my $oldmedia = $old_drive->{media} || 'disk';
4771         die "unable to change media type\n" if $media ne $oldmedia;
4772
4773         if (!drive_is_cdrom($old_drive)) {
4774
4775             if ($drive->{file} ne $old_drive->{file}) {
4776
4777                 die "skip\n" if !$hotplug;
4778
4779                 # unplug and register as unused
4780                 vm_deviceunplug($vmid, $conf, $opt);
4781                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4782
4783             } else {
4784                 # update existing disk
4785
4786                 # skip non hotpluggable value
4787                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4788                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4789                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4790                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4791                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4792                     die "skip\n";
4793                 }
4794
4795                 # apply throttle
4796                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4797                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4798                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4799                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4800                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4801                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4802                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4803                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4804                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4805                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4806                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4807                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4808                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4809                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4810                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4811                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4812                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4813                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4814
4815                     qemu_block_set_io_throttle(
4816                         $vmid,"drive-$opt",
4817                         ($drive->{mbps} || 0)*1024*1024,
4818                         ($drive->{mbps_rd} || 0)*1024*1024,
4819                         ($drive->{mbps_wr} || 0)*1024*1024,
4820                          $drive->{iops} || 0,
4821                          $drive->{iops_rd} || 0,
4822                          $drive->{iops_wr} || 0,
4823                         ($drive->{mbps_max} || 0)*1024*1024,
4824                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4825                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4826                          $drive->{iops_max} || 0,
4827                          $drive->{iops_rd_max} || 0,
4828                          $drive->{iops_wr_max} || 0,
4829                          $drive->{bps_max_length} || 1,
4830                          $drive->{bps_rd_max_length} || 1,
4831                          $drive->{bps_wr_max_length} || 1,
4832                          $drive->{iops_max_length} || 1,
4833                          $drive->{iops_rd_max_length} || 1,
4834                          $drive->{iops_wr_max_length} || 1,
4835                     );
4836
4837                 }
4838
4839                 return 1;
4840             }
4841
4842         } else { # cdrom
4843
4844             if ($drive->{file} eq 'none') {
4845                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4846                 if (drive_is_cloudinit($old_drive)) {
4847                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4848                 }
4849             } else {
4850                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4851
4852                 # force eject if locked
4853                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4854
4855                 if ($path) {
4856                     mon_cmd($vmid, "blockdev-change-medium",
4857                         id => "$opt", filename => "$path");
4858                 }
4859             }
4860
4861             return 1;
4862         }
4863     }
4864
4865     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4866     # hotplug new disks
4867     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4868     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4869 }
4870
4871 # called in locked context by incoming migration
4872 sub vm_migrate_get_nbd_disks {
4873     my ($storecfg, $conf, $replicated_volumes) = @_;
4874
4875     my $local_volumes = {};
4876     PVE::QemuConfig->foreach_volume($conf, sub {
4877         my ($ds, $drive) = @_;
4878
4879         return if drive_is_cdrom($drive);
4880
4881         my $volid = $drive->{file};
4882
4883         return if !$volid;
4884
4885         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4886
4887         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4888         return if $scfg->{shared};
4889
4890         # replicated disks re-use existing state via bitmap
4891         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4892         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4893     });
4894     return $local_volumes;
4895 }
4896
4897 # called in locked context by incoming migration
4898 sub vm_migrate_alloc_nbd_disks {
4899     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4900
4901     my $format = undef;
4902
4903     my $nbd = {};
4904     foreach my $opt (sort keys %$source_volumes) {
4905         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4906
4907         if ($use_existing) {
4908             $nbd->{$opt}->{drivestr} = print_drive($drive);
4909             $nbd->{$opt}->{volid} = $volid;
4910             $nbd->{$opt}->{replicated} = 1;
4911             next;
4912         }
4913
4914         # If a remote storage is specified and the format of the original
4915         # volume is not available there, fall back to the default format.
4916         # Otherwise use the same format as the original.
4917         if (!$storagemap->{identity}) {
4918             $storeid = map_storage($storagemap, $storeid);
4919             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4920             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4921             my $fileFormat = qemu_img_format($scfg, $volname);
4922             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4923         } else {
4924             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4925             $format = qemu_img_format($scfg, $volname);
4926         }
4927
4928         my $size = $drive->{size} / 1024;
4929         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4930         my $newdrive = $drive;
4931         $newdrive->{format} = $format;
4932         $newdrive->{file} = $newvolid;
4933         my $drivestr = print_drive($newdrive);
4934         $nbd->{$opt}->{drivestr} = $drivestr;
4935         $nbd->{$opt}->{volid} = $newvolid;
4936     }
4937
4938     return $nbd;
4939 }
4940
4941 # see vm_start_nolock for parameters, additionally:
4942 # migrate_opts:
4943 #   storagemap = parsed storage map for allocating NBD disks
4944 sub vm_start {
4945     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4946
4947     return PVE::QemuConfig->lock_config($vmid, sub {
4948         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4949
4950         die "you can't start a vm if it's a template\n"
4951             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4952
4953         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4954         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4955
4956         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4957
4958         if ($has_backup_lock && $running) {
4959             # a backup is currently running, attempt to start the guest in the
4960             # existing QEMU instance
4961             return vm_resume($vmid);
4962         }
4963
4964         PVE::QemuConfig->check_lock($conf)
4965             if !($params->{skiplock} || $has_suspended_lock);
4966
4967         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4968
4969         die "VM $vmid already running\n" if $running;
4970
4971         if (my $storagemap = $migrate_opts->{storagemap}) {
4972             my $replicated = $migrate_opts->{replicated_volumes};
4973             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4974             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4975
4976             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4977                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4978             }
4979         }
4980
4981         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4982     });
4983 }
4984
4985
4986 # params:
4987 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4988 #   skiplock => 0/1, skip checking for config lock
4989 #   skiptemplate => 0/1, skip checking whether VM is template
4990 #   forcemachine => to force Qemu machine (rollback/migration)
4991 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4992 #   timeout => in seconds
4993 #   paused => start VM in paused state (backup)
4994 #   resume => resume from hibernation
4995 # migrate_opts:
4996 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4997 #   migratedfrom => source node
4998 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4999 #   network => CIDR of migration network
5000 #   type => secure/insecure - tunnel over encrypted connection or plain-text
5001 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
5002 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
5003 sub vm_start_nolock {
5004     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
5005
5006     my $statefile = $params->{statefile};
5007     my $resume = $params->{resume};
5008
5009     my $migratedfrom = $migrate_opts->{migratedfrom};
5010     my $migration_type = $migrate_opts->{type};
5011
5012     my $res = {};
5013
5014     # clean up leftover reboot request files
5015     eval { clear_reboot_request($vmid); };
5016     warn $@ if $@;
5017
5018     if (!$statefile && scalar(keys %{$conf->{pending}})) {
5019         vmconfig_apply_pending($vmid, $conf, $storecfg);
5020         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5021     }
5022
5023     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5024
5025     my $defaults = load_defaults();
5026
5027     # set environment variable useful inside network script
5028     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5029
5030     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5031
5032     my $forcemachine = $params->{forcemachine};
5033     my $forcecpu = $params->{forcecpu};
5034     if ($resume) {
5035         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5036         $forcemachine = $conf->{runningmachine};
5037         $forcecpu = $conf->{runningcpu};
5038         print "Resuming suspended VM\n";
5039     }
5040
5041     my ($cmd, $vollist, $spice_port) =
5042         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5043
5044     my $migration_ip;
5045     my $get_migration_ip = sub {
5046         my ($nodename) = @_;
5047
5048         return $migration_ip if defined($migration_ip);
5049
5050         my $cidr = $migrate_opts->{network};
5051
5052         if (!defined($cidr)) {
5053             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5054             $cidr = $dc_conf->{migration}->{network};
5055         }
5056
5057         if (defined($cidr)) {
5058             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5059
5060             die "could not get IP: no address configured on local " .
5061                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5062
5063             die "could not get IP: multiple addresses configured on local " .
5064                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5065
5066             $migration_ip = @$ips[0];
5067         }
5068
5069         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5070             if !defined($migration_ip);
5071
5072         return $migration_ip;
5073     };
5074
5075     my $migrate_uri;
5076     if ($statefile) {
5077         if ($statefile eq 'tcp') {
5078             my $localip = "localhost";
5079             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5080             my $nodename = nodename();
5081
5082             if (!defined($migration_type)) {
5083                 if (defined($datacenterconf->{migration}->{type})) {
5084                     $migration_type = $datacenterconf->{migration}->{type};
5085                 } else {
5086                     $migration_type = 'secure';
5087                 }
5088             }
5089
5090             if ($migration_type eq 'insecure') {
5091                 $localip = $get_migration_ip->($nodename);
5092                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5093             }
5094
5095             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5096             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5097             $migrate_uri = "tcp:${localip}:${migrate_port}";
5098             push @$cmd, '-incoming', $migrate_uri;
5099             push @$cmd, '-S';
5100
5101         } elsif ($statefile eq 'unix') {
5102             # should be default for secure migrations as a ssh TCP forward
5103             # tunnel is not deterministic reliable ready and fails regurarly
5104             # to set up in time, so use UNIX socket forwards
5105             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5106             unlink $socket_addr;
5107
5108             $migrate_uri = "unix:$socket_addr";
5109
5110             push @$cmd, '-incoming', $migrate_uri;
5111             push @$cmd, '-S';
5112
5113         } elsif (-e $statefile) {
5114             push @$cmd, '-loadstate', $statefile;
5115         } else {
5116             my $statepath = PVE::Storage::path($storecfg, $statefile);
5117             push @$vollist, $statefile;
5118             push @$cmd, '-loadstate', $statepath;
5119         }
5120     } elsif ($params->{paused}) {
5121         push @$cmd, '-S';
5122     }
5123
5124     # host pci devices
5125     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5126       my $d = parse_hostpci($conf->{"hostpci$i"});
5127       next if !$d;
5128       my $pcidevices = $d->{pciid};
5129       foreach my $pcidevice (@$pcidevices) {
5130             my $pciid = $pcidevice->{id};
5131
5132             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5133             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5134             die "no pci device info for device '$pciid'\n" if !$info;
5135
5136             if ($d->{mdev}) {
5137                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5138                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5139             } else {
5140                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5141                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5142                 die "can't reset PCI device '$pciid'\n"
5143                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5144             }
5145       }
5146     }
5147
5148     PVE::Storage::activate_volumes($storecfg, $vollist);
5149
5150     eval {
5151         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5152             outfunc => sub {}, errfunc => sub {});
5153     };
5154     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5155     # timeout should be more than enough here...
5156     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5157
5158     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5159                                               : $defaults->{cpuunits};
5160
5161     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5162     my %run_params = (
5163         timeout => $statefile ? undef : $start_timeout,
5164         umask => 0077,
5165         noerr => 1,
5166     );
5167
5168     # when migrating, prefix QEMU output so other side can pick up any
5169     # errors that might occur and show the user
5170     if ($migratedfrom) {
5171         $run_params{quiet} = 1;
5172         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5173     }
5174
5175     my %properties = (
5176         Slice => 'qemu.slice',
5177         KillMode => 'none'
5178     );
5179
5180     if (PVE::CGroup::cgroup_mode() == 2) {
5181         $properties{CPUWeight} = $cpuunits;
5182     } else {
5183         $properties{CPUShares} = $cpuunits;
5184     }
5185
5186     if (my $cpulimit = $conf->{cpulimit}) {
5187         $properties{CPUQuota} = int($cpulimit * 100);
5188     }
5189     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5190
5191     my $run_qemu = sub {
5192         PVE::Tools::run_fork sub {
5193             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5194
5195             my $exitcode = run_command($cmd, %run_params);
5196             die "QEMU exited with code $exitcode\n" if $exitcode;
5197         };
5198     };
5199
5200     if ($conf->{hugepages}) {
5201
5202         my $code = sub {
5203             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5204             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5205
5206             PVE::QemuServer::Memory::hugepages_mount();
5207             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5208
5209             eval { $run_qemu->() };
5210             if (my $err = $@) {
5211                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5212                     if !$conf->{keephugepages};
5213                 die $err;
5214             }
5215
5216             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5217                 if !$conf->{keephugepages};
5218         };
5219         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5220
5221     } else {
5222         eval { $run_qemu->() };
5223     }
5224
5225     if (my $err = $@) {
5226         # deactivate volumes if start fails
5227         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5228         die "start failed: $err";
5229     }
5230
5231     print "migration listens on $migrate_uri\n" if $migrate_uri;
5232     $res->{migrate_uri} = $migrate_uri;
5233
5234     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5235         eval { mon_cmd($vmid, "cont"); };
5236         warn $@ if $@;
5237     }
5238
5239     #start nbd server for storage migration
5240     if (my $nbd = $migrate_opts->{nbd}) {
5241         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5242
5243         my $migrate_storage_uri;
5244         # nbd_protocol_version > 0 for unix socket support
5245         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5246             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5247             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5248             $migrate_storage_uri = "nbd:unix:$socket_path";
5249         } else {
5250             my $nodename = nodename();
5251             my $localip = $get_migration_ip->($nodename);
5252             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5253             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5254
5255             mon_cmd($vmid, "nbd-server-start", addr => {
5256                 type => 'inet',
5257                 data => {
5258                     host => "${localip}",
5259                     port => "${storage_migrate_port}",
5260                 },
5261             });
5262             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5263             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5264         }
5265
5266         $res->{migrate_storage_uri} = $migrate_storage_uri;
5267
5268         foreach my $opt (sort keys %$nbd) {
5269             my $drivestr = $nbd->{$opt}->{drivestr};
5270             my $volid = $nbd->{$opt}->{volid};
5271             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5272             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5273             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5274             print "re-using replicated volume: $opt - $volid\n"
5275                 if $nbd->{$opt}->{replicated};
5276
5277             $res->{drives}->{$opt} = $nbd->{$opt};
5278             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5279         }
5280     }
5281
5282     if ($migratedfrom) {
5283         eval {
5284             set_migration_caps($vmid);
5285         };
5286         warn $@ if $@;
5287
5288         if ($spice_port) {
5289             print "spice listens on port $spice_port\n";
5290             $res->{spice_port} = $spice_port;
5291             if ($migrate_opts->{spice_ticket}) {
5292                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5293                     $migrate_opts->{spice_ticket});
5294                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5295             }
5296         }
5297
5298     } else {
5299         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5300             if !$statefile && $conf->{balloon};
5301
5302         foreach my $opt (keys %$conf) {
5303             next if $opt !~  m/^net\d+$/;
5304             my $nicconf = parse_net($conf->{$opt});
5305             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5306         }
5307     }
5308
5309     mon_cmd($vmid, 'qom-set',
5310                 path => "machine/peripheral/balloon0",
5311                 property => "guest-stats-polling-interval",
5312                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5313
5314     if ($resume) {
5315         print "Resumed VM, removing state\n";
5316         if (my $vmstate = $conf->{vmstate}) {
5317             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5318             PVE::Storage::vdisk_free($storecfg, $vmstate);
5319         }
5320         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5321         PVE::QemuConfig->write_config($vmid, $conf);
5322     }
5323
5324     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5325
5326     return $res;
5327 }
5328
5329 sub vm_commandline {
5330     my ($storecfg, $vmid, $snapname) = @_;
5331
5332     my $conf = PVE::QemuConfig->load_config($vmid);
5333     my $forcemachine;
5334     my $forcecpu;
5335
5336     if ($snapname) {
5337         my $snapshot = $conf->{snapshots}->{$snapname};
5338         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5339
5340         # check for machine or CPU overrides in snapshot
5341         $forcemachine = $snapshot->{runningmachine};
5342         $forcecpu = $snapshot->{runningcpu};
5343
5344         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5345
5346         $conf = $snapshot;
5347     }
5348
5349     my $defaults = load_defaults();
5350
5351     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5352         $forcemachine, $forcecpu);
5353
5354     return PVE::Tools::cmd2string($cmd);
5355 }
5356
5357 sub vm_reset {
5358     my ($vmid, $skiplock) = @_;
5359
5360     PVE::QemuConfig->lock_config($vmid, sub {
5361
5362         my $conf = PVE::QemuConfig->load_config($vmid);
5363
5364         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5365
5366         mon_cmd($vmid, "system_reset");
5367     });
5368 }
5369
5370 sub get_vm_volumes {
5371     my ($conf) = @_;
5372
5373     my $vollist = [];
5374     foreach_volid($conf, sub {
5375         my ($volid, $attr) = @_;
5376
5377         return if $volid =~ m|^/|;
5378
5379         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5380         return if !$sid;
5381
5382         push @$vollist, $volid;
5383     });
5384
5385     return $vollist;
5386 }
5387
5388 sub vm_stop_cleanup {
5389     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5390
5391     eval {
5392
5393         if (!$keepActive) {
5394             my $vollist = get_vm_volumes($conf);
5395             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5396         }
5397
5398         foreach my $ext (qw(mon qmp pid vnc qga)) {
5399             unlink "/var/run/qemu-server/${vmid}.$ext";
5400         }
5401
5402         if ($conf->{ivshmem}) {
5403             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5404             # just delete it for now, VMs which have this already open do not
5405             # are affected, but new VMs will get a separated one. If this
5406             # becomes an issue we either add some sort of ref-counting or just
5407             # add a "don't delete on stop" flag to the ivshmem format.
5408             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5409         }
5410
5411         foreach my $key (keys %$conf) {
5412             next if $key !~ m/^hostpci(\d+)$/;
5413             my $hostpciindex = $1;
5414             my $d = parse_hostpci($conf->{$key});
5415             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5416
5417             foreach my $pci (@{$d->{pciid}}) {
5418                 my $pciid = $pci->{id};
5419                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5420             }
5421         }
5422
5423         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5424     };
5425     warn $@ if $@; # avoid errors - just warn
5426 }
5427
5428 # call only in locked context
5429 sub _do_vm_stop {
5430     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5431
5432     my $pid = check_running($vmid, $nocheck);
5433     return if !$pid;
5434
5435     my $conf;
5436     if (!$nocheck) {
5437         $conf = PVE::QemuConfig->load_config($vmid);
5438         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5439         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5440             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5441             $timeout = $opts->{down} if $opts->{down};
5442         }
5443         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5444     }
5445
5446     eval {
5447         if ($shutdown) {
5448             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5449                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5450             } else {
5451                 mon_cmd($vmid, "system_powerdown");
5452             }
5453         } else {
5454             mon_cmd($vmid, "quit");
5455         }
5456     };
5457     my $err = $@;
5458
5459     if (!$err) {
5460         $timeout = 60 if !defined($timeout);
5461
5462         my $count = 0;
5463         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5464             $count++;
5465             sleep 1;
5466         }
5467
5468         if ($count >= $timeout) {
5469             if ($force) {
5470                 warn "VM still running - terminating now with SIGTERM\n";
5471                 kill 15, $pid;
5472             } else {
5473                 die "VM quit/powerdown failed - got timeout\n";
5474             }
5475         } else {
5476             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5477             return;
5478         }
5479     } else {
5480        if (!check_running($vmid, $nocheck)) {
5481             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5482             return;
5483        }
5484        if ($force) {
5485             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5486             kill 15, $pid;
5487         } else {
5488             die "VM quit/powerdown failed\n";
5489         }
5490     }
5491
5492     # wait again
5493     $timeout = 10;
5494
5495     my $count = 0;
5496     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5497         $count++;
5498         sleep 1;
5499     }
5500
5501     if ($count >= $timeout) {
5502         warn "VM still running - terminating now with SIGKILL\n";
5503         kill 9, $pid;
5504         sleep 1;
5505     }
5506
5507     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5508 }
5509
5510 # Note: use $nocheck to skip tests if VM configuration file exists.
5511 # We need that when migration VMs to other nodes (files already moved)
5512 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5513 sub vm_stop {
5514     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5515
5516     $force = 1 if !defined($force) && !$shutdown;
5517
5518     if ($migratedfrom){
5519         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5520         kill 15, $pid if $pid;
5521         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5522         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5523         return;
5524     }
5525
5526     PVE::QemuConfig->lock_config($vmid, sub {
5527         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5528    });
5529 }
5530
5531 sub vm_reboot {
5532     my ($vmid, $timeout) = @_;
5533
5534     PVE::QemuConfig->lock_config($vmid, sub {
5535         eval {
5536
5537             # only reboot if running, as qmeventd starts it again on a stop event
5538             return if !check_running($vmid);
5539
5540             create_reboot_request($vmid);
5541
5542             my $storecfg = PVE::Storage::config();
5543             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5544
5545         };
5546         if (my $err = $@) {
5547             # avoid that the next normal shutdown will be confused for a reboot
5548             clear_reboot_request($vmid);
5549             die $err;
5550         }
5551    });
5552 }
5553
5554 # note: if using the statestorage parameter, the caller has to check privileges
5555 sub vm_suspend {
5556     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5557
5558     my $conf;
5559     my $path;
5560     my $storecfg;
5561     my $vmstate;
5562
5563     PVE::QemuConfig->lock_config($vmid, sub {
5564
5565         $conf = PVE::QemuConfig->load_config($vmid);
5566
5567         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5568         PVE::QemuConfig->check_lock($conf)
5569             if !($skiplock || $is_backing_up);
5570
5571         die "cannot suspend to disk during backup\n"
5572             if $is_backing_up && $includestate;
5573
5574         if ($includestate) {
5575             $conf->{lock} = 'suspending';
5576             my $date = strftime("%Y-%m-%d", localtime(time()));
5577             $storecfg = PVE::Storage::config();
5578             if (!$statestorage) {
5579                 $statestorage = find_vmstate_storage($conf, $storecfg);
5580                 # check permissions for the storage
5581                 my $rpcenv = PVE::RPCEnvironment::get();
5582                 if ($rpcenv->{type} ne 'cli') {
5583                     my $authuser = $rpcenv->get_user();
5584                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5585                 }
5586             }
5587
5588
5589             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5590                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5591             $path = PVE::Storage::path($storecfg, $vmstate);
5592             PVE::QemuConfig->write_config($vmid, $conf);
5593         } else {
5594             mon_cmd($vmid, "stop");
5595         }
5596     });
5597
5598     if ($includestate) {
5599         # save vm state
5600         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5601
5602         eval {
5603             mon_cmd($vmid, "savevm-start", statefile => $path);
5604             for(;;) {
5605                 my $state = mon_cmd($vmid, "query-savevm");
5606                 if (!$state->{status}) {
5607                     die "savevm not active\n";
5608                 } elsif ($state->{status} eq 'active') {
5609                     sleep(1);
5610                     next;
5611                 } elsif ($state->{status} eq 'completed') {
5612                     print "State saved, quitting\n";
5613                     last;
5614                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5615                     die "query-savevm failed with error '$state->{error}'\n"
5616                 } else {
5617                     die "query-savevm returned status '$state->{status}'\n";
5618                 }
5619             }
5620         };
5621         my $err = $@;
5622
5623         PVE::QemuConfig->lock_config($vmid, sub {
5624             $conf = PVE::QemuConfig->load_config($vmid);
5625             if ($err) {
5626                 # cleanup, but leave suspending lock, to indicate something went wrong
5627                 eval {
5628                     mon_cmd($vmid, "savevm-end");
5629                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5630                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5631                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5632                     PVE::QemuConfig->write_config($vmid, $conf);
5633                 };
5634                 warn $@ if $@;
5635                 die $err;
5636             }
5637
5638             die "lock changed unexpectedly\n"
5639                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5640
5641             mon_cmd($vmid, "quit");
5642             $conf->{lock} = 'suspended';
5643             PVE::QemuConfig->write_config($vmid, $conf);
5644         });
5645     }
5646 }
5647
5648 sub vm_resume {
5649     my ($vmid, $skiplock, $nocheck) = @_;
5650
5651     PVE::QemuConfig->lock_config($vmid, sub {
5652         my $res = mon_cmd($vmid, 'query-status');
5653         my $resume_cmd = 'cont';
5654         my $reset = 0;
5655
5656         if ($res->{status}) {
5657             return if $res->{status} eq 'running'; # job done, go home
5658             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5659             $reset = 1 if $res->{status} eq 'shutdown';
5660         }
5661
5662         if (!$nocheck) {
5663
5664             my $conf = PVE::QemuConfig->load_config($vmid);
5665
5666             PVE::QemuConfig->check_lock($conf)
5667                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5668         }
5669
5670         if ($reset) {
5671             # required if a VM shuts down during a backup and we get a resume
5672             # request before the backup finishes for example
5673             mon_cmd($vmid, "system_reset");
5674         }
5675         mon_cmd($vmid, $resume_cmd);
5676     });
5677 }
5678
5679 sub vm_sendkey {
5680     my ($vmid, $skiplock, $key) = @_;
5681
5682     PVE::QemuConfig->lock_config($vmid, sub {
5683
5684         my $conf = PVE::QemuConfig->load_config($vmid);
5685
5686         # there is no qmp command, so we use the human monitor command
5687         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5688         die $res if $res ne '';
5689     });
5690 }
5691
5692 # vzdump restore implementaion
5693
5694 sub tar_archive_read_firstfile {
5695     my $archive = shift;
5696
5697     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5698
5699     # try to detect archive type first
5700     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5701         die "unable to open file '$archive'\n";
5702     my $firstfile = <$fh>;
5703     kill 15, $pid;
5704     close $fh;
5705
5706     die "ERROR: archive contaions no data\n" if !$firstfile;
5707     chomp $firstfile;
5708
5709     return $firstfile;
5710 }
5711
5712 sub tar_restore_cleanup {
5713     my ($storecfg, $statfile) = @_;
5714
5715     print STDERR "starting cleanup\n";
5716
5717     if (my $fd = IO::File->new($statfile, "r")) {
5718         while (defined(my $line = <$fd>)) {
5719             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5720                 my $volid = $2;
5721                 eval {
5722                     if ($volid =~ m|^/|) {
5723                         unlink $volid || die 'unlink failed\n';
5724                     } else {
5725                         PVE::Storage::vdisk_free($storecfg, $volid);
5726                     }
5727                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5728                 };
5729                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5730             } else {
5731                 print STDERR "unable to parse line in statfile - $line";
5732             }
5733         }
5734         $fd->close();
5735     }
5736 }
5737
5738 sub restore_file_archive {
5739     my ($archive, $vmid, $user, $opts) = @_;
5740
5741     return restore_vma_archive($archive, $vmid, $user, $opts)
5742         if $archive eq '-';
5743
5744     my $info = PVE::Storage::archive_info($archive);
5745     my $format = $opts->{format} // $info->{format};
5746     my $comp = $info->{compression};
5747
5748     # try to detect archive format
5749     if ($format eq 'tar') {
5750         return restore_tar_archive($archive, $vmid, $user, $opts);
5751     } else {
5752         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5753     }
5754 }
5755
5756 # hepler to remove disks that will not be used after restore
5757 my $restore_cleanup_oldconf = sub {
5758     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5759
5760     PVE::QemuConfig->foreach_volume($oldconf, sub {
5761         my ($ds, $drive) = @_;
5762
5763         return if drive_is_cdrom($drive, 1);
5764
5765         my $volid = $drive->{file};
5766         return if !$volid || $volid =~ m|^/|;
5767
5768         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5769         return if !$path || !$owner || ($owner != $vmid);
5770
5771         # Note: only delete disk we want to restore
5772         # other volumes will become unused
5773         if ($virtdev_hash->{$ds}) {
5774             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5775             if (my $err = $@) {
5776                 warn $err;
5777             }
5778         }
5779     });
5780
5781     # delete vmstate files, after the restore we have no snapshots anymore
5782     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5783         my $snap = $oldconf->{snapshots}->{$snapname};
5784         if ($snap->{vmstate}) {
5785             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5786             if (my $err = $@) {
5787                 warn $err;
5788             }
5789         }
5790     }
5791 };
5792
5793 # Helper to parse vzdump backup device hints
5794 #
5795 # $rpcenv: Environment, used to ckeck storage permissions
5796 # $user: User ID, to check storage permissions
5797 # $storecfg: Storage configuration
5798 # $fh: the file handle for reading the configuration
5799 # $devinfo: should contain device sizes for all backu-up'ed devices
5800 # $options: backup options (pool, default storage)
5801 #
5802 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5803 my $parse_backup_hints = sub {
5804     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5805
5806     my $virtdev_hash = {};
5807
5808     while (defined(my $line = <$fh>)) {
5809         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5810             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5811             die "archive does not contain data for drive '$virtdev'\n"
5812                 if !$devinfo->{$devname};
5813
5814             if (defined($options->{storage})) {
5815                 $storeid = $options->{storage} || 'local';
5816             } elsif (!$storeid) {
5817                 $storeid = 'local';
5818             }
5819             $format = 'raw' if !$format;
5820             $devinfo->{$devname}->{devname} = $devname;
5821             $devinfo->{$devname}->{virtdev} = $virtdev;
5822             $devinfo->{$devname}->{format} = $format;
5823             $devinfo->{$devname}->{storeid} = $storeid;
5824
5825             # check permission on storage
5826             my $pool = $options->{pool}; # todo: do we need that?
5827             if ($user ne 'root@pam') {
5828                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5829             }
5830
5831             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5832         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5833             my $virtdev = $1;
5834             my $drive = parse_drive($virtdev, $2);
5835             if (drive_is_cloudinit($drive)) {
5836                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5837                 $storeid = $options->{storage} if defined ($options->{storage});
5838                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5839                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5840
5841                 $virtdev_hash->{$virtdev} = {
5842                     format => $format,
5843                     storeid => $storeid,
5844                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5845                     is_cloudinit => 1,
5846                 };
5847             }
5848         }
5849     }
5850
5851     return $virtdev_hash;
5852 };
5853
5854 # Helper to allocate and activate all volumes required for a restore
5855 #
5856 # $storecfg: Storage configuration
5857 # $virtdev_hash: as returned by parse_backup_hints()
5858 #
5859 # Returns: { $virtdev => $volid }
5860 my $restore_allocate_devices = sub {
5861     my ($storecfg, $virtdev_hash, $vmid) = @_;
5862
5863     my $map = {};
5864     foreach my $virtdev (sort keys %$virtdev_hash) {
5865         my $d = $virtdev_hash->{$virtdev};
5866         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5867         my $storeid = $d->{storeid};
5868         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5869
5870         # test if requested format is supported
5871         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5872         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5873         $d->{format} = $defFormat if !$supported;
5874
5875         my $name;
5876         if ($d->{is_cloudinit}) {
5877             $name = "vm-$vmid-cloudinit";
5878             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5879             if ($scfg->{path}) {
5880                 $name .= ".$d->{format}";
5881             }
5882         }
5883
5884         my $volid = PVE::Storage::vdisk_alloc(
5885             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5886
5887         print STDERR "new volume ID is '$volid'\n";
5888         $d->{volid} = $volid;
5889
5890         PVE::Storage::activate_volumes($storecfg, [$volid]);
5891
5892         $map->{$virtdev} = $volid;
5893     }
5894
5895     return $map;
5896 };
5897
5898 my $restore_update_config_line = sub {
5899     my ($cookie, $vmid, $map, $line, $unique) = @_;
5900
5901     return '' if $line =~ m/^\#qmdump\#/;
5902     return '' if $line =~ m/^\#vzdump\#/;
5903     return '' if $line =~ m/^lock:/;
5904     return '' if $line =~ m/^unused\d+:/;
5905     return '' if $line =~ m/^parent:/;
5906
5907     my $res = '';
5908
5909     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5910     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5911         # try to convert old 1.X settings
5912         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5913         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5914             my ($model, $macaddr) = split(/\=/, $devconfig);
5915             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5916             my $net = {
5917                 model => $model,
5918                 bridge => "vmbr$ind",
5919                 macaddr => $macaddr,
5920             };
5921             my $netstr = print_net($net);
5922
5923             $res .= "net$cookie->{netcount}: $netstr\n";
5924             $cookie->{netcount}++;
5925         }
5926     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5927         my ($id, $netstr) = ($1, $2);
5928         my $net = parse_net($netstr);
5929         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5930         $netstr = print_net($net);
5931         $res .= "$id: $netstr\n";
5932     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5933         my $virtdev = $1;
5934         my $value = $3;
5935         my $di = parse_drive($virtdev, $value);
5936         if (defined($di->{backup}) && !$di->{backup}) {
5937             $res .= "#$line";
5938         } elsif ($map->{$virtdev}) {
5939             delete $di->{format}; # format can change on restore
5940             $di->{file} = $map->{$virtdev};
5941             $value = print_drive($di);
5942             $res .= "$virtdev: $value\n";
5943         } else {
5944             $res .= $line;
5945         }
5946     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5947         my $vmgenid = $1;
5948         if ($vmgenid ne '0') {
5949             # always generate a new vmgenid if there was a valid one setup
5950             $vmgenid = generate_uuid();
5951         }
5952         $res .= "vmgenid: $vmgenid\n";
5953     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5954         my ($uuid, $uuid_str);
5955         UUID::generate($uuid);
5956         UUID::unparse($uuid, $uuid_str);
5957         my $smbios1 = parse_smbios1($2);
5958         $smbios1->{uuid} = $uuid_str;
5959         $res .= $1.print_smbios1($smbios1)."\n";
5960     } else {
5961         $res .= $line;
5962     }
5963
5964     return $res;
5965 };
5966
5967 my $restore_deactivate_volumes = sub {
5968     my ($storecfg, $devinfo) = @_;
5969
5970     my $vollist = [];
5971     foreach my $devname (keys %$devinfo) {
5972         my $volid = $devinfo->{$devname}->{volid};
5973         push @$vollist, $volid if $volid;
5974     }
5975
5976     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5977 };
5978
5979 my $restore_destroy_volumes = sub {
5980     my ($storecfg, $devinfo) = @_;
5981
5982     foreach my $devname (keys %$devinfo) {
5983         my $volid = $devinfo->{$devname}->{volid};
5984         next if !$volid;
5985         eval {
5986             if ($volid =~ m|^/|) {
5987                 unlink $volid || die 'unlink failed\n';
5988             } else {
5989                 PVE::Storage::vdisk_free($storecfg, $volid);
5990             }
5991             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5992         };
5993         print STDERR "unable to cleanup '$volid' - $@" if $@;
5994     }
5995 };
5996
5997 sub scan_volids {
5998     my ($cfg, $vmid) = @_;
5999
6000     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6001
6002     my $volid_hash = {};
6003     foreach my $storeid (keys %$info) {
6004         foreach my $item (@{$info->{$storeid}}) {
6005             next if !($item->{volid} && $item->{size});
6006             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6007             $volid_hash->{$item->{volid}} = $item;
6008         }
6009     }
6010
6011     return $volid_hash;
6012 }
6013
6014 sub update_disk_config {
6015     my ($vmid, $conf, $volid_hash) = @_;
6016
6017     my $changes;
6018     my $prefix = "VM $vmid";
6019
6020     # used and unused disks
6021     my $referenced = {};
6022
6023     # Note: it is allowed to define multiple storages with same path (alias), so
6024     # we need to check both 'volid' and real 'path' (two different volid can point
6025     # to the same path).
6026
6027     my $referencedpath = {};
6028
6029     # update size info
6030     PVE::QemuConfig->foreach_volume($conf, sub {
6031         my ($opt, $drive) = @_;
6032
6033         my $volid = $drive->{file};
6034         return if !$volid;
6035         my $volume = $volid_hash->{$volid};
6036
6037         # mark volid as "in-use" for next step
6038         $referenced->{$volid} = 1;
6039         if ($volume && (my $path = $volume->{path})) {
6040             $referencedpath->{$path} = 1;
6041         }
6042
6043         return if drive_is_cdrom($drive);
6044         return if !$volume;
6045
6046         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6047         if (defined($updated)) {
6048             $changes = 1;
6049             $conf->{$opt} = print_drive($updated);
6050             print "$prefix ($opt): $msg\n";
6051         }
6052     });
6053
6054     # remove 'unusedX' entry if volume is used
6055     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6056         my ($opt, $drive) = @_;
6057
6058         my $volid = $drive->{file};
6059         return if !$volid;
6060
6061         my $path;
6062         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6063         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6064             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6065             $changes = 1;
6066             delete $conf->{$opt};
6067         }
6068
6069         $referenced->{$volid} = 1;
6070         $referencedpath->{$path} = 1 if $path;
6071     });
6072
6073     foreach my $volid (sort keys %$volid_hash) {
6074         next if $volid =~ m/vm-$vmid-state-/;
6075         next if $referenced->{$volid};
6076         my $path = $volid_hash->{$volid}->{path};
6077         next if !$path; # just to be sure
6078         next if $referencedpath->{$path};
6079         $changes = 1;
6080         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6081         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6082         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6083     }
6084
6085     return $changes;
6086 }
6087
6088 sub rescan {
6089     my ($vmid, $nolock, $dryrun) = @_;
6090
6091     my $cfg = PVE::Storage::config();
6092
6093     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6094     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6095     foreach my $stor (keys %{$cfg->{ids}}) {
6096         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6097     }
6098
6099     print "rescan volumes...\n";
6100     my $volid_hash = scan_volids($cfg, $vmid);
6101
6102     my $updatefn =  sub {
6103         my ($vmid) = @_;
6104
6105         my $conf = PVE::QemuConfig->load_config($vmid);
6106
6107         PVE::QemuConfig->check_lock($conf);
6108
6109         my $vm_volids = {};
6110         foreach my $volid (keys %$volid_hash) {
6111             my $info = $volid_hash->{$volid};
6112             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6113         }
6114
6115         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6116
6117         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6118     };
6119
6120     if (defined($vmid)) {
6121         if ($nolock) {
6122             &$updatefn($vmid);
6123         } else {
6124             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6125         }
6126     } else {
6127         my $vmlist = config_list();
6128         foreach my $vmid (keys %$vmlist) {
6129             if ($nolock) {
6130                 &$updatefn($vmid);
6131             } else {
6132                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6133             }
6134         }
6135     }
6136 }
6137
6138 sub restore_proxmox_backup_archive {
6139     my ($archive, $vmid, $user, $options) = @_;
6140
6141     my $storecfg = PVE::Storage::config();
6142
6143     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6144     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6145
6146     my $fingerprint = $scfg->{fingerprint};
6147     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6148
6149     my $repo = PVE::PBSClient::get_repository($scfg);
6150
6151     # This is only used for `pbs-restore`!
6152     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6153     local $ENV{PBS_PASSWORD} = $password;
6154     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6155
6156     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6157         PVE::Storage::parse_volname($storecfg, $archive);
6158
6159     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6160
6161     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6162
6163     my $tmpdir = "/var/tmp/vzdumptmp$$";
6164     rmtree $tmpdir;
6165     mkpath $tmpdir;
6166
6167     my $conffile = PVE::QemuConfig->config_file($vmid);
6168      # disable interrupts (always do cleanups)
6169     local $SIG{INT} =
6170         local $SIG{TERM} =
6171         local $SIG{QUIT} =
6172         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6173
6174     # Note: $oldconf is undef if VM does not exists
6175     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6176     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6177     my $new_conf_raw = '';
6178
6179     my $rpcenv = PVE::RPCEnvironment::get();
6180     my $devinfo = {};
6181
6182     eval {
6183         # enable interrupts
6184         local $SIG{INT} =
6185             local $SIG{TERM} =
6186             local $SIG{QUIT} =
6187             local $SIG{HUP} =
6188             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6189
6190         my $cfgfn = "$tmpdir/qemu-server.conf";
6191         my $firewall_config_fn = "$tmpdir/fw.conf";
6192         my $index_fn = "$tmpdir/index.json";
6193
6194         my $cmd = "restore";
6195
6196         my $param = [$pbs_backup_name, "index.json", $index_fn];
6197         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6198         my $index = PVE::Tools::file_get_contents($index_fn);
6199         $index = decode_json($index);
6200
6201         # print Dumper($index);
6202         foreach my $info (@{$index->{files}}) {
6203             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6204                 my $devname = $1;
6205                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6206                     $devinfo->{$devname}->{size} = $1;
6207                 } else {
6208                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6209                 }
6210             }
6211         }
6212
6213         my $is_qemu_server_backup = scalar(
6214             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6215         );
6216         if (!$is_qemu_server_backup) {
6217             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6218         }
6219         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6220
6221         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6222         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6223
6224         if ($has_firewall_config) {
6225             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6226             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6227
6228             my $pve_firewall_dir = '/etc/pve/firewall';
6229             mkdir $pve_firewall_dir; # make sure the dir exists
6230             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6231         }
6232
6233         my $fh = IO::File->new($cfgfn, "r") ||
6234             die "unable to read qemu-server.conf - $!\n";
6235
6236         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6237
6238         # fixme: rate limit?
6239
6240         # create empty/temp config
6241         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6242
6243         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6244
6245         # allocate volumes
6246         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6247
6248         foreach my $virtdev (sort keys %$virtdev_hash) {
6249             my $d = $virtdev_hash->{$virtdev};
6250             next if $d->{is_cloudinit}; # no need to restore cloudinit
6251
6252             my $volid = $d->{volid};
6253
6254             my $path = PVE::Storage::path($storecfg, $volid);
6255
6256             # This is the ONLY user of the PBS_ env vars set on top of this function!
6257             my $pbs_restore_cmd = [
6258                 '/usr/bin/pbs-restore',
6259                 '--repository', $repo,
6260                 $pbs_backup_name,
6261                 "$d->{devname}.img.fidx",
6262                 $path,
6263                 '--verbose',
6264                 ];
6265
6266             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6267             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6268
6269             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6270                 push @$pbs_restore_cmd, '--skip-zero';
6271             }
6272
6273             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6274             print "restore proxmox backup image: $dbg_cmdstring\n";
6275             run_command($pbs_restore_cmd);
6276         }
6277
6278         $fh->seek(0, 0) || die "seek failed - $!\n";
6279
6280         my $cookie = { netcount => 0 };
6281         while (defined(my $line = <$fh>)) {
6282             $new_conf_raw .= $restore_update_config_line->(
6283                 $cookie,
6284                 $vmid,
6285                 $map,
6286                 $line,
6287                 $options->{unique},
6288             );
6289         }
6290
6291         $fh->close();
6292     };
6293     my $err = $@;
6294
6295     $restore_deactivate_volumes->($storecfg, $devinfo);
6296
6297     rmtree $tmpdir;
6298
6299     if ($err) {
6300         $restore_destroy_volumes->($storecfg, $devinfo);
6301         die $err;
6302     }
6303
6304     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6305
6306     PVE::Cluster::cfs_update(); # make sure we read new file
6307
6308     eval { rescan($vmid, 1); };
6309     warn $@ if $@;
6310 }
6311
6312 sub restore_vma_archive {
6313     my ($archive, $vmid, $user, $opts, $comp) = @_;
6314
6315     my $readfrom = $archive;
6316
6317     my $cfg = PVE::Storage::config();
6318     my $commands = [];
6319     my $bwlimit = $opts->{bwlimit};
6320
6321     my $dbg_cmdstring = '';
6322     my $add_pipe = sub {
6323         my ($cmd) = @_;
6324         push @$commands, $cmd;
6325         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6326         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6327         $readfrom = '-';
6328     };
6329
6330     my $input = undef;
6331     if ($archive eq '-') {
6332         $input = '<&STDIN';
6333     } else {
6334         # If we use a backup from a PVE defined storage we also consider that
6335         # storage's rate limit:
6336         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6337         if (defined($volid)) {
6338             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6339             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6340             if ($readlimit) {
6341                 print STDERR "applying read rate limit: $readlimit\n";
6342                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6343                 $add_pipe->($cstream);
6344             }
6345         }
6346     }
6347
6348     if ($comp) {
6349         my $info = PVE::Storage::decompressor_info('vma', $comp);
6350         my $cmd = $info->{decompressor};
6351         push @$cmd, $readfrom;
6352         $add_pipe->($cmd);
6353     }
6354
6355     my $tmpdir = "/var/tmp/vzdumptmp$$";
6356     rmtree $tmpdir;
6357
6358     # disable interrupts (always do cleanups)
6359     local $SIG{INT} =
6360         local $SIG{TERM} =
6361         local $SIG{QUIT} =
6362         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6363
6364     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6365     POSIX::mkfifo($mapfifo, 0600);
6366     my $fifofh;
6367     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6368
6369     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6370
6371     my $oldtimeout;
6372     my $timeout = 5;
6373
6374     my $devinfo = {};
6375
6376     my $rpcenv = PVE::RPCEnvironment::get();
6377
6378     my $conffile = PVE::QemuConfig->config_file($vmid);
6379
6380     # Note: $oldconf is undef if VM does not exist
6381     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6382     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6383     my $new_conf_raw = '';
6384
6385     my %storage_limits;
6386
6387     my $print_devmap = sub {
6388         my $cfgfn = "$tmpdir/qemu-server.conf";
6389
6390         # we can read the config - that is already extracted
6391         my $fh = IO::File->new($cfgfn, "r") ||
6392             die "unable to read qemu-server.conf - $!\n";
6393
6394         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6395         if (-f $fwcfgfn) {
6396             my $pve_firewall_dir = '/etc/pve/firewall';
6397             mkdir $pve_firewall_dir; # make sure the dir exists
6398             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6399         }
6400
6401         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6402
6403         foreach my $info (values %{$virtdev_hash}) {
6404             my $storeid = $info->{storeid};
6405             next if defined($storage_limits{$storeid});
6406
6407             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$storeid], $bwlimit) // 0;
6408             print STDERR "rate limit for storage $storeid: $limit KiB/s\n" if $limit;
6409             $storage_limits{$storeid} = $limit * 1024;
6410         }
6411
6412         foreach my $devname (keys %$devinfo) {
6413             die "found no device mapping information for device '$devname'\n"
6414                 if !$devinfo->{$devname}->{virtdev};
6415         }
6416
6417         # create empty/temp config
6418         if ($oldconf) {
6419             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6420             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6421         }
6422
6423         # allocate volumes
6424         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6425
6426         # print restore information to $fifofh
6427         foreach my $virtdev (sort keys %$virtdev_hash) {
6428             my $d = $virtdev_hash->{$virtdev};
6429             next if $d->{is_cloudinit}; # no need to restore cloudinit
6430
6431             my $storeid = $d->{storeid};
6432             my $volid = $d->{volid};
6433
6434             my $map_opts = '';
6435             if (my $limit = $storage_limits{$storeid}) {
6436                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6437             }
6438
6439             my $write_zeros = 1;
6440             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6441                 $write_zeros = 0;
6442             }
6443
6444             my $path = PVE::Storage::path($cfg, $volid);
6445
6446             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6447
6448             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6449         }
6450
6451         $fh->seek(0, 0) || die "seek failed - $!\n";
6452
6453         my $cookie = { netcount => 0 };
6454         while (defined(my $line = <$fh>)) {
6455             $new_conf_raw .= $restore_update_config_line->(
6456                 $cookie,
6457                 $vmid,
6458                 $map,
6459                 $line,
6460                 $opts->{unique},
6461             );
6462         }
6463
6464         $fh->close();
6465     };
6466
6467     eval {
6468         # enable interrupts
6469         local $SIG{INT} =
6470             local $SIG{TERM} =
6471             local $SIG{QUIT} =
6472             local $SIG{HUP} =
6473             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6474         local $SIG{ALRM} = sub { die "got timeout\n"; };
6475
6476         $oldtimeout = alarm($timeout);
6477
6478         my $parser = sub {
6479             my $line = shift;
6480
6481             print "$line\n";
6482
6483             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6484                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6485                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6486             } elsif ($line =~ m/^CTIME: /) {
6487                 # we correctly received the vma config, so we can disable
6488                 # the timeout now for disk allocation (set to 10 minutes, so
6489                 # that we always timeout if something goes wrong)
6490                 alarm(600);
6491                 &$print_devmap();
6492                 print $fifofh "done\n";
6493                 my $tmp = $oldtimeout || 0;
6494                 $oldtimeout = undef;
6495                 alarm($tmp);
6496                 close($fifofh);
6497                 $fifofh = undef;
6498             }
6499         };
6500
6501         print "restore vma archive: $dbg_cmdstring\n";
6502         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6503     };
6504     my $err = $@;
6505
6506     alarm($oldtimeout) if $oldtimeout;
6507
6508     $restore_deactivate_volumes->($cfg, $devinfo);
6509
6510     close($fifofh) if $fifofh;
6511     unlink $mapfifo;
6512     rmtree $tmpdir;
6513
6514     if ($err) {
6515         $restore_destroy_volumes->($cfg, $devinfo);
6516         die $err;
6517     }
6518
6519     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6520
6521     PVE::Cluster::cfs_update(); # make sure we read new file
6522
6523     eval { rescan($vmid, 1); };
6524     warn $@ if $@;
6525 }
6526
6527 sub restore_tar_archive {
6528     my ($archive, $vmid, $user, $opts) = @_;
6529
6530     if ($archive ne '-') {
6531         my $firstfile = tar_archive_read_firstfile($archive);
6532         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6533             if $firstfile ne 'qemu-server.conf';
6534     }
6535
6536     my $storecfg = PVE::Storage::config();
6537
6538     # avoid zombie disks when restoring over an existing VM -> cleanup first
6539     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6540     # skiplock=1 because qmrestore has set the 'create' lock itself already
6541     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6542     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6543
6544     my $tocmd = "/usr/lib/qemu-server/qmextract";
6545
6546     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6547     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6548     $tocmd .= ' --prealloc' if $opts->{prealloc};
6549     $tocmd .= ' --info' if $opts->{info};
6550
6551     # tar option "xf" does not autodetect compression when read from STDIN,
6552     # so we pipe to zcat
6553     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6554         PVE::Tools::shellquote("--to-command=$tocmd");
6555
6556     my $tmpdir = "/var/tmp/vzdumptmp$$";
6557     mkpath $tmpdir;
6558
6559     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6560     local $ENV{VZDUMP_VMID} = $vmid;
6561     local $ENV{VZDUMP_USER} = $user;
6562
6563     my $conffile = PVE::QemuConfig->config_file($vmid);
6564     my $new_conf_raw = '';
6565
6566     # disable interrupts (always do cleanups)
6567     local $SIG{INT} =
6568         local $SIG{TERM} =
6569         local $SIG{QUIT} =
6570         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6571
6572     eval {
6573         # enable interrupts
6574         local $SIG{INT} =
6575             local $SIG{TERM} =
6576             local $SIG{QUIT} =
6577             local $SIG{HUP} =
6578             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6579
6580         if ($archive eq '-') {
6581             print "extracting archive from STDIN\n";
6582             run_command($cmd, input => "<&STDIN");
6583         } else {
6584             print "extracting archive '$archive'\n";
6585             run_command($cmd);
6586         }
6587
6588         return if $opts->{info};
6589
6590         # read new mapping
6591         my $map = {};
6592         my $statfile = "$tmpdir/qmrestore.stat";
6593         if (my $fd = IO::File->new($statfile, "r")) {
6594             while (defined (my $line = <$fd>)) {
6595                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6596                     $map->{$1} = $2 if $1;
6597                 } else {
6598                     print STDERR "unable to parse line in statfile - $line\n";
6599                 }
6600             }
6601             $fd->close();
6602         }
6603
6604         my $confsrc = "$tmpdir/qemu-server.conf";
6605
6606         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6607
6608         my $cookie = { netcount => 0 };
6609         while (defined (my $line = <$srcfd>)) {
6610             $new_conf_raw .= $restore_update_config_line->(
6611                 $cookie,
6612                 $vmid,
6613                 $map,
6614                 $line,
6615                 $opts->{unique},
6616             );
6617         }
6618
6619         $srcfd->close();
6620     };
6621     if (my $err = $@) {
6622         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6623         die $err;
6624     }
6625
6626     rmtree $tmpdir;
6627
6628     PVE::Tools::file_set_contents($conffile, $new_conf_raw);
6629
6630     PVE::Cluster::cfs_update(); # make sure we read new file
6631
6632     eval { rescan($vmid, 1); };
6633     warn $@ if $@;
6634 };
6635
6636 sub foreach_storage_used_by_vm {
6637     my ($conf, $func) = @_;
6638
6639     my $sidhash = {};
6640
6641     PVE::QemuConfig->foreach_volume($conf, sub {
6642         my ($ds, $drive) = @_;
6643         return if drive_is_cdrom($drive);
6644
6645         my $volid = $drive->{file};
6646
6647         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6648         $sidhash->{$sid} = $sid if $sid;
6649     });
6650
6651     foreach my $sid (sort keys %$sidhash) {
6652         &$func($sid);
6653     }
6654 }
6655
6656 my $qemu_snap_storage = {
6657     rbd => 1,
6658 };
6659 sub do_snapshots_with_qemu {
6660     my ($storecfg, $volid) = @_;
6661
6662     my $storage_name = PVE::Storage::parse_volume_id($volid);
6663     my $scfg = $storecfg->{ids}->{$storage_name};
6664     die "could not find storage '$storage_name'\n" if !defined($scfg);
6665
6666     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6667         return 1;
6668     }
6669
6670     if ($volid =~ m/\.(qcow2|qed)$/){
6671         return 1;
6672     }
6673
6674     return;
6675 }
6676
6677 sub qga_check_running {
6678     my ($vmid, $nowarn) = @_;
6679
6680     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6681     if ($@) {
6682         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6683         return 0;
6684     }
6685     return 1;
6686 }
6687
6688 sub template_create {
6689     my ($vmid, $conf, $disk) = @_;
6690
6691     my $storecfg = PVE::Storage::config();
6692
6693     PVE::QemuConfig->foreach_volume($conf, sub {
6694         my ($ds, $drive) = @_;
6695
6696         return if drive_is_cdrom($drive);
6697         return if $disk && $ds ne $disk;
6698
6699         my $volid = $drive->{file};
6700         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6701
6702         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6703         $drive->{file} = $voliddst;
6704         $conf->{$ds} = print_drive($drive);
6705         PVE::QemuConfig->write_config($vmid, $conf);
6706     });
6707 }
6708
6709 sub convert_iscsi_path {
6710     my ($path) = @_;
6711
6712     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6713         my $portal = $1;
6714         my $target = $2;
6715         my $lun = $3;
6716
6717         my $initiator_name = get_initiator_name();
6718
6719         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6720                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6721     }
6722
6723     die "cannot convert iscsi path '$path', unkown format\n";
6724 }
6725
6726 sub qemu_img_convert {
6727     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6728
6729     my $storecfg = PVE::Storage::config();
6730     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6731     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6732
6733     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6734
6735     my $cachemode;
6736     my $src_path;
6737     my $src_is_iscsi = 0;
6738     my $src_format;
6739
6740     if ($src_storeid) {
6741         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6742         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6743         $src_format = qemu_img_format($src_scfg, $src_volname);
6744         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6745         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6746         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6747     } elsif (-f $src_volid) {
6748         $src_path = $src_volid;
6749         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6750             $src_format = $1;
6751         }
6752     }
6753
6754     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6755
6756     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6757     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6758     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6759     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6760
6761     my $cmd = [];
6762     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6763     push @$cmd, '-l', "snapshot.name=$snapname"
6764         if $snapname && $src_format && $src_format eq "qcow2";
6765     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6766     push @$cmd, '-T', $cachemode if defined($cachemode);
6767
6768     if ($src_is_iscsi) {
6769         push @$cmd, '--image-opts';
6770         $src_path = convert_iscsi_path($src_path);
6771     } elsif ($src_format) {
6772         push @$cmd, '-f', $src_format;
6773     }
6774
6775     if ($dst_is_iscsi) {
6776         push @$cmd, '--target-image-opts';
6777         $dst_path = convert_iscsi_path($dst_path);
6778     } else {
6779         push @$cmd, '-O', $dst_format;
6780     }
6781
6782     push @$cmd, $src_path;
6783
6784     if (!$dst_is_iscsi && $is_zero_initialized) {
6785         push @$cmd, "zeroinit:$dst_path";
6786     } else {
6787         push @$cmd, $dst_path;
6788     }
6789
6790     my $parser = sub {
6791         my $line = shift;
6792         if($line =~ m/\((\S+)\/100\%\)/){
6793             my $percent = $1;
6794             my $transferred = int($size * $percent / 100);
6795             my $remaining = $size - $transferred;
6796
6797             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6798         }
6799
6800     };
6801
6802     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6803     my $err = $@;
6804     die "copy failed: $err" if $err;
6805 }
6806
6807 sub qemu_img_format {
6808     my ($scfg, $volname) = @_;
6809
6810     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6811         return $1;
6812     } else {
6813         return "raw";
6814     }
6815 }
6816
6817 sub qemu_drive_mirror {
6818     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6819
6820     $jobs = {} if !$jobs;
6821
6822     my $qemu_target;
6823     my $format;
6824     $jobs->{"drive-$drive"} = {};
6825
6826     if ($dst_volid =~ /^nbd:/) {
6827         $qemu_target = $dst_volid;
6828         $format = "nbd";
6829     } else {
6830         my $storecfg = PVE::Storage::config();
6831         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6832
6833         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6834
6835         $format = qemu_img_format($dst_scfg, $dst_volname);
6836
6837         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6838
6839         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6840     }
6841
6842     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6843     $opts->{format} = $format if $format;
6844
6845     if (defined($src_bitmap)) {
6846         $opts->{sync} = 'incremental';
6847         $opts->{bitmap} = $src_bitmap;
6848         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6849     }
6850
6851     if (defined($bwlimit)) {
6852         $opts->{speed} = $bwlimit * 1024;
6853         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6854     } else {
6855         print "drive mirror is starting for drive-$drive\n";
6856     }
6857
6858     # if a job already runs for this device we get an error, catch it for cleanup
6859     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6860     if (my $err = $@) {
6861         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6862         warn "$@\n" if $@;
6863         die "mirroring error: $err\n";
6864     }
6865
6866     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6867 }
6868
6869 # $completion can be either
6870 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6871 # 'cancel': wait until all jobs are ready, block-job-cancel them
6872 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6873 sub qemu_drive_mirror_monitor {
6874     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6875
6876     $completion //= 'complete';
6877
6878     eval {
6879         my $err_complete = 0;
6880
6881         while (1) {
6882             die "storage migration timed out\n" if $err_complete > 300;
6883
6884             my $stats = mon_cmd($vmid, "query-block-jobs");
6885
6886             my $running_mirror_jobs = {};
6887             foreach my $stat (@$stats) {
6888                 next if $stat->{type} ne 'mirror';
6889                 $running_mirror_jobs->{$stat->{device}} = $stat;
6890             }
6891
6892             my $readycounter = 0;
6893
6894             foreach my $job (keys %$jobs) {
6895
6896                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6897                     print "$job : finished\n";
6898                     delete $jobs->{$job};
6899                     next;
6900                 }
6901
6902                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6903
6904                 my $busy = $running_mirror_jobs->{$job}->{busy};
6905                 my $ready = $running_mirror_jobs->{$job}->{ready};
6906                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6907                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6908                     my $remaining = $total - $transferred;
6909                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6910
6911                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6912                 }
6913
6914                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6915             }
6916
6917             last if scalar(keys %$jobs) == 0;
6918
6919             if ($readycounter == scalar(keys %$jobs)) {
6920                 print "all mirroring jobs are ready \n";
6921                 last if $completion eq 'skip'; #do the complete later
6922
6923                 if ($vmiddst && $vmiddst != $vmid) {
6924                     my $agent_running = $qga && qga_check_running($vmid);
6925                     if ($agent_running) {
6926                         print "freeze filesystem\n";
6927                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6928                     } else {
6929                         print "suspend vm\n";
6930                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6931                     }
6932
6933                     # if we clone a disk for a new target vm, we don't switch the disk
6934                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6935
6936                     if ($agent_running) {
6937                         print "unfreeze filesystem\n";
6938                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6939                     } else {
6940                         print "resume vm\n";
6941                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6942                     }
6943
6944                     last;
6945                 } else {
6946
6947                     foreach my $job (keys %$jobs) {
6948                         # try to switch the disk if source and destination are on the same guest
6949                         print "$job: Completing block job...\n";
6950
6951                         my $op;
6952                         if ($completion eq 'complete') {
6953                             $op = 'block-job-complete';
6954                         } elsif ($completion eq 'cancel') {
6955                             $op = 'block-job-cancel';
6956                         } else {
6957                             die "invalid completion value: $completion\n";
6958                         }
6959                         eval { mon_cmd($vmid, $op, device => $job) };
6960                         if ($@ =~ m/cannot be completed/) {
6961                             print "$job: Block job cannot be completed, try again.\n";
6962                             $err_complete++;
6963                         }else {
6964                             print "$job: Completed successfully.\n";
6965                             $jobs->{$job}->{complete} = 1;
6966                         }
6967                     }
6968                 }
6969             }
6970             sleep 1;
6971         }
6972     };
6973     my $err = $@;
6974
6975     if ($err) {
6976         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6977         die "mirroring error: $err";
6978     }
6979
6980 }
6981
6982 sub qemu_blockjobs_cancel {
6983     my ($vmid, $jobs) = @_;
6984
6985     foreach my $job (keys %$jobs) {
6986         print "$job: Cancelling block job\n";
6987         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6988         $jobs->{$job}->{cancel} = 1;
6989     }
6990
6991     while (1) {
6992         my $stats = mon_cmd($vmid, "query-block-jobs");
6993
6994         my $running_jobs = {};
6995         foreach my $stat (@$stats) {
6996             $running_jobs->{$stat->{device}} = $stat;
6997         }
6998
6999         foreach my $job (keys %$jobs) {
7000
7001             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7002                 print "$job: Done.\n";
7003                 delete $jobs->{$job};
7004             }
7005         }
7006
7007         last if scalar(keys %$jobs) == 0;
7008
7009         sleep 1;
7010     }
7011 }
7012
7013 sub clone_disk {
7014     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7015         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
7016
7017     my $newvolid;
7018
7019     if (!$full) {
7020         print "create linked clone of drive $drivename ($drive->{file})\n";
7021         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7022         push @$newvollist, $newvolid;
7023     } else {
7024
7025         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7026         $storeid = $storage if $storage;
7027
7028         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7029
7030         print "create full clone of drive $drivename ($drive->{file})\n";
7031         my $name = undef;
7032         my $size = undef;
7033         if (drive_is_cloudinit($drive)) {
7034             $name = "vm-$newvmid-cloudinit";
7035             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7036             if ($scfg->{path}) {
7037                 $name .= ".$dst_format";
7038             }
7039             $snapname = undef;
7040             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
7041         } elsif ($drivename eq 'efidisk0') {
7042             $size = get_efivars_size($conf);
7043         } else {
7044             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7045         }
7046         $newvolid = PVE::Storage::vdisk_alloc(
7047             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7048         );
7049         push @$newvollist, $newvolid;
7050
7051         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7052
7053         if (drive_is_cloudinit($drive)) {
7054             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7055             # if this is the case, we have to complete any block-jobs still there from
7056             # previous drive-mirrors
7057             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7058                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7059             }
7060             goto no_data_clone;
7061         }
7062
7063         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7064         if (!$running || $snapname) {
7065             # TODO: handle bwlimits
7066             if ($drivename eq 'efidisk0') {
7067                 # the relevant data on the efidisk may be smaller than the source
7068                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7069                 # that is given by the OVMF_VARS.fd
7070                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7071                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7072
7073                 # better for Ceph if block size is not too small, see bug #3324
7074                 my $bs = 1024*1024;
7075
7076                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7077                     "if=$src_path", "of=$dst_path"]);
7078             } else {
7079                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7080             }
7081         } else {
7082
7083             my $kvmver = get_running_qemu_version ($vmid);
7084             if (!min_version($kvmver, 2, 7)) {
7085                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7086                     if $drive->{iothread};
7087             }
7088
7089             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7090                 $completion, $qga, $bwlimit);
7091         }
7092     }
7093
7094 no_data_clone:
7095     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7096
7097     my $disk = $drive;
7098     $disk->{format} = undef;
7099     $disk->{file} = $newvolid;
7100     $disk->{size} = $size if defined($size);
7101
7102     return $disk;
7103 }
7104
7105 sub get_running_qemu_version {
7106     my ($vmid) = @_;
7107     my $res = mon_cmd($vmid, "query-version");
7108     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7109 }
7110
7111 sub qemu_use_old_bios_files {
7112     my ($machine_type) = @_;
7113
7114     return if !$machine_type;
7115
7116     my $use_old_bios_files = undef;
7117
7118     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7119         $machine_type = $1;
7120         $use_old_bios_files = 1;
7121     } else {
7122         my $version = extract_version($machine_type, kvm_user_version());
7123         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7124         # load new efi bios files on migration. So this hack is required to allow
7125         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7126         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7127         $use_old_bios_files = !min_version($version, 2, 4);
7128     }
7129
7130     return ($use_old_bios_files, $machine_type);
7131 }
7132
7133 sub get_efivars_size {
7134     my ($conf) = @_;
7135     my $arch = get_vm_arch($conf);
7136     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7137     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7138     return -s $ovmf_vars;
7139 }
7140
7141 sub update_efidisk_size {
7142     my ($conf) = @_;
7143
7144     return if !defined($conf->{efidisk0});
7145
7146     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7147     $disk->{size} = get_efivars_size($conf);
7148     $conf->{efidisk0} = print_drive($disk);
7149
7150     return;
7151 }
7152
7153 sub create_efidisk($$$$$) {
7154     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7155
7156     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7157     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7158
7159     my $vars_size_b = -s $ovmf_vars;
7160     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7161     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7162     PVE::Storage::activate_volumes($storecfg, [$volid]);
7163
7164     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7165     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7166
7167     return ($volid, $size/1024);
7168 }
7169
7170 sub vm_iothreads_list {
7171     my ($vmid) = @_;
7172
7173     my $res = mon_cmd($vmid, 'query-iothreads');
7174
7175     my $iothreads = {};
7176     foreach my $iothread (@$res) {
7177         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7178     }
7179
7180     return $iothreads;
7181 }
7182
7183 sub scsihw_infos {
7184     my ($conf, $drive) = @_;
7185
7186     my $maxdev = 0;
7187
7188     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7189         $maxdev = 7;
7190     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7191         $maxdev = 1;
7192     } else {
7193         $maxdev = 256;
7194     }
7195
7196     my $controller = int($drive->{index} / $maxdev);
7197     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7198         ? "virtioscsi"
7199         : "scsihw";
7200
7201     return ($maxdev, $controller, $controller_prefix);
7202 }
7203
7204 sub windows_version {
7205     my ($ostype) = @_;
7206
7207     return 0 if !$ostype;
7208
7209     my $winversion = 0;
7210
7211     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7212         $winversion = 5;
7213     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7214         $winversion = 6;
7215     } elsif ($ostype =~ m/^win(\d+)$/) {
7216         $winversion = $1;
7217     }
7218
7219     return $winversion;
7220 }
7221
7222 sub resolve_dst_disk_format {
7223         my ($storecfg, $storeid, $src_volname, $format) = @_;
7224         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7225
7226         if (!$format) {
7227             # if no target format is specified, use the source disk format as hint
7228             if ($src_volname) {
7229                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7230                 $format = qemu_img_format($scfg, $src_volname);
7231             } else {
7232                 return $defFormat;
7233             }
7234         }
7235
7236         # test if requested format is supported - else use default
7237         my $supported = grep { $_ eq $format } @$validFormats;
7238         $format = $defFormat if !$supported;
7239         return $format;
7240 }
7241
7242 # NOTE: if this logic changes, please update docs & possibly gui logic
7243 sub find_vmstate_storage {
7244     my ($conf, $storecfg) = @_;
7245
7246     # first, return storage from conf if set
7247     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7248
7249     my ($target, $shared, $local);
7250
7251     foreach_storage_used_by_vm($conf, sub {
7252         my ($sid) = @_;
7253         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7254         my $dst = $scfg->{shared} ? \$shared : \$local;
7255         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7256     });
7257
7258     # second, use shared storage where VM has at least one disk
7259     # third, use local storage where VM has at least one disk
7260     # fall back to local storage
7261     $target = $shared // $local // 'local';
7262
7263     return $target;
7264 }
7265
7266 sub generate_uuid {
7267     my ($uuid, $uuid_str);
7268     UUID::generate($uuid);
7269     UUID::unparse($uuid, $uuid_str);
7270     return $uuid_str;
7271 }
7272
7273 sub generate_smbios1_uuid {
7274     return "uuid=".generate_uuid();
7275 }
7276
7277 sub nbd_stop {
7278     my ($vmid) = @_;
7279
7280     mon_cmd($vmid, 'nbd-server-stop');
7281 }
7282
7283 sub create_reboot_request {
7284     my ($vmid) = @_;
7285     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7286         or die "failed to create reboot trigger file: $!\n";
7287     close($fh);
7288 }
7289
7290 sub clear_reboot_request {
7291     my ($vmid) = @_;
7292     my $path = "/run/qemu-server/$vmid.reboot";
7293     my $res = 0;
7294
7295     $res = unlink($path);
7296     die "could not remove reboot request for $vmid: $!"
7297         if !$res && $! != POSIX::ENOENT;
7298
7299     return $res;
7300 }
7301
7302 sub bootorder_from_legacy {
7303     my ($conf, $bootcfg) = @_;
7304
7305     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7306     my $bootindex_hash = {};
7307     my $i = 1;
7308     foreach my $o (split(//, $boot)) {
7309         $bootindex_hash->{$o} = $i*100;
7310         $i++;
7311     }
7312
7313     my $bootorder = {};
7314
7315     PVE::QemuConfig->foreach_volume($conf, sub {
7316         my ($ds, $drive) = @_;
7317
7318         if (drive_is_cdrom ($drive, 1)) {
7319             if ($bootindex_hash->{d}) {
7320                 $bootorder->{$ds} = $bootindex_hash->{d};
7321                 $bootindex_hash->{d} += 1;
7322             }
7323         } elsif ($bootindex_hash->{c}) {
7324             $bootorder->{$ds} = $bootindex_hash->{c}
7325                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7326             $bootindex_hash->{c} += 1;
7327         }
7328     });
7329
7330     if ($bootindex_hash->{n}) {
7331         for (my $i = 0; $i < $MAX_NETS; $i++) {
7332             my $netname = "net$i";
7333             next if !$conf->{$netname};
7334             $bootorder->{$netname} = $bootindex_hash->{n};
7335             $bootindex_hash->{n} += 1;
7336         }
7337     }
7338
7339     return $bootorder;
7340 }
7341
7342 # Generate default device list for 'boot: order=' property. Matches legacy
7343 # default boot order, but with explicit device names. This is important, since
7344 # the fallback for when neither 'order' nor the old format is specified relies
7345 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7346 sub get_default_bootdevices {
7347     my ($conf) = @_;
7348
7349     my @ret = ();
7350
7351     # harddisk
7352     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7353     push @ret, $first if $first;
7354
7355     # cdrom
7356     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7357     push @ret, $first if $first;
7358
7359     # network
7360     for (my $i = 0; $i < $MAX_NETS; $i++) {
7361         my $netname = "net$i";
7362         next if !$conf->{$netname};
7363         push @ret, $netname;
7364         last;
7365     }
7366
7367     return \@ret;
7368 }
7369
7370 sub device_bootorder {
7371     my ($conf) = @_;
7372
7373     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7374
7375     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7376
7377     my $bootorder = {};
7378     if (!defined($boot) || $boot->{legacy}) {
7379         $bootorder = bootorder_from_legacy($conf, $boot);
7380     } elsif ($boot->{order}) {
7381         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7382         for my $dev (PVE::Tools::split_list($boot->{order})) {
7383             $bootorder->{$dev} = $i++;
7384         }
7385     }
7386
7387     return $bootorder;
7388 }
7389
7390 # bash completion helper
7391
7392 sub complete_backup_archives {
7393     my ($cmdname, $pname, $cvalue) = @_;
7394
7395     my $cfg = PVE::Storage::config();
7396
7397     my $storeid;
7398
7399     if ($cvalue =~ m/^([^:]+):/) {
7400         $storeid = $1;
7401     }
7402
7403     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7404
7405     my $res = [];
7406     foreach my $id (keys %$data) {
7407         foreach my $item (@{$data->{$id}}) {
7408             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7409             push @$res, $item->{volid} if defined($item->{volid});
7410         }
7411     }
7412
7413     return $res;
7414 }
7415
7416 my $complete_vmid_full = sub {
7417     my ($running) = @_;
7418
7419     my $idlist = vmstatus();
7420
7421     my $res = [];
7422
7423     foreach my $id (keys %$idlist) {
7424         my $d = $idlist->{$id};
7425         if (defined($running)) {
7426             next if $d->{template};
7427             next if $running && $d->{status} ne 'running';
7428             next if !$running && $d->{status} eq 'running';
7429         }
7430         push @$res, $id;
7431
7432     }
7433     return $res;
7434 };
7435
7436 sub complete_vmid {
7437     return &$complete_vmid_full();
7438 }
7439
7440 sub complete_vmid_stopped {
7441     return &$complete_vmid_full(0);
7442 }
7443
7444 sub complete_vmid_running {
7445     return &$complete_vmid_full(1);
7446 }
7447
7448 sub complete_storage {
7449
7450     my $cfg = PVE::Storage::config();
7451     my $ids = $cfg->{ids};
7452
7453     my $res = [];
7454     foreach my $sid (keys %$ids) {
7455         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7456         next if !$ids->{$sid}->{content}->{images};
7457         push @$res, $sid;
7458     }
7459
7460     return $res;
7461 }
7462
7463 sub complete_migration_storage {
7464     my ($cmd, $param, $current_value, $all_args) = @_;
7465
7466     my $targetnode = @$all_args[1];
7467
7468     my $cfg = PVE::Storage::config();
7469     my $ids = $cfg->{ids};
7470
7471     my $res = [];
7472     foreach my $sid (keys %$ids) {
7473         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7474         next if !$ids->{$sid}->{content}->{images};
7475         push @$res, $sid;
7476     }
7477
7478     return $res;
7479 }
7480
7481 sub vm_is_paused {
7482     my ($vmid) = @_;
7483     my $qmpstatus = eval {
7484         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7485         mon_cmd($vmid, "query-status");
7486     };
7487     warn "$@\n" if $@;
7488     return $qmpstatus && $qmpstatus->{status} eq "paused";
7489 }
7490
7491 1;