PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 970 dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $usb_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string', format => 'pve-qm-usb-device',
1005         format_description => 'HOSTUSBDEVICE|spice',
1006         description => <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1008
1009  'bus-port(.port)*' (decimal numbers) or
1010  'vendor_id:product_id' (hexadeciaml numbers) or
1011  'spice'
1012
1013 You can use the 'lsusb -t' command to list existing usb devices.
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1017
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1019 EODESCR
1020     },
1021     usb3 => {
1022         optional => 1,
1023         type => 'boolean',
1024         description => "Specifies whether if given host option is a USB3 device or port.",
1025         default => 0,
1026     },
1027 };
1028
1029 my $usbdesc = {
1030     optional => 1,
1031     type => 'string', format => $usb_fmt,
1032     description => "Configure an USB device (n is 0 to 4).",
1033 };
1034 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1035
1036 my $serialdesc = {
1037         optional => 1,
1038         type => 'string',
1039         pattern => '(/dev/.+|socket)',
1040         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1041         verbose_description =>  <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1045
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052
1053 my $paralleldesc= {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1057         description =>  "Map host parallel devices (n is 0 to 2).",
1058         verbose_description =>  <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1078 }
1079
1080 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1081     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1082 }
1083
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1085     $confdesc->{"usb$i"} = $usbdesc;
1086 }
1087
1088 my $boot_fmt = {
1089     legacy => {
1090         optional => 1,
1091         default_key => 1,
1092         type => 'string',
1093         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094                      . " Deprecated, use 'order=' instead.",
1095         pattern => '[acdn]{1,4}',
1096         format_description => "[acdn]{1,4}",
1097
1098         # note: this is also the fallback if boot: is not given at all
1099         default => 'cdn',
1100     },
1101     order => {
1102         optional => 1,
1103         type => 'string',
1104         format => 'pve-qm-bootdev-list',
1105         format_description => "device[;device...]",
1106         description => <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1108
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1112
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1116
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1118 EODESC
1119     },
1120 };
1121 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1122
1123 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1124 sub verify_bootdev {
1125     my ($dev, $noerr) = @_;
1126
1127     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1128
1129     my $check = sub {
1130         my ($base) = @_;
1131         return 0 if $dev !~ m/^$base\d+$/;
1132         return 0 if !$confdesc->{$dev};
1133         return 1;
1134     };
1135
1136     return $dev if $check->("net");
1137     return $dev if $check->("usb");
1138     return $dev if $check->("hostpci");
1139
1140     return if $noerr;
1141     die "invalid boot device '$dev'\n";
1142 }
1143
1144 sub print_bootorder {
1145     my ($devs) = @_;
1146     return "" if !@$devs;
1147     my $data = { order => join(';', @$devs) };
1148     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1149 }
1150
1151 my $kvm_api_version = 0;
1152
1153 sub kvm_version {
1154     return $kvm_api_version if $kvm_api_version;
1155
1156     open my $fh, '<', '/dev/kvm' or return;
1157
1158     # 0xae00 => KVM_GET_API_VERSION
1159     $kvm_api_version = ioctl($fh, 0xae00, 0);
1160     close($fh);
1161
1162     return $kvm_api_version;
1163 }
1164
1165 my $kvm_user_version = {};
1166 my $kvm_mtime = {};
1167
1168 sub kvm_user_version {
1169     my ($binary) = @_;
1170
1171     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1172     my $st = stat($binary);
1173
1174     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176         $cachedmtime == $st->mtime;
1177
1178     $kvm_user_version->{$binary} = 'unknown';
1179     $kvm_mtime->{$binary} = $st->mtime;
1180
1181     my $code = sub {
1182         my $line = shift;
1183         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184             $kvm_user_version->{$binary} = $2;
1185         }
1186     };
1187
1188     eval { run_command([$binary, '--version'], outfunc => $code); };
1189     warn $@ if $@;
1190
1191     return $kvm_user_version->{$binary};
1192
1193 }
1194 my sub extract_version {
1195     my ($machine_type, $version) = @_;
1196     $version = kvm_user_version() if !defined($version);
1197     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1198 }
1199
1200 sub kernel_has_vhost_net {
1201     return -c '/dev/vhost-net';
1202 }
1203
1204 sub option_exists {
1205     my $key = shift;
1206     return defined($confdesc->{$key});
1207 }
1208
1209 my $cdrom_path;
1210 sub get_cdrom_path {
1211
1212     return  $cdrom_path if $cdrom_path;
1213
1214     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1215     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1216     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1217 }
1218
1219 sub get_iso_path {
1220     my ($storecfg, $vmid, $cdrom) = @_;
1221
1222     if ($cdrom eq 'cdrom') {
1223         return get_cdrom_path();
1224     } elsif ($cdrom eq 'none') {
1225         return '';
1226     } elsif ($cdrom =~ m|^/|) {
1227         return $cdrom;
1228     } else {
1229         return PVE::Storage::path($storecfg, $cdrom);
1230     }
1231 }
1232
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id {
1235     my ($vmid, $file, $media) = @_;
1236
1237      if (!($file eq 'none' || $file eq 'cdrom' ||
1238           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1239
1240         return if $file =~ m|/|;
1241
1242         if ($media && $media eq 'cdrom') {
1243             $file = "local:iso/$file";
1244         } else {
1245             $file = "local:$vmid/$file";
1246         }
1247     }
1248
1249     return $file;
1250 }
1251
1252 sub verify_media_type {
1253     my ($opt, $vtype, $media) = @_;
1254
1255     return if !$media;
1256
1257     my $etype;
1258     if ($media eq 'disk') {
1259         $etype = 'images';
1260     } elsif ($media eq 'cdrom') {
1261         $etype = 'iso';
1262     } else {
1263         die "internal error";
1264     }
1265
1266     return if ($vtype eq $etype);
1267
1268     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1269 }
1270
1271 sub cleanup_drive_path {
1272     my ($opt, $storecfg, $drive) = @_;
1273
1274     # try to convert filesystem paths to volume IDs
1275
1276     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1277         ($drive->{file} !~ m|^/dev/.+|) &&
1278         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1279         ($drive->{file} !~ m/^\d+$/)) {
1280         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1281         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1282             if !$vtype;
1283         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1284         verify_media_type($opt, $vtype, $drive->{media});
1285         $drive->{file} = $volid;
1286     }
1287
1288     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1289 }
1290
1291 sub parse_hotplug_features {
1292     my ($data) = @_;
1293
1294     my $res = {};
1295
1296     return $res if $data eq '0';
1297
1298     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1299
1300     foreach my $feature (PVE::Tools::split_list($data)) {
1301         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1302             $res->{$1} = 1;
1303         } else {
1304             die "invalid hotplug feature '$feature'\n";
1305         }
1306     }
1307     return $res;
1308 }
1309
1310 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1311 sub pve_verify_hotplug_features {
1312     my ($value, $noerr) = @_;
1313
1314     return $value if parse_hotplug_features($value);
1315
1316     return if $noerr;
1317
1318     die "unable to parse hotplug option\n";
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub print_tabletdevice_full {
1383     my ($conf, $arch) = @_;
1384
1385     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1386
1387     # we use uhci for old VMs because tablet driver was buggy in older qemu
1388     my $usbbus;
1389     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1390         $usbbus = 'ehci';
1391     } else {
1392         $usbbus = 'uhci';
1393     }
1394
1395     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1396 }
1397
1398 sub print_keyboarddevice_full {
1399     my ($conf, $arch, $machine) = @_;
1400
1401     return if $arch ne 'aarch64';
1402
1403     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1404 }
1405
1406 sub print_drivedevice_full {
1407     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1408
1409     my $device = '';
1410     my $maxdev = 0;
1411
1412     my $drive_id = "$drive->{interface}$drive->{index}";
1413     if ($drive->{interface} eq 'virtio') {
1414         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1415         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1417     } elsif ($drive->{interface} eq 'scsi') {
1418
1419         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1420         my $unit = $drive->{index} % $maxdev;
1421         my $devicetype = 'hd';
1422         my $path = '';
1423         if (drive_is_cdrom($drive)) {
1424             $devicetype = 'cd';
1425         } else {
1426             if ($drive->{file} =~ m|^/|) {
1427                 $path = $drive->{file};
1428                 if (my $info = path_is_scsi($path)) {
1429                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1430                         $devicetype = 'block';
1431                     } elsif ($info->{type} == 1) { # tape
1432                         $devicetype = 'generic';
1433                     }
1434                 }
1435             } else {
1436                  $path = PVE::Storage::path($storecfg, $drive->{file});
1437             }
1438
1439             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440             my $version = extract_version($machine_type, kvm_user_version());
1441             if ($path =~ m/^iscsi\:\/\// &&
1442                !min_version($version, 4, 1)) {
1443                 $devicetype = 'generic';
1444             }
1445         }
1446
1447         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1448             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1449         } else {
1450             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451                 .",lun=$drive->{index}";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456             $device .= ",rotation_rate=1";
1457         }
1458         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1459
1460     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1461         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1462         my $controller = int($drive->{index} / $maxdev);
1463         my $unit = $drive->{index} % $maxdev;
1464         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1465
1466         $device = "ide-$devicetype";
1467         if ($drive->{interface} eq 'ide') {
1468             $device .= ",bus=ide.$controller,unit=$unit";
1469         } else {
1470             $device .= ",bus=ahci$controller.$unit";
1471         }
1472         $device .= ",drive=drive-$drive_id,id=$drive_id";
1473
1474         if ($devicetype eq 'hd') {
1475             if (my $model = $drive->{model}) {
1476                 $model = URI::Escape::uri_unescape($model);
1477                 $device .= ",model=$model";
1478             }
1479             if ($drive->{ssd}) {
1480                 $device .= ",rotation_rate=1";
1481             }
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484     } elsif ($drive->{interface} eq 'usb') {
1485         die "implement me";
1486         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1487     } else {
1488         die "unsupported interface type";
1489     }
1490
1491     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1492
1493     if (my $serial = $drive->{serial}) {
1494         $serial = URI::Escape::uri_unescape($serial);
1495         $device .= ",serial=$serial";
1496     }
1497
1498
1499     return $device;
1500 }
1501
1502 sub get_initiator_name {
1503     my $initiator;
1504
1505     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1506     while (defined(my $line = <$fh>)) {
1507         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1508         $initiator = $1;
1509         last;
1510     }
1511     $fh->close();
1512
1513     return $initiator;
1514 }
1515
1516 sub print_drive_commandline_full {
1517     my ($storecfg, $vmid, $drive) = @_;
1518
1519     my $path;
1520     my $volid = $drive->{file};
1521     my $format;
1522
1523     if (drive_is_cdrom($drive)) {
1524         $path = get_iso_path($storecfg, $vmid, $volid);
1525     } else {
1526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1527         if ($storeid) {
1528             $path = PVE::Storage::path($storecfg, $volid);
1529             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1530             $format = qemu_img_format($scfg, $volname);
1531         } else {
1532             $path = $volid;
1533             $format = "raw";
1534         }
1535    }
1536
1537     my $opts = '';
1538     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539     foreach my $o (@qemu_drive_options) {
1540         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1541     }
1542
1543     # snapshot only accepts on|off
1544     if (defined($drive->{snapshot})) {
1545         my $v = $drive->{snapshot} ? 'on' : 'off';
1546         $opts .= ",snapshot=$v";
1547     }
1548
1549     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1550         my ($dir, $qmpname) = @$type;
1551         if (my $v = $drive->{"mbps$dir"}) {
1552             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1553         }
1554         if (my $v = $drive->{"mbps${dir}_max"}) {
1555             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1556         }
1557         if (my $v = $drive->{"bps${dir}_max_length"}) {
1558             $opts .= ",throttling.bps$qmpname-max-length=$v";
1559         }
1560         if (my $v = $drive->{"iops${dir}"}) {
1561             $opts .= ",throttling.iops$qmpname=$v";
1562         }
1563         if (my $v = $drive->{"iops${dir}_max"}) {
1564             $opts .= ",throttling.iops$qmpname-max=$v";
1565         }
1566         if (my $v = $drive->{"iops${dir}_max_length"}) {
1567             $opts .= ",throttling.iops$qmpname-max-length=$v";
1568         }
1569     }
1570
1571     $opts .= ",format=$format" if $format && !$drive->{format};
1572
1573     my $cache_direct = 0;
1574
1575     if (my $cache = $drive->{cache}) {
1576         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577     } elsif (!drive_is_cdrom($drive)) {
1578         $opts .= ",cache=none";
1579         $cache_direct = 1;
1580     }
1581
1582     # aio native works only with O_DIRECT
1583     if (!$drive->{aio}) {
1584         if($cache_direct) {
1585             $opts .= ",aio=native";
1586         } else {
1587             $opts .= ",aio=threads";
1588         }
1589     }
1590
1591     if (!drive_is_cdrom($drive)) {
1592         my $detectzeroes;
1593         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1594             $detectzeroes = 'off';
1595         } elsif ($drive->{discard}) {
1596             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1597         } else {
1598             # This used to be our default with discard not being specified:
1599             $detectzeroes = 'on';
1600         }
1601         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1602     }
1603
1604     my $pathinfo = $path ? "file=$path," : '';
1605
1606     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1607 }
1608
1609 sub print_netdevice_full {
1610     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1611
1612     my $device = $net->{model};
1613     if ($net->{model} eq 'virtio') {
1614          $device = 'virtio-net-pci';
1615      };
1616
1617     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1618     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1620         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621         # and out of each queue plus one config interrupt and control vector queue
1622         my $vectors = $net->{queues} * 2 + 2;
1623         $tmpstr .= ",vectors=$vectors,mq=on";
1624     }
1625     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1626
1627     if (my $mtu = $net->{mtu}) {
1628         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1629             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1630             if ($mtu == 1) {
1631                  $mtu = $bridge_mtu;
1632             } elsif ($mtu < 576) {
1633                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634             } elsif ($mtu > $bridge_mtu) {
1635                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1636             }
1637             $tmpstr .= ",host_mtu=$mtu";
1638         } else {
1639             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1640         }
1641     }
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     if (is_native($arch)) {
1680         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1681     }
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1691     } else {
1692         $netdev = "type=user,id=$netid,hostname=$vmname";
1693     }
1694
1695     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1696
1697     return $netdev;
1698 }
1699
1700 my $vga_map = {
1701     'cirrus' => 'cirrus-vga',
1702     'std' => 'VGA',
1703     'vmware' => 'vmware-svga',
1704     'virtio' => 'virtio-vga',
1705 };
1706
1707 sub print_vga_device {
1708     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1709
1710     my $type = $vga_map->{$vga->{type}};
1711     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712         $type = 'virtio-gpu';
1713     }
1714     my $vgamem_mb = $vga->{memory};
1715
1716     my $max_outputs = '';
1717     if ($qxlnum) {
1718         $type = $id ? 'qxl' : 'qxl-vga';
1719
1720         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1721             # set max outputs so linux can have up to 4 qxl displays with one device
1722             if (min_version($machine_version, 4, 1)) {
1723                 $max_outputs = ",max_outputs=4";
1724             }
1725         }
1726     }
1727
1728     die "no devicetype for $vga->{type}\n" if !$type;
1729
1730     my $memory = "";
1731     if ($vgamem_mb) {
1732         if ($vga->{type} eq 'virtio') {
1733             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1734             $memory = ",max_hostmem=$bytes";
1735         } elsif ($qxlnum) {
1736             # from https://www.spice-space.org/multiple-monitors.html
1737             $memory = ",vgamem_mb=$vga->{memory}";
1738             my $ram = $vgamem_mb * 4;
1739             my $vram = $vgamem_mb * 2;
1740             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1741         } else {
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743         }
1744     } elsif ($qxlnum && $id) {
1745         $memory = ",ram_size=67108864,vram_size=33554432";
1746     }
1747
1748     my $edidoff = "";
1749     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1750         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1751     }
1752
1753     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1754     my $vgaid = "vga" . ($id // '');
1755     my $pciaddr;
1756
1757     if ($q35 && $vgaid eq 'vga') {
1758         # the first display uses pcie.0 bus on q35 machines
1759         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1760     } else {
1761         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1762     }
1763
1764     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1765 }
1766
1767 sub parse_number_sets {
1768     my ($set) = @_;
1769     my $res = [];
1770     foreach my $part (split(/;/, $set)) {
1771         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773             push @$res, [ $1, $2 ];
1774         } else {
1775             die "invalid range: $part\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 sub parse_numa {
1782     my ($data) = @_;
1783
1784     my $res = parse_property_string($numa_fmt, $data);
1785     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1786     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1787     return $res;
1788 }
1789
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1791 sub parse_net {
1792     my ($data) = @_;
1793
1794     my $res = eval { parse_property_string($net_fmt, $data) };
1795     if ($@) {
1796         warn $@;
1797         return;
1798     }
1799     if (!defined($res->{macaddr})) {
1800         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1801         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1802     }
1803     return $res;
1804 }
1805
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig {
1808     my ($data) = @_;
1809
1810     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1811     if ($@) {
1812         warn $@;
1813         return;
1814     }
1815
1816     if ($res->{gw} && !$res->{ip}) {
1817         warn 'gateway specified without specifying an IP address';
1818         return;
1819     }
1820     if ($res->{gw6} && !$res->{ip6}) {
1821         warn 'IPv6 gateway specified without specifying an IPv6 address';
1822         return;
1823     }
1824     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1825         warn 'gateway specified together with DHCP';
1826         return;
1827     }
1828     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1829         # gw6 + auto/dhcp
1830         warn "IPv6 gateway specified together with $res->{ip6} address";
1831         return;
1832     }
1833
1834     if (!$res->{ip} && !$res->{ip6}) {
1835         return { ip => 'dhcp', ip6 => 'dhcp' };
1836     }
1837
1838     return $res;
1839 }
1840
1841 sub print_net {
1842     my $net = shift;
1843
1844     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1845 }
1846
1847 sub add_random_macs {
1848     my ($settings) = @_;
1849
1850     foreach my $opt (keys %$settings) {
1851         next if $opt !~ m/^net(\d+)$/;
1852         my $net = parse_net($settings->{$opt});
1853         next if !$net;
1854         $settings->{$opt} = print_net($net);
1855     }
1856 }
1857
1858 sub vm_is_volid_owner {
1859     my ($storecfg, $vmid, $volid) = @_;
1860
1861     if ($volid !~  m|^/|) {
1862         my ($path, $owner);
1863         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1864         if ($owner && ($owner == $vmid)) {
1865             return 1;
1866         }
1867     }
1868
1869     return;
1870 }
1871
1872 sub vmconfig_register_unused_drive {
1873     my ($storecfg, $vmid, $conf, $drive) = @_;
1874
1875     if (drive_is_cloudinit($drive)) {
1876         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1877         warn $@ if $@;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         my $volid = $drive->{file};
1880         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1881             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1882         }
1883     }
1884 }
1885
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1887 my $smbios1_fmt = {
1888     uuid => {
1889         type => 'string',
1890         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891         format_description => 'UUID',
1892         description => "Set SMBIOS1 UUID.",
1893         optional => 1,
1894     },
1895     version => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 version.",
1900         optional => 1,
1901     },
1902     serial => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 serial number.",
1907         optional => 1,
1908     },
1909     manufacturer => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 manufacturer.",
1914         optional => 1,
1915     },
1916     product => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 product ID.",
1921         optional => 1,
1922     },
1923     sku => {
1924         type => 'string',
1925         pattern => '[A-Za-z0-9+\/]+={0,2}',
1926         format_description => 'Base64 encoded string',
1927         description => "Set SMBIOS1 SKU string.",
1928         optional => 1,
1929     },
1930     family => {
1931         type => 'string',
1932         pattern => '[A-Za-z0-9+\/]+={0,2}',
1933         format_description => 'Base64 encoded string',
1934         description => "Set SMBIOS1 family string.",
1935         optional => 1,
1936     },
1937     base64 => {
1938         type => 'boolean',
1939         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1940         optional => 1,
1941     },
1942 };
1943
1944 sub parse_smbios1 {
1945     my ($data) = @_;
1946
1947     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1948     warn $@ if $@;
1949     return $res;
1950 }
1951
1952 sub print_smbios1 {
1953     my ($smbios1) = @_;
1954     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1958
1959 sub parse_watchdog {
1960     my ($value) = @_;
1961
1962     return if !$value;
1963
1964     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1965     warn $@ if $@;
1966     return $res;
1967 }
1968
1969 sub parse_guest_agent {
1970     my ($conf) = @_;
1971
1972     return {} if !defined($conf->{agent});
1973
1974     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1975     warn $@ if $@;
1976
1977     # if the agent is disabled ignore the other potentially set properties
1978     return {} if !$res->{enabled};
1979     return $res;
1980 }
1981
1982 sub get_qga_key {
1983     my ($conf, $key) = @_;
1984     return undef if !defined($conf->{agent});
1985
1986     my $agent = parse_guest_agent($conf);
1987     return $agent->{$key};
1988 }
1989
1990 sub parse_vga {
1991     my ($value) = @_;
1992
1993     return {} if !$value;
1994     my $res = eval { parse_property_string($vga_fmt, $value) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub parse_rng {
2000     my ($value) = @_;
2001
2002     return if !$value;
2003
2004     my $res = eval { parse_property_string($rng_fmt, $value) };
2005     warn $@ if $@;
2006     return $res;
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2010 sub verify_usb_device {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if parse_usb_device($value);
2014
2015     return if $noerr;
2016
2017     die "unable to parse usb device\n";
2018 }
2019
2020 # add JSON properties for create and set function
2021 sub json_config_properties {
2022     my $prop = shift;
2023
2024     foreach my $opt (keys %$confdesc) {
2025         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027         $prop->{$opt} = $confdesc->{$opt};
2028     }
2029
2030     return $prop;
2031 }
2032
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties {
2035
2036     return dclone($confdesc_cloudinit);
2037 }
2038
2039 sub check_type {
2040     my ($key, $value) = @_;
2041
2042     die "unknown setting '$key'\n" if !$confdesc->{$key};
2043
2044     my $type = $confdesc->{$key}->{type};
2045
2046     if (!defined($value)) {
2047         die "got undefined value\n";
2048     }
2049
2050     if ($value =~ m/[\n\r]/) {
2051         die "property contains a line feed\n";
2052     }
2053
2054     if ($type eq 'boolean') {
2055         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057         die "type check ('boolean') failed - got '$value'\n";
2058     } elsif ($type eq 'integer') {
2059         return int($1) if $value =~ m/^(\d+)$/;
2060         die "type check ('integer') failed - got '$value'\n";
2061     } elsif ($type eq 'number') {
2062         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063         die "type check ('number') failed - got '$value'\n";
2064     } elsif ($type eq 'string') {
2065         if (my $fmt = $confdesc->{$key}->{format}) {
2066             PVE::JSONSchema::check_format($fmt, $value);
2067             return $value;
2068         }
2069         $value =~ s/^\"(.*)\"$/$1/;
2070         return $value;
2071     } else {
2072         die "internal error"
2073     }
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     if ($conf->{template}) {
2084         # check if any base image is still used by a linked clone
2085         PVE::QemuConfig->foreach_volume($conf, sub {
2086                 my ($ds, $drive) = @_;
2087                 return if drive_is_cdrom($drive);
2088
2089                 my $volid = $drive->{file};
2090                 return if !$volid || $volid =~ m|^/|;
2091
2092                 die "base volume '$volid' is still in use by linked cloned\n"
2093                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2094
2095         });
2096     }
2097
2098     # only remove disks owned by this VM (referenced in the config)
2099     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2100         my ($ds, $drive) = @_;
2101         return if drive_is_cdrom($drive, 1);
2102
2103         my $volid = $drive->{file};
2104         return if !$volid || $volid =~ m|^/|;
2105
2106         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2107         return if !$path || !$owner || ($owner != $vmid);
2108
2109         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2110         warn "Could not remove disk '$volid', check manually: $@" if $@;
2111     });
2112
2113     if ($purge_unreferenced) { # also remove unreferenced disk
2114         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2115         PVE::Storage::foreach_volid($vmdisks, sub {
2116             my ($volid, $sid, $volname, $d) = @_;
2117             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2118             warn $@ if $@;
2119         });
2120     }
2121
2122     if (defined $replacement_conf) {
2123         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2124     } else {
2125         PVE::QemuConfig->destroy_config($vmid);
2126     }
2127 }
2128
2129 sub parse_vm_config {
2130     my ($filename, $raw) = @_;
2131
2132     return if !defined($raw);
2133
2134     my $res = {
2135         digest => Digest::SHA::sha1_hex($raw),
2136         snapshots => {},
2137         pending => {},
2138     };
2139
2140     $filename =~ m|/qemu-server/(\d+)\.conf$|
2141         || die "got strange filename '$filename'";
2142
2143     my $vmid = $1;
2144
2145     my $conf = $res;
2146     my $descr;
2147     my $section = '';
2148
2149     my @lines = split(/\n/, $raw);
2150     foreach my $line (@lines) {
2151         next if $line =~ m/^\s*$/;
2152
2153         if ($line =~ m/^\[PENDING\]\s*$/i) {
2154             $section = 'pending';
2155             if (defined($descr)) {
2156                 $descr =~ s/\s+$//;
2157                 $conf->{description} = $descr;
2158             }
2159             $descr = undef;
2160             $conf = $res->{$section} = {};
2161             next;
2162
2163         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2164             $section = $1;
2165             if (defined($descr)) {
2166                 $descr =~ s/\s+$//;
2167                 $conf->{description} = $descr;
2168             }
2169             $descr = undef;
2170             $conf = $res->{snapshots}->{$section} = {};
2171             next;
2172         }
2173
2174         if ($line =~ m/^\#(.*)\s*$/) {
2175             $descr = '' if !defined($descr);
2176             $descr .= PVE::Tools::decode_text($1) . "\n";
2177             next;
2178         }
2179
2180         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181             $descr = '' if !defined($descr);
2182             $descr .= PVE::Tools::decode_text($2);
2183         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184             $conf->{snapstate} = $1;
2185         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2186             my $key = $1;
2187             my $value = $2;
2188             $conf->{$key} = $value;
2189         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2190             my $value = $1;
2191             if ($section eq 'pending') {
2192                 $conf->{delete} = $value; # we parse this later
2193             } else {
2194                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2195             }
2196         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2197             my $key = $1;
2198             my $value = $2;
2199             eval { $value = check_type($key, $value); };
2200             if ($@) {
2201                 warn "vm $vmid - unable to parse value of '$key' - $@";
2202             } else {
2203                 $key = 'ide2' if $key eq 'cdrom';
2204                 my $fmt = $confdesc->{$key}->{format};
2205                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206                     my $v = parse_drive($key, $value);
2207                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2208                         $v->{file} = $volid;
2209                         $value = print_drive($v);
2210                     } else {
2211                         warn "vm $vmid - unable to parse value of '$key'\n";
2212                         next;
2213                     }
2214                 }
2215
2216                 $conf->{$key} = $value;
2217             }
2218         }
2219     }
2220
2221     if (defined($descr)) {
2222         $descr =~ s/\s+$//;
2223         $conf->{description} = $descr;
2224     }
2225     delete $res->{snapstate}; # just to be sure
2226
2227     return $res;
2228 }
2229
2230 sub write_vm_config {
2231     my ($filename, $conf) = @_;
2232
2233     delete $conf->{snapstate}; # just to be sure
2234
2235     if ($conf->{cdrom}) {
2236         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2237         $conf->{ide2} = $conf->{cdrom};
2238         delete $conf->{cdrom};
2239     }
2240
2241     # we do not use 'smp' any longer
2242     if ($conf->{sockets}) {
2243         delete $conf->{smp};
2244     } elsif ($conf->{smp}) {
2245         $conf->{sockets} = $conf->{smp};
2246         delete $conf->{cores};
2247         delete $conf->{smp};
2248     }
2249
2250     my $used_volids = {};
2251
2252     my $cleanup_config = sub {
2253         my ($cref, $pending, $snapname) = @_;
2254
2255         foreach my $key (keys %$cref) {
2256             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2257                 $key eq 'snapstate' || $key eq 'pending';
2258             my $value = $cref->{$key};
2259             if ($key eq 'delete') {
2260                 die "propertry 'delete' is only allowed in [PENDING]\n"
2261                     if !$pending;
2262                 # fixme: check syntax?
2263                 next;
2264             }
2265             eval { $value = check_type($key, $value); };
2266             die "unable to parse value of '$key' - $@" if $@;
2267
2268             $cref->{$key} = $value;
2269
2270             if (!$snapname && is_valid_drivename($key)) {
2271                 my $drive = parse_drive($key, $value);
2272                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2273             }
2274         }
2275     };
2276
2277     &$cleanup_config($conf);
2278
2279     &$cleanup_config($conf->{pending}, 1);
2280
2281     foreach my $snapname (keys %{$conf->{snapshots}}) {
2282         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2283         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2284     }
2285
2286     # remove 'unusedX' settings if we re-add a volume
2287     foreach my $key (keys %$conf) {
2288         my $value = $conf->{$key};
2289         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2290             delete $conf->{$key};
2291         }
2292     }
2293
2294     my $generate_raw_config = sub {
2295         my ($conf, $pending) = @_;
2296
2297         my $raw = '';
2298
2299         # add description as comment to top of file
2300         if (defined(my $descr = $conf->{description})) {
2301             if ($descr) {
2302                 foreach my $cl (split(/\n/, $descr)) {
2303                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2304                 }
2305             } else {
2306                 $raw .= "#\n" if $pending;
2307             }
2308         }
2309
2310         foreach my $key (sort keys %$conf) {
2311             next if $key =~ /^(digest|description|pending|snapshots)$/;
2312             $raw .= "$key: $conf->{$key}\n";
2313         }
2314         return $raw;
2315     };
2316
2317     my $raw = &$generate_raw_config($conf);
2318
2319     if (scalar(keys %{$conf->{pending}})){
2320         $raw .= "\n[PENDING]\n";
2321         $raw .= &$generate_raw_config($conf->{pending}, 1);
2322     }
2323
2324     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2325         $raw .= "\n[$snapname]\n";
2326         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2327     }
2328
2329     return $raw;
2330 }
2331
2332 sub load_defaults {
2333
2334     my $res = {};
2335
2336     # we use static defaults from our JSON schema configuration
2337     foreach my $key (keys %$confdesc) {
2338         if (defined(my $default = $confdesc->{$key}->{default})) {
2339             $res->{$key} = $default;
2340         }
2341     }
2342
2343     return $res;
2344 }
2345
2346 sub config_list {
2347     my $vmlist = PVE::Cluster::get_vmlist();
2348     my $res = {};
2349     return $res if !$vmlist || !$vmlist->{ids};
2350     my $ids = $vmlist->{ids};
2351     my $nodename = nodename();
2352
2353     foreach my $vmid (keys %$ids) {
2354         my $d = $ids->{$vmid};
2355         next if !$d->{node} || $d->{node} ne $nodename;
2356         next if !$d->{type} || $d->{type} ne 'qemu';
2357         $res->{$vmid}->{exists} = 1;
2358     }
2359     return $res;
2360 }
2361
2362 # test if VM uses local resources (to prevent migration)
2363 sub check_local_resources {
2364     my ($conf, $noerr) = @_;
2365
2366     my @loc_res = ();
2367
2368     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2369     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2370
2371     push @loc_res, "ivshmem" if $conf->{ivshmem};
2372
2373     foreach my $k (keys %$conf) {
2374         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2375         # sockets are safe: they will recreated be on the target side post-migrate
2376         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2377         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2378     }
2379
2380     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2381
2382     return \@loc_res;
2383 }
2384
2385 # check if used storages are available on all nodes (use by migrate)
2386 sub check_storage_availability {
2387     my ($storecfg, $conf, $node) = @_;
2388
2389     PVE::QemuConfig->foreach_volume($conf, sub {
2390         my ($ds, $drive) = @_;
2391
2392         my $volid = $drive->{file};
2393         return if !$volid;
2394
2395         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2396         return if !$sid;
2397
2398         # check if storage is available on both nodes
2399         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2400         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2401    });
2402 }
2403
2404 # list nodes where all VM images are available (used by has_feature API)
2405 sub shared_nodes {
2406     my ($conf, $storecfg) = @_;
2407
2408     my $nodelist = PVE::Cluster::get_nodelist();
2409     my $nodehash = { map { $_ => 1 } @$nodelist };
2410     my $nodename = nodename();
2411
2412     PVE::QemuConfig->foreach_volume($conf, sub {
2413         my ($ds, $drive) = @_;
2414
2415         my $volid = $drive->{file};
2416         return if !$volid;
2417
2418         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2419         if ($storeid) {
2420             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2421             if ($scfg->{disable}) {
2422                 $nodehash = {};
2423             } elsif (my $avail = $scfg->{nodes}) {
2424                 foreach my $node (keys %$nodehash) {
2425                     delete $nodehash->{$node} if !$avail->{$node};
2426                 }
2427             } elsif (!$scfg->{shared}) {
2428                 foreach my $node (keys %$nodehash) {
2429                     delete $nodehash->{$node} if $node ne $nodename
2430                 }
2431             }
2432         }
2433     });
2434
2435     return $nodehash
2436 }
2437
2438 sub check_local_storage_availability {
2439     my ($conf, $storecfg) = @_;
2440
2441     my $nodelist = PVE::Cluster::get_nodelist();
2442     my $nodehash = { map { $_ => {} } @$nodelist };
2443
2444     PVE::QemuConfig->foreach_volume($conf, sub {
2445         my ($ds, $drive) = @_;
2446
2447         my $volid = $drive->{file};
2448         return if !$volid;
2449
2450         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2451         if ($storeid) {
2452             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2453
2454             if ($scfg->{disable}) {
2455                 foreach my $node (keys %$nodehash) {
2456                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2457                 }
2458             } elsif (my $avail = $scfg->{nodes}) {
2459                 foreach my $node (keys %$nodehash) {
2460                     if (!$avail->{$node}) {
2461                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2462                     }
2463                 }
2464             }
2465         }
2466     });
2467
2468     foreach my $node (values %$nodehash) {
2469         if (my $unavail = $node->{unavailable_storages}) {
2470             $node->{unavailable_storages} = [ sort keys %$unavail ];
2471         }
2472     }
2473
2474     return $nodehash
2475 }
2476
2477 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2478 sub check_running {
2479     my ($vmid, $nocheck, $node) = @_;
2480
2481     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2482     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2483 }
2484
2485 sub vzlist {
2486
2487     my $vzlist = config_list();
2488
2489     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2490
2491     while (defined(my $de = $fd->read)) {
2492         next if $de !~ m/^(\d+)\.pid$/;
2493         my $vmid = $1;
2494         next if !defined($vzlist->{$vmid});
2495         if (my $pid = check_running($vmid)) {
2496             $vzlist->{$vmid}->{pid} = $pid;
2497         }
2498     }
2499
2500     return $vzlist;
2501 }
2502
2503 our $vmstatus_return_properties = {
2504     vmid => get_standard_option('pve-vmid'),
2505     status => {
2506         description => "Qemu process status.",
2507         type => 'string',
2508         enum => ['stopped', 'running'],
2509     },
2510     maxmem => {
2511         description => "Maximum memory in bytes.",
2512         type => 'integer',
2513         optional => 1,
2514         renderer => 'bytes',
2515     },
2516     maxdisk => {
2517         description => "Root disk size in bytes.",
2518         type => 'integer',
2519         optional => 1,
2520         renderer => 'bytes',
2521     },
2522     name => {
2523         description => "VM name.",
2524         type => 'string',
2525         optional => 1,
2526     },
2527     qmpstatus => {
2528         description => "Qemu QMP agent status.",
2529         type => 'string',
2530         optional => 1,
2531     },
2532     pid => {
2533         description => "PID of running qemu process.",
2534         type => 'integer',
2535         optional => 1,
2536     },
2537     uptime => {
2538         description => "Uptime.",
2539         type => 'integer',
2540         optional => 1,
2541         renderer => 'duration',
2542     },
2543     cpus => {
2544         description => "Maximum usable CPUs.",
2545         type => 'number',
2546         optional => 1,
2547     },
2548     lock => {
2549         description => "The current config lock, if any.",
2550         type => 'string',
2551         optional => 1,
2552     },
2553     tags => {
2554         description  => "The current configured tags, if any",
2555         type => 'string',
2556         optional => 1,
2557     },
2558 };
2559
2560 my $last_proc_pid_stat;
2561
2562 # get VM status information
2563 # This must be fast and should not block ($full == false)
2564 # We only query KVM using QMP if $full == true (this can be slow)
2565 sub vmstatus {
2566     my ($opt_vmid, $full) = @_;
2567
2568     my $res = {};
2569
2570     my $storecfg = PVE::Storage::config();
2571
2572     my $list = vzlist();
2573     my $defaults = load_defaults();
2574
2575     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2576
2577     my $cpucount = $cpuinfo->{cpus} || 1;
2578
2579     foreach my $vmid (keys %$list) {
2580         next if $opt_vmid && ($vmid ne $opt_vmid);
2581
2582         my $conf = PVE::QemuConfig->load_config($vmid);
2583
2584         my $d = { vmid => $vmid };
2585         $d->{pid} = $list->{$vmid}->{pid};
2586
2587         # fixme: better status?
2588         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2589
2590         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2591         if (defined($size)) {
2592             $d->{disk} = 0; # no info available
2593             $d->{maxdisk} = $size;
2594         } else {
2595             $d->{disk} = 0;
2596             $d->{maxdisk} = 0;
2597         }
2598
2599         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2600             * ($conf->{cores} || $defaults->{cores});
2601         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2602         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2603
2604         $d->{name} = $conf->{name} || "VM $vmid";
2605         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2606             : $defaults->{memory}*(1024*1024);
2607
2608         if ($conf->{balloon}) {
2609             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2610             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2611                 : $defaults->{shares};
2612         }
2613
2614         $d->{uptime} = 0;
2615         $d->{cpu} = 0;
2616         $d->{mem} = 0;
2617
2618         $d->{netout} = 0;
2619         $d->{netin} = 0;
2620
2621         $d->{diskread} = 0;
2622         $d->{diskwrite} = 0;
2623
2624         $d->{template} = PVE::QemuConfig->is_template($conf);
2625
2626         $d->{serial} = 1 if conf_has_serial($conf);
2627         $d->{lock} = $conf->{lock} if $conf->{lock};
2628         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2629
2630         $res->{$vmid} = $d;
2631     }
2632
2633     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2634     foreach my $dev (keys %$netdev) {
2635         next if $dev !~ m/^tap([1-9]\d*)i/;
2636         my $vmid = $1;
2637         my $d = $res->{$vmid};
2638         next if !$d;
2639
2640         $d->{netout} += $netdev->{$dev}->{receive};
2641         $d->{netin} += $netdev->{$dev}->{transmit};
2642
2643         if ($full) {
2644             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2645             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2646         }
2647
2648     }
2649
2650     my $ctime = gettimeofday;
2651
2652     foreach my $vmid (keys %$list) {
2653
2654         my $d = $res->{$vmid};
2655         my $pid = $d->{pid};
2656         next if !$pid;
2657
2658         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2659         next if !$pstat; # not running
2660
2661         my $used = $pstat->{utime} + $pstat->{stime};
2662
2663         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2664
2665         if ($pstat->{vsize}) {
2666             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2667         }
2668
2669         my $old = $last_proc_pid_stat->{$pid};
2670         if (!$old) {
2671             $last_proc_pid_stat->{$pid} = {
2672                 time => $ctime,
2673                 used => $used,
2674                 cpu => 0,
2675             };
2676             next;
2677         }
2678
2679         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2680
2681         if ($dtime > 1000) {
2682             my $dutime = $used -  $old->{used};
2683
2684             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2685             $last_proc_pid_stat->{$pid} = {
2686                 time => $ctime,
2687                 used => $used,
2688                 cpu => $d->{cpu},
2689             };
2690         } else {
2691             $d->{cpu} = $old->{cpu};
2692         }
2693     }
2694
2695     return $res if !$full;
2696
2697     my $qmpclient = PVE::QMPClient->new();
2698
2699     my $ballooncb = sub {
2700         my ($vmid, $resp) = @_;
2701
2702         my $info = $resp->{'return'};
2703         return if !$info->{max_mem};
2704
2705         my $d = $res->{$vmid};
2706
2707         # use memory assigned to VM
2708         $d->{maxmem} = $info->{max_mem};
2709         $d->{balloon} = $info->{actual};
2710
2711         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2712             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2713             $d->{freemem} = $info->{free_mem};
2714         }
2715
2716         $d->{ballooninfo} = $info;
2717     };
2718
2719     my $blockstatscb = sub {
2720         my ($vmid, $resp) = @_;
2721         my $data = $resp->{'return'} || [];
2722         my $totalrdbytes = 0;
2723         my $totalwrbytes = 0;
2724
2725         for my $blockstat (@$data) {
2726             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2727             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2728
2729             $blockstat->{device} =~ s/drive-//;
2730             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2731         }
2732         $res->{$vmid}->{diskread} = $totalrdbytes;
2733         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2734     };
2735
2736     my $statuscb = sub {
2737         my ($vmid, $resp) = @_;
2738
2739         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2740         # this fails if ballon driver is not loaded, so this must be
2741         # the last commnand (following command are aborted if this fails).
2742         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2743
2744         my $status = 'unknown';
2745         if (!defined($status = $resp->{'return'}->{status})) {
2746             warn "unable to get VM status\n";
2747             return;
2748         }
2749
2750         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2751     };
2752
2753     foreach my $vmid (keys %$list) {
2754         next if $opt_vmid && ($vmid ne $opt_vmid);
2755         next if !$res->{$vmid}->{pid}; # not running
2756         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2757     }
2758
2759     $qmpclient->queue_execute(undef, 2);
2760
2761     foreach my $vmid (keys %$list) {
2762         next if $opt_vmid && ($vmid ne $opt_vmid);
2763         next if !$res->{$vmid}->{pid}; #not running
2764
2765         # we can't use the $qmpclient since it might have already aborted on
2766         # 'query-balloon', but this might also fail for older versions...
2767         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2768         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2769     }
2770
2771     foreach my $vmid (keys %$list) {
2772         next if $opt_vmid && ($vmid ne $opt_vmid);
2773         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2774     }
2775
2776     return $res;
2777 }
2778
2779 sub conf_has_serial {
2780     my ($conf) = @_;
2781
2782     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2783         if ($conf->{"serial$i"}) {
2784             return 1;
2785         }
2786     }
2787
2788     return 0;
2789 }
2790
2791 sub conf_has_audio {
2792     my ($conf, $id) = @_;
2793
2794     $id //= 0;
2795     my $audio = $conf->{"audio$id"};
2796     return if !defined($audio);
2797
2798     my $audioproperties = parse_property_string($audio_fmt, $audio);
2799     my $audiodriver = $audioproperties->{driver} // 'spice';
2800
2801     return {
2802         dev => $audioproperties->{device},
2803         dev_id => "audiodev$id",
2804         backend => $audiodriver,
2805         backend_id => "$audiodriver-backend${id}",
2806     };
2807 }
2808
2809 sub audio_devs {
2810     my ($audio, $audiopciaddr, $machine_version) = @_;
2811
2812     my $devs = [];
2813
2814     my $id = $audio->{dev_id};
2815     my $audiodev = "";
2816     if (min_version($machine_version, 4, 2)) {
2817         $audiodev = ",audiodev=$audio->{backend_id}";
2818     }
2819
2820     if ($audio->{dev} eq 'AC97') {
2821         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2822     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2823         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2824         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2825         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2826     } else {
2827         die "unkown audio device '$audio->{dev}', implement me!";
2828     }
2829
2830     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2831
2832     return $devs;
2833 }
2834
2835 sub vga_conf_has_spice {
2836     my ($vga) = @_;
2837
2838     my $vgaconf = parse_vga($vga);
2839     my $vgatype = $vgaconf->{type};
2840     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2841
2842     return $1 || 1;
2843 }
2844
2845 sub is_native($) {
2846     my ($arch) = @_;
2847     return get_host_arch() eq $arch;
2848 }
2849
2850 sub get_vm_arch {
2851     my ($conf) = @_;
2852     return $conf->{arch} // get_host_arch();
2853 }
2854
2855 my $default_machines = {
2856     x86_64 => 'pc',
2857     aarch64 => 'virt',
2858 };
2859
2860 sub get_vm_machine {
2861     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2862
2863     my $machine = $forcemachine || $conf->{machine};
2864
2865     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2866         $arch //= 'x86_64';
2867         $machine ||= $default_machines->{$arch};
2868         if ($add_pve_version) {
2869             $kvmversion //= kvm_user_version();
2870             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2871             $machine .= "+pve$pvever";
2872         }
2873     }
2874
2875     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2876         # for version-pinned machines that do not include a pve-version (e.g.
2877         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2878         $machine .= '+pve0';
2879     }
2880
2881     return $machine;
2882 }
2883
2884 sub get_ovmf_files($) {
2885     my ($arch) = @_;
2886
2887     my $ovmf = $OVMF->{$arch}
2888         or die "no OVMF images known for architecture '$arch'\n";
2889
2890     return @$ovmf;
2891 }
2892
2893 my $Arch2Qemu = {
2894     aarch64 => '/usr/bin/qemu-system-aarch64',
2895     x86_64 => '/usr/bin/qemu-system-x86_64',
2896 };
2897 sub get_command_for_arch($) {
2898     my ($arch) = @_;
2899     return '/usr/bin/kvm' if is_native($arch);
2900
2901     my $cmd = $Arch2Qemu->{$arch}
2902         or die "don't know how to emulate architecture '$arch'\n";
2903     return $cmd;
2904 }
2905
2906 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2907 # to use in a QEMU command line (-cpu element), first array_intersect the result
2908 # of query_supported_ with query_understood_. This is necessary because:
2909 #
2910 # a) query_understood_ returns flags the host cannot use and
2911 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2912 #    flags, but CPU settings - with most of them being flags. Those settings
2913 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2914 #
2915 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2916 # expensive. If you need the value returned from this, you can get it much
2917 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2918 # $accel being 'kvm' or 'tcg'.
2919 #
2920 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2921 # changes, automatically populating pmxcfs.
2922 #
2923 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2924 # since kvm and tcg machines support different flags
2925 #
2926 sub query_supported_cpu_flags {
2927     my ($arch) = @_;
2928
2929     $arch //= get_host_arch();
2930     my $default_machine = $default_machines->{$arch};
2931
2932     my $flags = {};
2933
2934     # FIXME: Once this is merged, the code below should work for ARM as well:
2935     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2936     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2937         $arch eq "aarch64";
2938
2939     my $kvm_supported = defined(kvm_version());
2940     my $qemu_cmd = get_command_for_arch($arch);
2941     my $fakevmid = -1;
2942     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2943
2944     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2945     my $query_supported_run_qemu = sub {
2946         my ($kvm) = @_;
2947
2948         my $flags = {};
2949         my $cmd = [
2950             $qemu_cmd,
2951             '-machine', $default_machine,
2952             '-display', 'none',
2953             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2954             '-mon', 'chardev=qmp,mode=control',
2955             '-pidfile', $pidfile,
2956             '-S', '-daemonize'
2957         ];
2958
2959         if (!$kvm) {
2960             push @$cmd, '-accel', 'tcg';
2961         }
2962
2963         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2964         die "QEMU flag querying VM exited with code " . $rc if $rc;
2965
2966         eval {
2967             my $cmd_result = mon_cmd(
2968                 $fakevmid,
2969                 'query-cpu-model-expansion',
2970                 type => 'full',
2971                 model => { name => 'host' }
2972             );
2973
2974             my $props = $cmd_result->{model}->{props};
2975             foreach my $prop (keys %$props) {
2976                 next if $props->{$prop} ne '1';
2977                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2978                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2979                 # We only keep those with underscores, to match /proc/cpuinfo
2980                 $prop =~ s/\.|-/_/g;
2981                 $flags->{$prop} = 1;
2982             }
2983         };
2984         my $err = $@;
2985
2986         # force stop with 10 sec timeout and 'nocheck'
2987         # always stop, even if QMP failed
2988         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2989
2990         die $err if $err;
2991
2992         return [ sort keys %$flags ];
2993     };
2994
2995     # We need to query QEMU twice, since KVM and TCG have different supported flags
2996     PVE::QemuConfig->lock_config($fakevmid, sub {
2997         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
2998         warn "warning: failed querying supported tcg flags: $@\n" if $@;
2999
3000         if ($kvm_supported) {
3001             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3002             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3003         }
3004     });
3005
3006     return $flags;
3007 }
3008
3009 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3010 my $understood_cpu_flag_dir = "/usr/share/kvm";
3011 sub query_understood_cpu_flags {
3012     my $arch = get_host_arch();
3013     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3014
3015     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3016         if ! -e $filepath;
3017
3018     my $raw = file_get_contents($filepath);
3019     $raw =~ s/^\s+|\s+$//g;
3020     my @flags = split(/\s+/, $raw);
3021
3022     return \@flags;
3023 }
3024
3025 sub config_to_command {
3026     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3027
3028     my $cmd = [];
3029     my $globalFlags = [];
3030     my $machineFlags = [];
3031     my $rtcFlags = [];
3032     my $devices = [];
3033     my $pciaddr = '';
3034     my $bridges = {};
3035     my $ostype = $conf->{ostype};
3036     my $winversion = windows_version($ostype);
3037     my $kvm = $conf->{kvm};
3038     my $nodename = nodename();
3039
3040     my $arch = get_vm_arch($conf);
3041     my $kvm_binary = get_command_for_arch($arch);
3042     my $kvmver = kvm_user_version($kvm_binary);
3043
3044     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3045         $kvmver //= "undefined";
3046         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3047     }
3048
3049     my $add_pve_version = min_version($kvmver, 4, 1);
3050
3051     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3052     my $machine_version = extract_version($machine_type, $kvmver);
3053     $kvm //= 1 if is_native($arch);
3054
3055     $machine_version =~ m/(\d+)\.(\d+)/;
3056     my ($machine_major, $machine_minor) = ($1, $2);
3057
3058     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3059         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3060     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3061         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3062             ." please upgrade node '$nodename'\n"
3063     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3064         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3065         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3066             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3067             ." node '$nodename'\n";
3068     }
3069
3070     # if a specific +pve version is required for a feature, use $version_guard
3071     # instead of min_version to allow machines to be run with the minimum
3072     # required version
3073     my $required_pve_version = 0;
3074     my $version_guard = sub {
3075         my ($major, $minor, $pve) = @_;
3076         return 0 if !min_version($machine_version, $major, $minor, $pve);
3077         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3078         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3079         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3080         return 1;
3081     };
3082
3083     if ($kvm && !defined kvm_version()) {
3084         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3085             ." or enable in BIOS.\n";
3086     }
3087
3088     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3089     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3090     my $use_old_bios_files = undef;
3091     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3092
3093     my $cpuunits = defined($conf->{cpuunits}) ?
3094             $conf->{cpuunits} : $defaults->{cpuunits};
3095
3096     push @$cmd, $kvm_binary;
3097
3098     push @$cmd, '-id', $vmid;
3099
3100     my $vmname = $conf->{name} || "vm$vmid";
3101
3102     push @$cmd, '-name', $vmname;
3103
3104     push @$cmd, '-no-shutdown';
3105
3106     my $use_virtio = 0;
3107
3108     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3109     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3110     push @$cmd, '-mon', "chardev=qmp,mode=control";
3111
3112     if (min_version($machine_version, 2, 12)) {
3113         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3114         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3115     }
3116
3117     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3118
3119     push @$cmd, '-daemonize';
3120
3121     if ($conf->{smbios1}) {
3122         my $smbios_conf = parse_smbios1($conf->{smbios1});
3123         if ($smbios_conf->{base64}) {
3124             # Do not pass base64 flag to qemu
3125             delete $smbios_conf->{base64};
3126             my $smbios_string = "";
3127             foreach my $key (keys %$smbios_conf) {
3128                 my $value;
3129                 if ($key eq "uuid") {
3130                     $value = $smbios_conf->{uuid}
3131                 } else {
3132                     $value = decode_base64($smbios_conf->{$key});
3133                 }
3134                 # qemu accepts any binary data, only commas need escaping by double comma
3135                 $value =~ s/,/,,/g;
3136                 $smbios_string .= "," . $key . "=" . $value if $value;
3137             }
3138             push @$cmd, '-smbios', "type=1" . $smbios_string;
3139         } else {
3140             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3141         }
3142     }
3143
3144     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3145         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3146         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3147
3148         my ($path, $format);
3149         if (my $efidisk = $conf->{efidisk0}) {
3150             my $d = parse_drive('efidisk0', $efidisk);
3151             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3152             $format = $d->{format};
3153             if ($storeid) {
3154                 $path = PVE::Storage::path($storecfg, $d->{file});
3155                 if (!defined($format)) {
3156                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3157                     $format = qemu_img_format($scfg, $volname);
3158                 }
3159             } else {
3160                 $path = $d->{file};
3161                 die "efidisk format must be specified\n"
3162                     if !defined($format);
3163             }
3164         } else {
3165             warn "no efidisk configured! Using temporary efivars disk.\n";
3166             $path = "/tmp/$vmid-ovmf.fd";
3167             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3168             $format = 'raw';
3169         }
3170
3171         my $size_str = "";
3172
3173         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3174             $size_str = ",size=" . (-s $ovmf_vars);
3175         }
3176
3177         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3178         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3179     }
3180
3181     # load q35 config
3182     if ($q35) {
3183         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3184         if (min_version($machine_version, 4, 0)) {
3185             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3186         } else {
3187             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3188         }
3189     }
3190
3191     if ($conf->{vmgenid}) {
3192         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3193     }
3194
3195     # add usb controllers
3196     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3197         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3198     push @$devices, @usbcontrollers if @usbcontrollers;
3199     my $vga = parse_vga($conf->{vga});
3200
3201     my $qxlnum = vga_conf_has_spice($conf->{vga});
3202     $vga->{type} = 'qxl' if $qxlnum;
3203
3204     if (!$vga->{type}) {
3205         if ($arch eq 'aarch64') {
3206             $vga->{type} = 'virtio';
3207         } elsif (min_version($machine_version, 2, 9)) {
3208             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3209         } else {
3210             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3211         }
3212     }
3213
3214     # enable absolute mouse coordinates (needed by vnc)
3215     my $tablet;
3216     if (defined($conf->{tablet})) {
3217         $tablet = $conf->{tablet};
3218     } else {
3219         $tablet = $defaults->{tablet};
3220         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3221         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3222     }
3223
3224     if ($tablet) {
3225         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3226         my $kbd = print_keyboarddevice_full($conf, $arch);
3227         push @$devices, '-device', $kbd if defined($kbd);
3228     }
3229
3230     my $bootorder = device_bootorder($conf);
3231
3232     # host pci device passthrough
3233     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3234         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3235
3236     # usb devices
3237     my $usb_dev_features = {};
3238     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3239
3240     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3241         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3242     push @$devices, @usbdevices if @usbdevices;
3243
3244     # serial devices
3245     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3246         if (my $path = $conf->{"serial$i"}) {
3247             if ($path eq 'socket') {
3248                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3249                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3250                 # On aarch64, serial0 is the UART device. Qemu only allows
3251                 # connecting UART devices via the '-serial' command line, as
3252                 # the device has a fixed slot on the hardware...
3253                 if ($arch eq 'aarch64' && $i == 0) {
3254                     push @$devices, '-serial', "chardev:serial$i";
3255                 } else {
3256                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3257                 }
3258             } else {
3259                 die "no such serial device\n" if ! -c $path;
3260                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3261                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3262             }
3263         }
3264     }
3265
3266     # parallel devices
3267     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3268         if (my $path = $conf->{"parallel$i"}) {
3269             die "no such parallel device\n" if ! -c $path;
3270             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3271             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3272             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3273         }
3274     }
3275
3276     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3277         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3278         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3279         push @$devices, @$audio_devs;
3280     }
3281
3282     my $sockets = 1;
3283     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3284     $sockets = $conf->{sockets} if  $conf->{sockets};
3285
3286     my $cores = $conf->{cores} || 1;
3287
3288     my $maxcpus = $sockets * $cores;
3289
3290     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3291
3292     my $allowed_vcpus = $cpuinfo->{cpus};
3293
3294     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3295         if ($allowed_vcpus < $maxcpus);
3296
3297     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3298
3299         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3300         for (my $i = 2; $i <= $vcpus; $i++)  {
3301             my $cpustr = print_cpu_device($conf,$i);
3302             push @$cmd, '-device', $cpustr;
3303         }
3304
3305     } else {
3306
3307         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3308     }
3309     push @$cmd, '-nodefaults';
3310
3311     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3312
3313     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3314
3315     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3316
3317     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3318         push @$devices, '-device', print_vga_device(
3319             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3320         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3321         push @$cmd,  '-vnc', "unix:$socket,password";
3322     } else {
3323         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3324         push @$cmd, '-nographic';
3325     }
3326
3327     # time drift fix
3328     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3329     my $useLocaltime = $conf->{localtime};
3330
3331     if ($winversion >= 5) { # windows
3332         $useLocaltime = 1 if !defined($conf->{localtime});
3333
3334         # use time drift fix when acpi is enabled
3335         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3336             $tdf = 1 if !defined($conf->{tdf});
3337         }
3338     }
3339
3340     if ($winversion >= 6) {
3341         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3342         push @$cmd, '-no-hpet';
3343     }
3344
3345     push @$rtcFlags, 'driftfix=slew' if $tdf;
3346
3347     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3348         push @$rtcFlags, "base=$conf->{startdate}";
3349     } elsif ($useLocaltime) {
3350         push @$rtcFlags, 'base=localtime';
3351     }
3352
3353     if ($forcecpu) {
3354         push @$cmd, '-cpu', $forcecpu;
3355     } else {
3356         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3357     }
3358
3359     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3360
3361     push @$cmd, '-S' if $conf->{freeze};
3362
3363     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3364
3365     my $guest_agent = parse_guest_agent($conf);
3366
3367     if ($guest_agent->{enabled}) {
3368         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3369         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3370
3371         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3372             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3373             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3374             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3375         } elsif ($guest_agent->{type} eq 'isa') {
3376             push @$devices, '-device', "isa-serial,chardev=qga0";
3377         }
3378     }
3379
3380     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3381     if ($rng && $version_guard->(4, 1, 2)) {
3382         check_rng_source($rng->{source});
3383
3384         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3385         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3386         my $limiter_str = "";
3387         if ($max_bytes) {
3388             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3389         }
3390
3391         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3392         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3393         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3394     }
3395
3396     my $spice_port;
3397
3398     if ($qxlnum) {
3399         if ($qxlnum > 1) {
3400             if ($winversion){
3401                 for (my $i = 1; $i < $qxlnum; $i++){
3402                     push @$devices, '-device', print_vga_device(
3403                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3404                 }
3405             } else {
3406                 # assume other OS works like Linux
3407                 my ($ram, $vram) = ("134217728", "67108864");
3408                 if ($vga->{memory}) {
3409                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3410                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3411                 }
3412                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3413                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3414             }
3415         }
3416
3417         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3418
3419         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3420         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3421         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3422
3423         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3424         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3425         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3426
3427         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3428         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3429
3430         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3431         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3432         if ($spice_enhancement->{foldersharing}) {
3433             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3434             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3435         }
3436
3437         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3438         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3439             if $spice_enhancement->{videostreaming};
3440
3441         push @$devices, '-spice', "$spice_opts";
3442     }
3443
3444     # enable balloon by default, unless explicitly disabled
3445     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3446         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3447         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3448     }
3449
3450     if ($conf->{watchdog}) {
3451         my $wdopts = parse_watchdog($conf->{watchdog});
3452         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3453         my $watchdog = $wdopts->{model} || 'i6300esb';
3454         push @$devices, '-device', "$watchdog$pciaddr";
3455         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3456     }
3457
3458     my $vollist = [];
3459     my $scsicontroller = {};
3460     my $ahcicontroller = {};
3461     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3462
3463     # Add iscsi initiator name if available
3464     if (my $initiator = get_initiator_name()) {
3465         push @$devices, '-iscsi', "initiator-name=$initiator";
3466     }
3467
3468     PVE::QemuConfig->foreach_volume($conf, sub {
3469         my ($ds, $drive) = @_;
3470
3471         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3472             push @$vollist, $drive->{file};
3473         }
3474
3475         # ignore efidisk here, already added in bios/fw handling code above
3476         return if $drive->{interface} eq 'efidisk';
3477
3478         $use_virtio = 1 if $ds =~ m/^virtio/;
3479
3480         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3481
3482         if ($drive->{interface} eq 'virtio'){
3483            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3484         }
3485
3486         if ($drive->{interface} eq 'scsi') {
3487
3488             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3489
3490             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3491                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3492
3493             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3494             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3495
3496             my $iothread = '';
3497             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3498                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3499                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3500             } elsif ($drive->{iothread}) {
3501                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3502             }
3503
3504             my $queues = '';
3505             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3506                 $queues = ",num_queues=$drive->{queues}";
3507             }
3508
3509             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3510                 if !$scsicontroller->{$controller};
3511             $scsicontroller->{$controller}=1;
3512         }
3513
3514         if ($drive->{interface} eq 'sata') {
3515             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3516             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3517             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3518                 if !$ahcicontroller->{$controller};
3519             $ahcicontroller->{$controller}=1;
3520         }
3521
3522         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3523         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3524
3525         push @$devices, '-drive',$drive_cmd;
3526         push @$devices, '-device', print_drivedevice_full(
3527             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3528     });
3529
3530     for (my $i = 0; $i < $MAX_NETS; $i++) {
3531         my $netname = "net$i";
3532
3533         next if !$conf->{$netname};
3534         my $d = parse_net($conf->{$netname});
3535         next if !$d;
3536
3537         $use_virtio = 1 if $d->{model} eq 'virtio';
3538
3539         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3540
3541         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3542         push @$devices, '-netdev', $netdevfull;
3543
3544         my $netdevicefull = print_netdevice_full(
3545             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3546
3547         push @$devices, '-device', $netdevicefull;
3548     }
3549
3550     if ($conf->{ivshmem}) {
3551         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3552
3553         my $bus;
3554         if ($q35) {
3555             $bus = print_pcie_addr("ivshmem");
3556         } else {
3557             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3558         }
3559
3560         my $ivshmem_name = $ivshmem->{name} // $vmid;
3561         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3562
3563         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3564         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3565             .",size=$ivshmem->{size}M";
3566     }
3567
3568     # pci.4 is nested in pci.1
3569     $bridges->{1} = 1 if $bridges->{4};
3570
3571     if (!$q35) {
3572         # add pci bridges
3573         if (min_version($machine_version, 2, 3)) {
3574            $bridges->{1} = 1;
3575            $bridges->{2} = 1;
3576         }
3577
3578         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3579
3580     }
3581
3582     for my $k (sort {$b cmp $a} keys %$bridges) {
3583         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3584
3585         my $k_name = $k;
3586         if ($k == 2 && $legacy_igd) {
3587             $k_name = "$k-igd";
3588         }
3589         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3590
3591         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3592         if ($q35) {
3593             # add after -readconfig pve-q35.cfg
3594             splice @$devices, 2, 0, '-device', $devstr;
3595         } else {
3596             unshift @$devices, '-device', $devstr if $k > 0;
3597         }
3598     }
3599
3600     if (!$kvm) {
3601         push @$machineFlags, 'accel=tcg';
3602     }
3603
3604     my $machine_type_min = $machine_type;
3605     if ($add_pve_version) {
3606         $machine_type_min =~ s/\+pve\d+$//;
3607         $machine_type_min .= "+pve$required_pve_version";
3608     }
3609     push @$machineFlags, "type=${machine_type_min}";
3610
3611     push @$cmd, @$devices;
3612     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3613     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3614     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3615
3616     if (my $vmstate = $conf->{vmstate}) {
3617         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3618         push @$vollist, $vmstate;
3619         push @$cmd, '-loadstate', $statepath;
3620         print "activating and using '$vmstate' as vmstate\n";
3621     }
3622
3623     # add custom args
3624     if ($conf->{args}) {
3625         my $aa = PVE::Tools::split_args($conf->{args});
3626         push @$cmd, @$aa;
3627     }
3628
3629     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3630 }
3631
3632 sub check_rng_source {
3633     my ($source) = @_;
3634
3635     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3636     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3637         if ! -e $source;
3638
3639     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3640     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3641         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3642         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3643         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3644             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3645             ." to the host.\n";
3646     }
3647 }
3648
3649 sub spice_port {
3650     my ($vmid) = @_;
3651
3652     my $res = mon_cmd($vmid, 'query-spice');
3653
3654     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3655 }
3656
3657 sub vm_devices_list {
3658     my ($vmid) = @_;
3659
3660     my $res = mon_cmd($vmid, 'query-pci');
3661     my $devices_to_check = [];
3662     my $devices = {};
3663     foreach my $pcibus (@$res) {
3664         push @$devices_to_check, @{$pcibus->{devices}},
3665     }
3666
3667     while (@$devices_to_check) {
3668         my $to_check = [];
3669         for my $d (@$devices_to_check) {
3670             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3671             next if !$d->{'pci_bridge'};
3672
3673             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3674             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3675         }
3676         $devices_to_check = $to_check;
3677     }
3678
3679     my $resblock = mon_cmd($vmid, 'query-block');
3680     foreach my $block (@$resblock) {
3681         if($block->{device} =~ m/^drive-(\S+)/){
3682                 $devices->{$1} = 1;
3683         }
3684     }
3685
3686     my $resmice = mon_cmd($vmid, 'query-mice');
3687     foreach my $mice (@$resmice) {
3688         if ($mice->{name} eq 'QEMU HID Tablet') {
3689             $devices->{tablet} = 1;
3690             last;
3691         }
3692     }
3693
3694     # for usb devices there is no query-usb
3695     # but we can iterate over the entries in
3696     # qom-list path=/machine/peripheral
3697     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3698     foreach my $per (@$resperipheral) {
3699         if ($per->{name} =~ m/^usb\d+$/) {
3700             $devices->{$per->{name}} = 1;
3701         }
3702     }
3703
3704     return $devices;
3705 }
3706
3707 sub vm_deviceplug {
3708     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3709
3710     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3711
3712     my $devices_list = vm_devices_list($vmid);
3713     return 1 if defined($devices_list->{$deviceid});
3714
3715     # add PCI bridge if we need it for the device
3716     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3717
3718     if ($deviceid eq 'tablet') {
3719
3720         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3721
3722     } elsif ($deviceid eq 'keyboard') {
3723
3724         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3725
3726     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3727
3728         die "usb hotplug currently not reliable\n";
3729         # since we can't reliably hot unplug all added usb devices and usb
3730         # passthrough breaks live migration we disable usb hotplugging for now
3731         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3732
3733     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3734
3735         qemu_iothread_add($vmid, $deviceid, $device);
3736
3737         qemu_driveadd($storecfg, $vmid, $device);
3738         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3739
3740         qemu_deviceadd($vmid, $devicefull);
3741         eval { qemu_deviceaddverify($vmid, $deviceid); };
3742         if (my $err = $@) {
3743             eval { qemu_drivedel($vmid, $deviceid); };
3744             warn $@ if $@;
3745             die $err;
3746         }
3747
3748     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3749
3750
3751         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3752         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3753         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3754
3755         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3756
3757         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3758             qemu_iothread_add($vmid, $deviceid, $device);
3759             $devicefull .= ",iothread=iothread-$deviceid";
3760         }
3761
3762         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3763             $devicefull .= ",num_queues=$device->{queues}";
3764         }
3765
3766         qemu_deviceadd($vmid, $devicefull);
3767         qemu_deviceaddverify($vmid, $deviceid);
3768
3769     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3770
3771         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3772         qemu_driveadd($storecfg, $vmid, $device);
3773
3774         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3775         eval { qemu_deviceadd($vmid, $devicefull); };
3776         if (my $err = $@) {
3777             eval { qemu_drivedel($vmid, $deviceid); };
3778             warn $@ if $@;
3779             die $err;
3780         }
3781
3782     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3783
3784         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3785
3786         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3787         my $use_old_bios_files = undef;
3788         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3789
3790         my $netdevicefull = print_netdevice_full(
3791             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3792         qemu_deviceadd($vmid, $netdevicefull);
3793         eval {
3794             qemu_deviceaddverify($vmid, $deviceid);
3795             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3796         };
3797         if (my $err = $@) {
3798             eval { qemu_netdevdel($vmid, $deviceid); };
3799             warn $@ if $@;
3800             die $err;
3801         }
3802
3803     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3804
3805         my $bridgeid = $2;
3806         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3807         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3808
3809         qemu_deviceadd($vmid, $devicefull);
3810         qemu_deviceaddverify($vmid, $deviceid);
3811
3812     } else {
3813         die "can't hotplug device '$deviceid'\n";
3814     }
3815
3816     return 1;
3817 }
3818
3819 # fixme: this should raise exceptions on error!
3820 sub vm_deviceunplug {
3821     my ($vmid, $conf, $deviceid) = @_;
3822
3823     my $devices_list = vm_devices_list($vmid);
3824     return 1 if !defined($devices_list->{$deviceid});
3825
3826     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3827     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3828
3829     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3830
3831         qemu_devicedel($vmid, $deviceid);
3832
3833     } elsif ($deviceid =~ m/^usb\d+$/) {
3834
3835         die "usb hotplug currently not reliable\n";
3836         # when unplugging usb devices this way, there may be remaining usb
3837         # controllers/hubs so we disable it for now
3838         #qemu_devicedel($vmid, $deviceid);
3839         #qemu_devicedelverify($vmid, $deviceid);
3840
3841     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3842
3843         qemu_devicedel($vmid, $deviceid);
3844         qemu_devicedelverify($vmid, $deviceid);
3845         qemu_drivedel($vmid, $deviceid);
3846         qemu_iothread_del($conf, $vmid, $deviceid);
3847
3848     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3849
3850         qemu_devicedel($vmid, $deviceid);
3851         qemu_devicedelverify($vmid, $deviceid);
3852         qemu_iothread_del($conf, $vmid, $deviceid);
3853
3854     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3855
3856         qemu_devicedel($vmid, $deviceid);
3857         qemu_drivedel($vmid, $deviceid);
3858         qemu_deletescsihw($conf, $vmid, $deviceid);
3859
3860     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3861
3862         qemu_devicedel($vmid, $deviceid);
3863         qemu_devicedelverify($vmid, $deviceid);
3864         qemu_netdevdel($vmid, $deviceid);
3865
3866     } else {
3867         die "can't unplug device '$deviceid'\n";
3868     }
3869
3870     return 1;
3871 }
3872
3873 sub qemu_deviceadd {
3874     my ($vmid, $devicefull) = @_;
3875
3876     $devicefull = "driver=".$devicefull;
3877     my %options =  split(/[=,]/, $devicefull);
3878
3879     mon_cmd($vmid, "device_add" , %options);
3880 }
3881
3882 sub qemu_devicedel {
3883     my ($vmid, $deviceid) = @_;
3884
3885     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3886 }
3887
3888 sub qemu_iothread_add {
3889     my($vmid, $deviceid, $device) = @_;
3890
3891     if ($device->{iothread}) {
3892         my $iothreads = vm_iothreads_list($vmid);
3893         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3894     }
3895 }
3896
3897 sub qemu_iothread_del {
3898     my($conf, $vmid, $deviceid) = @_;
3899
3900     my $confid = $deviceid;
3901     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3902         $confid = 'scsi' . $1;
3903     }
3904     my $device = parse_drive($confid, $conf->{$confid});
3905     if ($device->{iothread}) {
3906         my $iothreads = vm_iothreads_list($vmid);
3907         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3908     }
3909 }
3910
3911 sub qemu_objectadd {
3912     my($vmid, $objectid, $qomtype) = @_;
3913
3914     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3915
3916     return 1;
3917 }
3918
3919 sub qemu_objectdel {
3920     my($vmid, $objectid) = @_;
3921
3922     mon_cmd($vmid, "object-del", id => $objectid);
3923
3924     return 1;
3925 }
3926
3927 sub qemu_driveadd {
3928     my ($storecfg, $vmid, $device) = @_;
3929
3930     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3931     $drive =~ s/\\/\\\\/g;
3932     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3933
3934     # If the command succeeds qemu prints: "OK"
3935     return 1 if $ret =~ m/OK/s;
3936
3937     die "adding drive failed: $ret\n";
3938 }
3939
3940 sub qemu_drivedel {
3941     my($vmid, $deviceid) = @_;
3942
3943     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3944     $ret =~ s/^\s+//;
3945
3946     return 1 if $ret eq "";
3947
3948     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3949     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3950
3951     die "deleting drive $deviceid failed : $ret\n";
3952 }
3953
3954 sub qemu_deviceaddverify {
3955     my ($vmid, $deviceid) = @_;
3956
3957     for (my $i = 0; $i <= 5; $i++) {
3958          my $devices_list = vm_devices_list($vmid);
3959          return 1 if defined($devices_list->{$deviceid});
3960          sleep 1;
3961     }
3962
3963     die "error on hotplug device '$deviceid'\n";
3964 }
3965
3966
3967 sub qemu_devicedelverify {
3968     my ($vmid, $deviceid) = @_;
3969
3970     # need to verify that the device is correctly removed as device_del
3971     # is async and empty return is not reliable
3972
3973     for (my $i = 0; $i <= 5; $i++) {
3974          my $devices_list = vm_devices_list($vmid);
3975          return 1 if !defined($devices_list->{$deviceid});
3976          sleep 1;
3977     }
3978
3979     die "error on hot-unplugging device '$deviceid'\n";
3980 }
3981
3982 sub qemu_findorcreatescsihw {
3983     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3984
3985     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3986
3987     my $scsihwid="$controller_prefix$controller";
3988     my $devices_list = vm_devices_list($vmid);
3989
3990     if(!defined($devices_list->{$scsihwid})) {
3991         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3992     }
3993
3994     return 1;
3995 }
3996
3997 sub qemu_deletescsihw {
3998     my ($conf, $vmid, $opt) = @_;
3999
4000     my $device = parse_drive($opt, $conf->{$opt});
4001
4002     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4003         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4004         return 1;
4005     }
4006
4007     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4008
4009     my $devices_list = vm_devices_list($vmid);
4010     foreach my $opt (keys %{$devices_list}) {
4011         if (is_valid_drivename($opt)) {
4012             my $drive = parse_drive($opt, $conf->{$opt});
4013             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4014                 return 1;
4015             }
4016         }
4017     }
4018
4019     my $scsihwid="scsihw$controller";
4020
4021     vm_deviceunplug($vmid, $conf, $scsihwid);
4022
4023     return 1;
4024 }
4025
4026 sub qemu_add_pci_bridge {
4027     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4028
4029     my $bridges = {};
4030
4031     my $bridgeid;
4032
4033     print_pci_addr($device, $bridges, $arch, $machine_type);
4034
4035     while (my ($k, $v) = each %$bridges) {
4036         $bridgeid = $k;
4037     }
4038     return 1 if !defined($bridgeid) || $bridgeid < 1;
4039
4040     my $bridge = "pci.$bridgeid";
4041     my $devices_list = vm_devices_list($vmid);
4042
4043     if (!defined($devices_list->{$bridge})) {
4044         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4045     }
4046
4047     return 1;
4048 }
4049
4050 sub qemu_set_link_status {
4051     my ($vmid, $device, $up) = @_;
4052
4053     mon_cmd($vmid, "set_link", name => $device,
4054                up => $up ? JSON::true : JSON::false);
4055 }
4056
4057 sub qemu_netdevadd {
4058     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4059
4060     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4061     my %options =  split(/[=,]/, $netdev);
4062
4063     if (defined(my $vhost = $options{vhost})) {
4064         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4065     }
4066
4067     if (defined(my $queues = $options{queues})) {
4068         $options{queues} = $queues + 0;
4069     }
4070
4071     mon_cmd($vmid, "netdev_add",  %options);
4072     return 1;
4073 }
4074
4075 sub qemu_netdevdel {
4076     my ($vmid, $deviceid) = @_;
4077
4078     mon_cmd($vmid, "netdev_del", id => $deviceid);
4079 }
4080
4081 sub qemu_usb_hotplug {
4082     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4083
4084     return if !$device;
4085
4086     # remove the old one first
4087     vm_deviceunplug($vmid, $conf, $deviceid);
4088
4089     # check if xhci controller is necessary and available
4090     if ($device->{usb3}) {
4091
4092         my $devicelist = vm_devices_list($vmid);
4093
4094         if (!$devicelist->{xhci}) {
4095             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4096             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4097         }
4098     }
4099     my $d = parse_usb_device($device->{host});
4100     $d->{usb3} = $device->{usb3};
4101
4102     # add the new one
4103     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4104 }
4105
4106 sub qemu_cpu_hotplug {
4107     my ($vmid, $conf, $vcpus) = @_;
4108
4109     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4110
4111     my $sockets = 1;
4112     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4113     $sockets = $conf->{sockets} if  $conf->{sockets};
4114     my $cores = $conf->{cores} || 1;
4115     my $maxcpus = $sockets * $cores;
4116
4117     $vcpus = $maxcpus if !$vcpus;
4118
4119     die "you can't add more vcpus than maxcpus\n"
4120         if $vcpus > $maxcpus;
4121
4122     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4123
4124     if ($vcpus < $currentvcpus) {
4125
4126         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4127
4128             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4129                 qemu_devicedel($vmid, "cpu$i");
4130                 my $retry = 0;
4131                 my $currentrunningvcpus = undef;
4132                 while (1) {
4133                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4134                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4135                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4136                     $retry++;
4137                     sleep 1;
4138                 }
4139                 #update conf after each succesfull cpu unplug
4140                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4141                 PVE::QemuConfig->write_config($vmid, $conf);
4142             }
4143         } else {
4144             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4145         }
4146
4147         return;
4148     }
4149
4150     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4151     die "vcpus in running vm does not match its configuration\n"
4152         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4153
4154     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4155
4156         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4157             my $cpustr = print_cpu_device($conf, $i);
4158             qemu_deviceadd($vmid, $cpustr);
4159
4160             my $retry = 0;
4161             my $currentrunningvcpus = undef;
4162             while (1) {
4163                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4164                 last if scalar(@{$currentrunningvcpus}) == $i;
4165                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4166                 sleep 1;
4167                 $retry++;
4168             }
4169             #update conf after each succesfull cpu hotplug
4170             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4171             PVE::QemuConfig->write_config($vmid, $conf);
4172         }
4173     } else {
4174
4175         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4176             mon_cmd($vmid, "cpu-add", id => int($i));
4177         }
4178     }
4179 }
4180
4181 sub qemu_block_set_io_throttle {
4182     my ($vmid, $deviceid,
4183         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4184         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4185         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4186         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4187
4188     return if !check_running($vmid) ;
4189
4190     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4191         bps => int($bps),
4192         bps_rd => int($bps_rd),
4193         bps_wr => int($bps_wr),
4194         iops => int($iops),
4195         iops_rd => int($iops_rd),
4196         iops_wr => int($iops_wr),
4197         bps_max => int($bps_max),
4198         bps_rd_max => int($bps_rd_max),
4199         bps_wr_max => int($bps_wr_max),
4200         iops_max => int($iops_max),
4201         iops_rd_max => int($iops_rd_max),
4202         iops_wr_max => int($iops_wr_max),
4203         bps_max_length => int($bps_max_length),
4204         bps_rd_max_length => int($bps_rd_max_length),
4205         bps_wr_max_length => int($bps_wr_max_length),
4206         iops_max_length => int($iops_max_length),
4207         iops_rd_max_length => int($iops_rd_max_length),
4208         iops_wr_max_length => int($iops_wr_max_length),
4209     );
4210
4211 }
4212
4213 sub qemu_block_resize {
4214     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4215
4216     my $running = check_running($vmid);
4217
4218     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4219
4220     return if !$running;
4221
4222     my $padding = (1024 - $size % 1024) % 1024;
4223     $size = $size + $padding;
4224
4225     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4226
4227 }
4228
4229 sub qemu_volume_snapshot {
4230     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4231
4232     my $running = check_running($vmid);
4233
4234     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4235         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4236     } else {
4237         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4238     }
4239 }
4240
4241 sub qemu_volume_snapshot_delete {
4242     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4243
4244     my $running = check_running($vmid);
4245
4246     if($running) {
4247
4248         $running = undef;
4249         my $conf = PVE::QemuConfig->load_config($vmid);
4250         PVE::QemuConfig->foreach_volume($conf, sub {
4251             my ($ds, $drive) = @_;
4252             $running = 1 if $drive->{file} eq $volid;
4253         });
4254     }
4255
4256     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4257         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4258     } else {
4259         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4260     }
4261 }
4262
4263 sub set_migration_caps {
4264     my ($vmid) = @_;
4265
4266     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4267
4268     my $cap_ref = [];
4269
4270     my $enabled_cap = {
4271         "auto-converge" => 1,
4272         "xbzrle" => 1,
4273         "x-rdma-pin-all" => 0,
4274         "zero-blocks" => 0,
4275         "compress" => 0,
4276         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4277     };
4278
4279     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4280
4281     for my $supported_capability (@$supported_capabilities) {
4282         push @$cap_ref, {
4283             capability => $supported_capability->{capability},
4284             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4285         };
4286     }
4287
4288     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4289 }
4290
4291 sub foreach_volid {
4292     my ($conf, $func, @param) = @_;
4293
4294     my $volhash = {};
4295
4296     my $test_volid = sub {
4297         my ($key, $drive, $snapname) = @_;
4298
4299         my $volid = $drive->{file};
4300         return if !$volid;
4301
4302         $volhash->{$volid}->{cdrom} //= 1;
4303         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4304
4305         my $replicate = $drive->{replicate} // 1;
4306         $volhash->{$volid}->{replicate} //= 0;
4307         $volhash->{$volid}->{replicate} = 1 if $replicate;
4308
4309         $volhash->{$volid}->{shared} //= 0;
4310         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4311
4312         $volhash->{$volid}->{referenced_in_config} //= 0;
4313         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4314
4315         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4316             if defined($snapname);
4317
4318         my $size = $drive->{size};
4319         $volhash->{$volid}->{size} //= $size if $size;
4320
4321         $volhash->{$volid}->{is_vmstate} //= 0;
4322         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4323
4324         $volhash->{$volid}->{is_unused} //= 0;
4325         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4326     };
4327
4328     my $include_opts = {
4329         extra_keys => ['vmstate'],
4330         include_unused => 1,
4331     };
4332
4333     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4334     foreach my $snapname (keys %{$conf->{snapshots}}) {
4335         my $snap = $conf->{snapshots}->{$snapname};
4336         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4337     }
4338
4339     foreach my $volid (keys %$volhash) {
4340         &$func($volid, $volhash->{$volid}, @param);
4341     }
4342 }
4343
4344 my $fast_plug_option = {
4345     'lock' => 1,
4346     'name' => 1,
4347     'onboot' => 1,
4348     'shares' => 1,
4349     'startup' => 1,
4350     'description' => 1,
4351     'protection' => 1,
4352     'vmstatestorage' => 1,
4353     'hookscript' => 1,
4354     'tags' => 1,
4355 };
4356
4357 # hotplug changes in [PENDING]
4358 # $selection hash can be used to only apply specified options, for
4359 # example: { cores => 1 } (only apply changed 'cores')
4360 # $errors ref is used to return error messages
4361 sub vmconfig_hotplug_pending {
4362     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4363
4364     my $defaults = load_defaults();
4365     my $arch = get_vm_arch($conf);
4366     my $machine_type = get_vm_machine($conf, undef, $arch);
4367
4368     # commit values which do not have any impact on running VM first
4369     # Note: those option cannot raise errors, we we do not care about
4370     # $selection and always apply them.
4371
4372     my $add_error = sub {
4373         my ($opt, $msg) = @_;
4374         $errors->{$opt} = "hotplug problem - $msg";
4375     };
4376
4377     my $changes = 0;
4378     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4379         if ($fast_plug_option->{$opt}) {
4380             $conf->{$opt} = $conf->{pending}->{$opt};
4381             delete $conf->{pending}->{$opt};
4382             $changes = 1;
4383         }
4384     }
4385
4386     if ($changes) {
4387         PVE::QemuConfig->write_config($vmid, $conf);
4388     }
4389
4390     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4391
4392     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4393     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4394     foreach my $opt (sort keys %$pending_delete_hash) {
4395         next if $selection && !$selection->{$opt};
4396         my $force = $pending_delete_hash->{$opt}->{force};
4397         eval {
4398             if ($opt eq 'hotplug') {
4399                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4400             } elsif ($opt eq 'tablet') {
4401                 die "skip\n" if !$hotplug_features->{usb};
4402                 if ($defaults->{tablet}) {
4403                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4404                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4405                         if $arch eq 'aarch64';
4406                 } else {
4407                     vm_deviceunplug($vmid, $conf, 'tablet');
4408                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4409                 }
4410             } elsif ($opt =~ m/^usb\d+/) {
4411                 die "skip\n";
4412                 # since we cannot reliably hot unplug usb devices we are disabling it
4413                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4414                 #vm_deviceunplug($vmid, $conf, $opt);
4415             } elsif ($opt eq 'vcpus') {
4416                 die "skip\n" if !$hotplug_features->{cpu};
4417                 qemu_cpu_hotplug($vmid, $conf, undef);
4418             } elsif ($opt eq 'balloon') {
4419                 # enable balloon device is not hotpluggable
4420                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4421                 # here we reset the ballooning value to memory
4422                 my $balloon = $conf->{memory} || $defaults->{memory};
4423                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4424             } elsif ($fast_plug_option->{$opt}) {
4425                 # do nothing
4426             } elsif ($opt =~ m/^net(\d+)$/) {
4427                 die "skip\n" if !$hotplug_features->{network};
4428                 vm_deviceunplug($vmid, $conf, $opt);
4429             } elsif (is_valid_drivename($opt)) {
4430                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4431                 vm_deviceunplug($vmid, $conf, $opt);
4432                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4433             } elsif ($opt =~ m/^memory$/) {
4434                 die "skip\n" if !$hotplug_features->{memory};
4435                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4436             } elsif ($opt eq 'cpuunits') {
4437                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4438             } elsif ($opt eq 'cpulimit') {
4439                 $cgroup->change_cpu_quota(-1, 100000);
4440             } else {
4441                 die "skip\n";
4442             }
4443         };
4444         if (my $err = $@) {
4445             &$add_error($opt, $err) if $err ne "skip\n";
4446         } else {
4447             delete $conf->{$opt};
4448             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4449         }
4450     }
4451
4452     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4453     $apply_pending_cloudinit = sub {
4454         return if $apply_pending_cloudinit_done; # once is enough
4455         $apply_pending_cloudinit_done = 1; # once is enough
4456
4457         my ($key, $value) = @_;
4458
4459         my @cloudinit_opts = keys %$confdesc_cloudinit;
4460         foreach my $opt (keys %{$conf->{pending}}) {
4461             next if !grep { $_ eq $opt } @cloudinit_opts;
4462             $conf->{$opt} = delete $conf->{pending}->{$opt};
4463         }
4464
4465         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4466         foreach my $opt (sort keys %$pending_delete_hash) {
4467             next if !grep { $_ eq $opt } @cloudinit_opts;
4468             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4469             delete $conf->{$opt};
4470         }
4471
4472         my $new_conf = { %$conf };
4473         $new_conf->{$key} = $value;
4474         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4475     };
4476
4477     foreach my $opt (keys %{$conf->{pending}}) {
4478         next if $selection && !$selection->{$opt};
4479         my $value = $conf->{pending}->{$opt};
4480         eval {
4481             if ($opt eq 'hotplug') {
4482                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4483             } elsif ($opt eq 'tablet') {
4484                 die "skip\n" if !$hotplug_features->{usb};
4485                 if ($value == 1) {
4486                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4487                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4488                         if $arch eq 'aarch64';
4489                 } elsif ($value == 0) {
4490                     vm_deviceunplug($vmid, $conf, 'tablet');
4491                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4492                 }
4493             } elsif ($opt =~ m/^usb\d+$/) {
4494                 die "skip\n";
4495                 # since we cannot reliably hot unplug usb devices we disable it for now
4496                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4497                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4498                 #die "skip\n" if !$d;
4499                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4500             } elsif ($opt eq 'vcpus') {
4501                 die "skip\n" if !$hotplug_features->{cpu};
4502                 qemu_cpu_hotplug($vmid, $conf, $value);
4503             } elsif ($opt eq 'balloon') {
4504                 # enable/disable balloning device is not hotpluggable
4505                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4506                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4507                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4508
4509                 # allow manual ballooning if shares is set to zero
4510                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4511                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4512                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4513                 }
4514             } elsif ($opt =~ m/^net(\d+)$/) {
4515                 # some changes can be done without hotplug
4516                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4517                                     $vmid, $opt, $value, $arch, $machine_type);
4518             } elsif (is_valid_drivename($opt)) {
4519                 die "skip\n" if $opt eq 'efidisk0';
4520                 # some changes can be done without hotplug
4521                 my $drive = parse_drive($opt, $value);
4522                 if (drive_is_cloudinit($drive)) {
4523                     &$apply_pending_cloudinit($opt, $value);
4524                 }
4525                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4526                                      $vmid, $opt, $value, $arch, $machine_type);
4527             } elsif ($opt =~ m/^memory$/) { #dimms
4528                 die "skip\n" if !$hotplug_features->{memory};
4529                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4530             } elsif ($opt eq 'cpuunits') {
4531                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4532             } elsif ($opt eq 'cpulimit') {
4533                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4534                 $cgroup->change_cpu_quota($cpulimit, 100000);
4535             } else {
4536                 die "skip\n";  # skip non-hot-pluggable options
4537             }
4538         };
4539         if (my $err = $@) {
4540             &$add_error($opt, $err) if $err ne "skip\n";
4541         } else {
4542             $conf->{$opt} = $value;
4543             delete $conf->{pending}->{$opt};
4544         }
4545     }
4546
4547     PVE::QemuConfig->write_config($vmid, $conf);
4548 }
4549
4550 sub try_deallocate_drive {
4551     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4552
4553     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4554         my $volid = $drive->{file};
4555         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4556             my $sid = PVE::Storage::parse_volume_id($volid);
4557             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4558
4559             # check if the disk is really unused
4560             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4561                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4562             PVE::Storage::vdisk_free($storecfg, $volid);
4563             return 1;
4564         } else {
4565             # If vm is not owner of this disk remove from config
4566             return 1;
4567         }
4568     }
4569
4570     return;
4571 }
4572
4573 sub vmconfig_delete_or_detach_drive {
4574     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4575
4576     my $drive = parse_drive($opt, $conf->{$opt});
4577
4578     my $rpcenv = PVE::RPCEnvironment::get();
4579     my $authuser = $rpcenv->get_user();
4580
4581     if ($force) {
4582         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4583         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4584     } else {
4585         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4586     }
4587 }
4588
4589
4590
4591 sub vmconfig_apply_pending {
4592     my ($vmid, $conf, $storecfg, $errors) = @_;
4593
4594     my $add_apply_error = sub {
4595         my ($opt, $msg) = @_;
4596         my $err_msg = "unable to apply pending change $opt : $msg";
4597         $errors->{$opt} = $err_msg;
4598         warn $err_msg;
4599     };
4600
4601     # cold plug
4602
4603     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4604     foreach my $opt (sort keys %$pending_delete_hash) {
4605         my $force = $pending_delete_hash->{$opt}->{force};
4606         eval {
4607             if ($opt =~ m/^unused/) {
4608                 die "internal error";
4609             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4610                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4611             }
4612         };
4613         if (my $err = $@) {
4614             $add_apply_error->($opt, $err);
4615         } else {
4616             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4617             delete $conf->{$opt};
4618         }
4619     }
4620
4621     PVE::QemuConfig->cleanup_pending($conf);
4622
4623     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4624         next if $opt eq 'delete'; # just to be sure
4625         eval {
4626             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4627                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4628             }
4629         };
4630         if (my $err = $@) {
4631             $add_apply_error->($opt, $err);
4632         } else {
4633             $conf->{$opt} = delete $conf->{pending}->{$opt};
4634         }
4635     }
4636
4637     # write all changes at once to avoid unnecessary i/o
4638     PVE::QemuConfig->write_config($vmid, $conf);
4639 }
4640
4641 sub vmconfig_update_net {
4642     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4643
4644     my $newnet = parse_net($value);
4645
4646     if ($conf->{$opt}) {
4647         my $oldnet = parse_net($conf->{$opt});
4648
4649         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4650             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4651             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4652             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4653
4654             # for non online change, we try to hot-unplug
4655             die "skip\n" if !$hotplug;
4656             vm_deviceunplug($vmid, $conf, $opt);
4657         } else {
4658
4659             die "internal error" if $opt !~ m/net(\d+)/;
4660             my $iface = "tap${vmid}i$1";
4661
4662             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4663                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4664                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4665                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4666                 PVE::Network::tap_unplug($iface);
4667
4668                 if ($have_sdn) {
4669                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4670                 } else {
4671                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4672                 }
4673             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4674                 # Rate can be applied on its own but any change above needs to
4675                 # include the rate in tap_plug since OVS resets everything.
4676                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4677             }
4678
4679             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4680                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4681             }
4682
4683             return 1;
4684         }
4685     }
4686
4687     if ($hotplug) {
4688         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4689     } else {
4690         die "skip\n";
4691     }
4692 }
4693
4694 sub vmconfig_update_disk {
4695     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4696
4697     my $drive = parse_drive($opt, $value);
4698
4699     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4700         my $media = $drive->{media} || 'disk';
4701         my $oldmedia = $old_drive->{media} || 'disk';
4702         die "unable to change media type\n" if $media ne $oldmedia;
4703
4704         if (!drive_is_cdrom($old_drive)) {
4705
4706             if ($drive->{file} ne $old_drive->{file}) {
4707
4708                 die "skip\n" if !$hotplug;
4709
4710                 # unplug and register as unused
4711                 vm_deviceunplug($vmid, $conf, $opt);
4712                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4713
4714             } else {
4715                 # update existing disk
4716
4717                 # skip non hotpluggable value
4718                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4719                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4720                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4721                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4722                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4723                     die "skip\n";
4724                 }
4725
4726                 # apply throttle
4727                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4728                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4729                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4730                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4731                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4732                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4733                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4734                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4735                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4736                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4737                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4738                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4739                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4740                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4741                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4742                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4743                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4744                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4745
4746                     qemu_block_set_io_throttle(
4747                         $vmid,"drive-$opt",
4748                         ($drive->{mbps} || 0)*1024*1024,
4749                         ($drive->{mbps_rd} || 0)*1024*1024,
4750                         ($drive->{mbps_wr} || 0)*1024*1024,
4751                          $drive->{iops} || 0,
4752                          $drive->{iops_rd} || 0,
4753                          $drive->{iops_wr} || 0,
4754                         ($drive->{mbps_max} || 0)*1024*1024,
4755                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4756                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4757                          $drive->{iops_max} || 0,
4758                          $drive->{iops_rd_max} || 0,
4759                          $drive->{iops_wr_max} || 0,
4760                          $drive->{bps_max_length} || 1,
4761                          $drive->{bps_rd_max_length} || 1,
4762                          $drive->{bps_wr_max_length} || 1,
4763                          $drive->{iops_max_length} || 1,
4764                          $drive->{iops_rd_max_length} || 1,
4765                          $drive->{iops_wr_max_length} || 1,
4766                     );
4767
4768                 }
4769
4770                 return 1;
4771             }
4772
4773         } else { # cdrom
4774
4775             if ($drive->{file} eq 'none') {
4776                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4777                 if (drive_is_cloudinit($old_drive)) {
4778                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4779                 }
4780             } else {
4781                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4782
4783                 # force eject if locked
4784                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4785
4786                 if ($path) {
4787                     mon_cmd($vmid, "blockdev-change-medium",
4788                         id => "$opt", filename => "$path");
4789                 }
4790             }
4791
4792             return 1;
4793         }
4794     }
4795
4796     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4797     # hotplug new disks
4798     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4799     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4800 }
4801
4802 # called in locked context by incoming migration
4803 sub vm_migrate_get_nbd_disks {
4804     my ($storecfg, $conf, $replicated_volumes) = @_;
4805
4806     my $local_volumes = {};
4807     PVE::QemuConfig->foreach_volume($conf, sub {
4808         my ($ds, $drive) = @_;
4809
4810         return if drive_is_cdrom($drive);
4811
4812         my $volid = $drive->{file};
4813
4814         return if !$volid;
4815
4816         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4817
4818         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4819         return if $scfg->{shared};
4820
4821         # replicated disks re-use existing state via bitmap
4822         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4823         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4824     });
4825     return $local_volumes;
4826 }
4827
4828 # called in locked context by incoming migration
4829 sub vm_migrate_alloc_nbd_disks {
4830     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4831
4832     my $format = undef;
4833
4834     my $nbd = {};
4835     foreach my $opt (sort keys %$source_volumes) {
4836         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4837
4838         if ($use_existing) {
4839             $nbd->{$opt}->{drivestr} = print_drive($drive);
4840             $nbd->{$opt}->{volid} = $volid;
4841             $nbd->{$opt}->{replicated} = 1;
4842             next;
4843         }
4844
4845         # If a remote storage is specified and the format of the original
4846         # volume is not available there, fall back to the default format.
4847         # Otherwise use the same format as the original.
4848         if (!$storagemap->{identity}) {
4849             $storeid = map_storage($storagemap, $storeid);
4850             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4851             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4852             my $fileFormat = qemu_img_format($scfg, $volname);
4853             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4854         } else {
4855             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4856             $format = qemu_img_format($scfg, $volname);
4857         }
4858
4859         my $size = $drive->{size} / 1024;
4860         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4861         my $newdrive = $drive;
4862         $newdrive->{format} = $format;
4863         $newdrive->{file} = $newvolid;
4864         my $drivestr = print_drive($newdrive);
4865         $nbd->{$opt}->{drivestr} = $drivestr;
4866         $nbd->{$opt}->{volid} = $newvolid;
4867     }
4868
4869     return $nbd;
4870 }
4871
4872 # see vm_start_nolock for parameters, additionally:
4873 # migrate_opts:
4874 #   storagemap = parsed storage map for allocating NBD disks
4875 sub vm_start {
4876     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4877
4878     return PVE::QemuConfig->lock_config($vmid, sub {
4879         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4880
4881         die "you can't start a vm if it's a template\n"
4882             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4883
4884         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4885         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4886
4887         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4888
4889         if ($has_backup_lock && $running) {
4890             # a backup is currently running, attempt to start the guest in the
4891             # existing QEMU instance
4892             return vm_resume($vmid);
4893         }
4894
4895         PVE::QemuConfig->check_lock($conf)
4896             if !($params->{skiplock} || $has_suspended_lock);
4897
4898         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4899
4900         die "VM $vmid already running\n" if $running;
4901
4902         if (my $storagemap = $migrate_opts->{storagemap}) {
4903             my $replicated = $migrate_opts->{replicated_volumes};
4904             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4905             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4906
4907             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4908                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4909             }
4910         }
4911
4912         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4913     });
4914 }
4915
4916
4917 # params:
4918 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4919 #   skiplock => 0/1, skip checking for config lock
4920 #   skiptemplate => 0/1, skip checking whether VM is template
4921 #   forcemachine => to force Qemu machine (rollback/migration)
4922 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4923 #   timeout => in seconds
4924 #   paused => start VM in paused state (backup)
4925 #   resume => resume from hibernation
4926 # migrate_opts:
4927 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4928 #   migratedfrom => source node
4929 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4930 #   network => CIDR of migration network
4931 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4932 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4933 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4934 sub vm_start_nolock {
4935     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4936
4937     my $statefile = $params->{statefile};
4938     my $resume = $params->{resume};
4939
4940     my $migratedfrom = $migrate_opts->{migratedfrom};
4941     my $migration_type = $migrate_opts->{type};
4942
4943     my $res = {};
4944
4945     # clean up leftover reboot request files
4946     eval { clear_reboot_request($vmid); };
4947     warn $@ if $@;
4948
4949     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4950         vmconfig_apply_pending($vmid, $conf, $storecfg);
4951         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4952     }
4953
4954     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4955
4956     my $defaults = load_defaults();
4957
4958     # set environment variable useful inside network script
4959     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4960
4961     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4962
4963     my $forcemachine = $params->{forcemachine};
4964     my $forcecpu = $params->{forcecpu};
4965     if ($resume) {
4966         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4967         $forcemachine = $conf->{runningmachine};
4968         $forcecpu = $conf->{runningcpu};
4969         print "Resuming suspended VM\n";
4970     }
4971
4972     my ($cmd, $vollist, $spice_port) =
4973         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4974
4975     my $migration_ip;
4976     my $get_migration_ip = sub {
4977         my ($nodename) = @_;
4978
4979         return $migration_ip if defined($migration_ip);
4980
4981         my $cidr = $migrate_opts->{network};
4982
4983         if (!defined($cidr)) {
4984             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4985             $cidr = $dc_conf->{migration}->{network};
4986         }
4987
4988         if (defined($cidr)) {
4989             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4990
4991             die "could not get IP: no address configured on local " .
4992                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4993
4994             die "could not get IP: multiple addresses configured on local " .
4995                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4996
4997             $migration_ip = @$ips[0];
4998         }
4999
5000         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5001             if !defined($migration_ip);
5002
5003         return $migration_ip;
5004     };
5005
5006     my $migrate_uri;
5007     if ($statefile) {
5008         if ($statefile eq 'tcp') {
5009             my $localip = "localhost";
5010             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5011             my $nodename = nodename();
5012
5013             if (!defined($migration_type)) {
5014                 if (defined($datacenterconf->{migration}->{type})) {
5015                     $migration_type = $datacenterconf->{migration}->{type};
5016                 } else {
5017                     $migration_type = 'secure';
5018                 }
5019             }
5020
5021             if ($migration_type eq 'insecure') {
5022                 $localip = $get_migration_ip->($nodename);
5023                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5024             }
5025
5026             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5027             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5028             $migrate_uri = "tcp:${localip}:${migrate_port}";
5029             push @$cmd, '-incoming', $migrate_uri;
5030             push @$cmd, '-S';
5031
5032         } elsif ($statefile eq 'unix') {
5033             # should be default for secure migrations as a ssh TCP forward
5034             # tunnel is not deterministic reliable ready and fails regurarly
5035             # to set up in time, so use UNIX socket forwards
5036             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5037             unlink $socket_addr;
5038
5039             $migrate_uri = "unix:$socket_addr";
5040
5041             push @$cmd, '-incoming', $migrate_uri;
5042             push @$cmd, '-S';
5043
5044         } elsif (-e $statefile) {
5045             push @$cmd, '-loadstate', $statefile;
5046         } else {
5047             my $statepath = PVE::Storage::path($storecfg, $statefile);
5048             push @$vollist, $statefile;
5049             push @$cmd, '-loadstate', $statepath;
5050         }
5051     } elsif ($params->{paused}) {
5052         push @$cmd, '-S';
5053     }
5054
5055     # host pci devices
5056     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5057       my $d = parse_hostpci($conf->{"hostpci$i"});
5058       next if !$d;
5059       my $pcidevices = $d->{pciid};
5060       foreach my $pcidevice (@$pcidevices) {
5061             my $pciid = $pcidevice->{id};
5062
5063             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5064             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5065             die "no pci device info for device '$pciid'\n" if !$info;
5066
5067             if ($d->{mdev}) {
5068                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5069                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5070             } else {
5071                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5072                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5073                 die "can't reset PCI device '$pciid'\n"
5074                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5075             }
5076       }
5077     }
5078
5079     PVE::Storage::activate_volumes($storecfg, $vollist);
5080
5081     eval {
5082         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5083             outfunc => sub {}, errfunc => sub {});
5084     };
5085     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5086     # timeout should be more than enough here...
5087     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5088
5089     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5090                                               : $defaults->{cpuunits};
5091
5092     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5093     my %run_params = (
5094         timeout => $statefile ? undef : $start_timeout,
5095         umask => 0077,
5096         noerr => 1,
5097     );
5098
5099     # when migrating, prefix QEMU output so other side can pick up any
5100     # errors that might occur and show the user
5101     if ($migratedfrom) {
5102         $run_params{quiet} = 1;
5103         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5104     }
5105
5106     my %properties = (
5107         Slice => 'qemu.slice',
5108         KillMode => 'none'
5109     );
5110
5111     if (PVE::CGroup::cgroup_mode() == 2) {
5112         $properties{CPUWeight} = $cpuunits;
5113     } else {
5114         $properties{CPUShares} = $cpuunits;
5115     }
5116
5117     if (my $cpulimit = $conf->{cpulimit}) {
5118         $properties{CPUQuota} = int($cpulimit * 100);
5119     }
5120     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5121
5122     my $run_qemu = sub {
5123         PVE::Tools::run_fork sub {
5124             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5125
5126             my $exitcode = run_command($cmd, %run_params);
5127             die "QEMU exited with code $exitcode\n" if $exitcode;
5128         };
5129     };
5130
5131     if ($conf->{hugepages}) {
5132
5133         my $code = sub {
5134             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5135             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5136
5137             PVE::QemuServer::Memory::hugepages_mount();
5138             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5139
5140             eval { $run_qemu->() };
5141             if (my $err = $@) {
5142                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5143                     if !$conf->{keephugepages};
5144                 die $err;
5145             }
5146
5147             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5148                 if !$conf->{keephugepages};
5149         };
5150         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5151
5152     } else {
5153         eval { $run_qemu->() };
5154     }
5155
5156     if (my $err = $@) {
5157         # deactivate volumes if start fails
5158         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5159         die "start failed: $err";
5160     }
5161
5162     print "migration listens on $migrate_uri\n" if $migrate_uri;
5163     $res->{migrate_uri} = $migrate_uri;
5164
5165     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5166         eval { mon_cmd($vmid, "cont"); };
5167         warn $@ if $@;
5168     }
5169
5170     #start nbd server for storage migration
5171     if (my $nbd = $migrate_opts->{nbd}) {
5172         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5173
5174         my $migrate_storage_uri;
5175         # nbd_protocol_version > 0 for unix socket support
5176         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5177             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5178             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5179             $migrate_storage_uri = "nbd:unix:$socket_path";
5180         } else {
5181             my $nodename = nodename();
5182             my $localip = $get_migration_ip->($nodename);
5183             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5184             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5185
5186             mon_cmd($vmid, "nbd-server-start", addr => {
5187                 type => 'inet',
5188                 data => {
5189                     host => "${localip}",
5190                     port => "${storage_migrate_port}",
5191                 },
5192             });
5193             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5194             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5195         }
5196
5197         $res->{migrate_storage_uri} = $migrate_storage_uri;
5198
5199         foreach my $opt (sort keys %$nbd) {
5200             my $drivestr = $nbd->{$opt}->{drivestr};
5201             my $volid = $nbd->{$opt}->{volid};
5202             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5203             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5204             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5205             print "re-using replicated volume: $opt - $volid\n"
5206                 if $nbd->{$opt}->{replicated};
5207
5208             $res->{drives}->{$opt} = $nbd->{$opt};
5209             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5210         }
5211     }
5212
5213     if ($migratedfrom) {
5214         eval {
5215             set_migration_caps($vmid);
5216         };
5217         warn $@ if $@;
5218
5219         if ($spice_port) {
5220             print "spice listens on port $spice_port\n";
5221             $res->{spice_port} = $spice_port;
5222             if ($migrate_opts->{spice_ticket}) {
5223                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5224                     $migrate_opts->{spice_ticket});
5225                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5226             }
5227         }
5228
5229     } else {
5230         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5231             if !$statefile && $conf->{balloon};
5232
5233         foreach my $opt (keys %$conf) {
5234             next if $opt !~  m/^net\d+$/;
5235             my $nicconf = parse_net($conf->{$opt});
5236             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5237         }
5238     }
5239
5240     mon_cmd($vmid, 'qom-set',
5241                 path => "machine/peripheral/balloon0",
5242                 property => "guest-stats-polling-interval",
5243                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5244
5245     if ($resume) {
5246         print "Resumed VM, removing state\n";
5247         if (my $vmstate = $conf->{vmstate}) {
5248             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5249             PVE::Storage::vdisk_free($storecfg, $vmstate);
5250         }
5251         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5252         PVE::QemuConfig->write_config($vmid, $conf);
5253     }
5254
5255     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5256
5257     return $res;
5258 }
5259
5260 sub vm_commandline {
5261     my ($storecfg, $vmid, $snapname) = @_;
5262
5263     my $conf = PVE::QemuConfig->load_config($vmid);
5264     my $forcemachine;
5265     my $forcecpu;
5266
5267     if ($snapname) {
5268         my $snapshot = $conf->{snapshots}->{$snapname};
5269         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5270
5271         # check for machine or CPU overrides in snapshot
5272         $forcemachine = $snapshot->{runningmachine};
5273         $forcecpu = $snapshot->{runningcpu};
5274
5275         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5276
5277         $conf = $snapshot;
5278     }
5279
5280     my $defaults = load_defaults();
5281
5282     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5283         $forcemachine, $forcecpu);
5284
5285     return PVE::Tools::cmd2string($cmd);
5286 }
5287
5288 sub vm_reset {
5289     my ($vmid, $skiplock) = @_;
5290
5291     PVE::QemuConfig->lock_config($vmid, sub {
5292
5293         my $conf = PVE::QemuConfig->load_config($vmid);
5294
5295         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5296
5297         mon_cmd($vmid, "system_reset");
5298     });
5299 }
5300
5301 sub get_vm_volumes {
5302     my ($conf) = @_;
5303
5304     my $vollist = [];
5305     foreach_volid($conf, sub {
5306         my ($volid, $attr) = @_;
5307
5308         return if $volid =~ m|^/|;
5309
5310         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5311         return if !$sid;
5312
5313         push @$vollist, $volid;
5314     });
5315
5316     return $vollist;
5317 }
5318
5319 sub vm_stop_cleanup {
5320     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5321
5322     eval {
5323
5324         if (!$keepActive) {
5325             my $vollist = get_vm_volumes($conf);
5326             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5327         }
5328
5329         foreach my $ext (qw(mon qmp pid vnc qga)) {
5330             unlink "/var/run/qemu-server/${vmid}.$ext";
5331         }
5332
5333         if ($conf->{ivshmem}) {
5334             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5335             # just delete it for now, VMs which have this already open do not
5336             # are affected, but new VMs will get a separated one. If this
5337             # becomes an issue we either add some sort of ref-counting or just
5338             # add a "don't delete on stop" flag to the ivshmem format.
5339             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5340         }
5341
5342         foreach my $key (keys %$conf) {
5343             next if $key !~ m/^hostpci(\d+)$/;
5344             my $hostpciindex = $1;
5345             my $d = parse_hostpci($conf->{$key});
5346             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5347
5348             foreach my $pci (@{$d->{pciid}}) {
5349                 my $pciid = $pci->{id};
5350                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5351             }
5352         }
5353
5354         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5355     };
5356     warn $@ if $@; # avoid errors - just warn
5357 }
5358
5359 # call only in locked context
5360 sub _do_vm_stop {
5361     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5362
5363     my $pid = check_running($vmid, $nocheck);
5364     return if !$pid;
5365
5366     my $conf;
5367     if (!$nocheck) {
5368         $conf = PVE::QemuConfig->load_config($vmid);
5369         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5370         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5371             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5372             $timeout = $opts->{down} if $opts->{down};
5373         }
5374         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5375     }
5376
5377     eval {
5378         if ($shutdown) {
5379             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5380                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5381             } else {
5382                 mon_cmd($vmid, "system_powerdown");
5383             }
5384         } else {
5385             mon_cmd($vmid, "quit");
5386         }
5387     };
5388     my $err = $@;
5389
5390     if (!$err) {
5391         $timeout = 60 if !defined($timeout);
5392
5393         my $count = 0;
5394         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5395             $count++;
5396             sleep 1;
5397         }
5398
5399         if ($count >= $timeout) {
5400             if ($force) {
5401                 warn "VM still running - terminating now with SIGTERM\n";
5402                 kill 15, $pid;
5403             } else {
5404                 die "VM quit/powerdown failed - got timeout\n";
5405             }
5406         } else {
5407             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5408             return;
5409         }
5410     } else {
5411        if (!check_running($vmid, $nocheck)) {
5412             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5413             return;
5414        }
5415        if ($force) {
5416             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5417             kill 15, $pid;
5418         } else {
5419             die "VM quit/powerdown failed\n";
5420         }
5421     }
5422
5423     # wait again
5424     $timeout = 10;
5425
5426     my $count = 0;
5427     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5428         $count++;
5429         sleep 1;
5430     }
5431
5432     if ($count >= $timeout) {
5433         warn "VM still running - terminating now with SIGKILL\n";
5434         kill 9, $pid;
5435         sleep 1;
5436     }
5437
5438     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5439 }
5440
5441 # Note: use $nocheck to skip tests if VM configuration file exists.
5442 # We need that when migration VMs to other nodes (files already moved)
5443 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5444 sub vm_stop {
5445     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5446
5447     $force = 1 if !defined($force) && !$shutdown;
5448
5449     if ($migratedfrom){
5450         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5451         kill 15, $pid if $pid;
5452         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5453         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5454         return;
5455     }
5456
5457     PVE::QemuConfig->lock_config($vmid, sub {
5458         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5459    });
5460 }
5461
5462 sub vm_reboot {
5463     my ($vmid, $timeout) = @_;
5464
5465     PVE::QemuConfig->lock_config($vmid, sub {
5466         eval {
5467
5468             # only reboot if running, as qmeventd starts it again on a stop event
5469             return if !check_running($vmid);
5470
5471             create_reboot_request($vmid);
5472
5473             my $storecfg = PVE::Storage::config();
5474             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5475
5476         };
5477         if (my $err = $@) {
5478             # avoid that the next normal shutdown will be confused for a reboot
5479             clear_reboot_request($vmid);
5480             die $err;
5481         }
5482    });
5483 }
5484
5485 # note: if using the statestorage parameter, the caller has to check privileges
5486 sub vm_suspend {
5487     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5488
5489     my $conf;
5490     my $path;
5491     my $storecfg;
5492     my $vmstate;
5493
5494     PVE::QemuConfig->lock_config($vmid, sub {
5495
5496         $conf = PVE::QemuConfig->load_config($vmid);
5497
5498         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5499         PVE::QemuConfig->check_lock($conf)
5500             if !($skiplock || $is_backing_up);
5501
5502         die "cannot suspend to disk during backup\n"
5503             if $is_backing_up && $includestate;
5504
5505         if ($includestate) {
5506             $conf->{lock} = 'suspending';
5507             my $date = strftime("%Y-%m-%d", localtime(time()));
5508             $storecfg = PVE::Storage::config();
5509             if (!$statestorage) {
5510                 $statestorage = find_vmstate_storage($conf, $storecfg);
5511                 # check permissions for the storage
5512                 my $rpcenv = PVE::RPCEnvironment::get();
5513                 if ($rpcenv->{type} ne 'cli') {
5514                     my $authuser = $rpcenv->get_user();
5515                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5516                 }
5517             }
5518
5519
5520             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5521                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5522             $path = PVE::Storage::path($storecfg, $vmstate);
5523             PVE::QemuConfig->write_config($vmid, $conf);
5524         } else {
5525             mon_cmd($vmid, "stop");
5526         }
5527     });
5528
5529     if ($includestate) {
5530         # save vm state
5531         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5532
5533         eval {
5534             mon_cmd($vmid, "savevm-start", statefile => $path);
5535             for(;;) {
5536                 my $state = mon_cmd($vmid, "query-savevm");
5537                 if (!$state->{status}) {
5538                     die "savevm not active\n";
5539                 } elsif ($state->{status} eq 'active') {
5540                     sleep(1);
5541                     next;
5542                 } elsif ($state->{status} eq 'completed') {
5543                     print "State saved, quitting\n";
5544                     last;
5545                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5546                     die "query-savevm failed with error '$state->{error}'\n"
5547                 } else {
5548                     die "query-savevm returned status '$state->{status}'\n";
5549                 }
5550             }
5551         };
5552         my $err = $@;
5553
5554         PVE::QemuConfig->lock_config($vmid, sub {
5555             $conf = PVE::QemuConfig->load_config($vmid);
5556             if ($err) {
5557                 # cleanup, but leave suspending lock, to indicate something went wrong
5558                 eval {
5559                     mon_cmd($vmid, "savevm-end");
5560                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5561                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5562                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5563                     PVE::QemuConfig->write_config($vmid, $conf);
5564                 };
5565                 warn $@ if $@;
5566                 die $err;
5567             }
5568
5569             die "lock changed unexpectedly\n"
5570                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5571
5572             mon_cmd($vmid, "quit");
5573             $conf->{lock} = 'suspended';
5574             PVE::QemuConfig->write_config($vmid, $conf);
5575         });
5576     }
5577 }
5578
5579 sub vm_resume {
5580     my ($vmid, $skiplock, $nocheck) = @_;
5581
5582     PVE::QemuConfig->lock_config($vmid, sub {
5583         my $res = mon_cmd($vmid, 'query-status');
5584         my $resume_cmd = 'cont';
5585         my $reset = 0;
5586
5587         if ($res->{status}) {
5588             return if $res->{status} eq 'running'; # job done, go home
5589             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5590             $reset = 1 if $res->{status} eq 'shutdown';
5591         }
5592
5593         if (!$nocheck) {
5594
5595             my $conf = PVE::QemuConfig->load_config($vmid);
5596
5597             PVE::QemuConfig->check_lock($conf)
5598                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5599         }
5600
5601         if ($reset) {
5602             # required if a VM shuts down during a backup and we get a resume
5603             # request before the backup finishes for example
5604             mon_cmd($vmid, "system_reset");
5605         }
5606         mon_cmd($vmid, $resume_cmd);
5607     });
5608 }
5609
5610 sub vm_sendkey {
5611     my ($vmid, $skiplock, $key) = @_;
5612
5613     PVE::QemuConfig->lock_config($vmid, sub {
5614
5615         my $conf = PVE::QemuConfig->load_config($vmid);
5616
5617         # there is no qmp command, so we use the human monitor command
5618         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5619         die $res if $res ne '';
5620     });
5621 }
5622
5623 # vzdump restore implementaion
5624
5625 sub tar_archive_read_firstfile {
5626     my $archive = shift;
5627
5628     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5629
5630     # try to detect archive type first
5631     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5632         die "unable to open file '$archive'\n";
5633     my $firstfile = <$fh>;
5634     kill 15, $pid;
5635     close $fh;
5636
5637     die "ERROR: archive contaions no data\n" if !$firstfile;
5638     chomp $firstfile;
5639
5640     return $firstfile;
5641 }
5642
5643 sub tar_restore_cleanup {
5644     my ($storecfg, $statfile) = @_;
5645
5646     print STDERR "starting cleanup\n";
5647
5648     if (my $fd = IO::File->new($statfile, "r")) {
5649         while (defined(my $line = <$fd>)) {
5650             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5651                 my $volid = $2;
5652                 eval {
5653                     if ($volid =~ m|^/|) {
5654                         unlink $volid || die 'unlink failed\n';
5655                     } else {
5656                         PVE::Storage::vdisk_free($storecfg, $volid);
5657                     }
5658                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5659                 };
5660                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5661             } else {
5662                 print STDERR "unable to parse line in statfile - $line";
5663             }
5664         }
5665         $fd->close();
5666     }
5667 }
5668
5669 sub restore_file_archive {
5670     my ($archive, $vmid, $user, $opts) = @_;
5671
5672     return restore_vma_archive($archive, $vmid, $user, $opts)
5673         if $archive eq '-';
5674
5675     my $info = PVE::Storage::archive_info($archive);
5676     my $format = $opts->{format} // $info->{format};
5677     my $comp = $info->{compression};
5678
5679     # try to detect archive format
5680     if ($format eq 'tar') {
5681         return restore_tar_archive($archive, $vmid, $user, $opts);
5682     } else {
5683         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5684     }
5685 }
5686
5687 # hepler to remove disks that will not be used after restore
5688 my $restore_cleanup_oldconf = sub {
5689     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5690
5691     PVE::QemuConfig->foreach_volume($oldconf, sub {
5692         my ($ds, $drive) = @_;
5693
5694         return if drive_is_cdrom($drive, 1);
5695
5696         my $volid = $drive->{file};
5697         return if !$volid || $volid =~ m|^/|;
5698
5699         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5700         return if !$path || !$owner || ($owner != $vmid);
5701
5702         # Note: only delete disk we want to restore
5703         # other volumes will become unused
5704         if ($virtdev_hash->{$ds}) {
5705             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5706             if (my $err = $@) {
5707                 warn $err;
5708             }
5709         }
5710     });
5711
5712     # delete vmstate files, after the restore we have no snapshots anymore
5713     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5714         my $snap = $oldconf->{snapshots}->{$snapname};
5715         if ($snap->{vmstate}) {
5716             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5717             if (my $err = $@) {
5718                 warn $err;
5719             }
5720         }
5721     }
5722 };
5723
5724 # Helper to parse vzdump backup device hints
5725 #
5726 # $rpcenv: Environment, used to ckeck storage permissions
5727 # $user: User ID, to check storage permissions
5728 # $storecfg: Storage configuration
5729 # $fh: the file handle for reading the configuration
5730 # $devinfo: should contain device sizes for all backu-up'ed devices
5731 # $options: backup options (pool, default storage)
5732 #
5733 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5734 my $parse_backup_hints = sub {
5735     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5736
5737     my $virtdev_hash = {};
5738
5739     while (defined(my $line = <$fh>)) {
5740         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5741             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5742             die "archive does not contain data for drive '$virtdev'\n"
5743                 if !$devinfo->{$devname};
5744
5745             if (defined($options->{storage})) {
5746                 $storeid = $options->{storage} || 'local';
5747             } elsif (!$storeid) {
5748                 $storeid = 'local';
5749             }
5750             $format = 'raw' if !$format;
5751             $devinfo->{$devname}->{devname} = $devname;
5752             $devinfo->{$devname}->{virtdev} = $virtdev;
5753             $devinfo->{$devname}->{format} = $format;
5754             $devinfo->{$devname}->{storeid} = $storeid;
5755
5756             # check permission on storage
5757             my $pool = $options->{pool}; # todo: do we need that?
5758             if ($user ne 'root@pam') {
5759                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5760             }
5761
5762             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5763         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5764             my $virtdev = $1;
5765             my $drive = parse_drive($virtdev, $2);
5766             if (drive_is_cloudinit($drive)) {
5767                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5768                 $storeid = $options->{storage} if defined ($options->{storage});
5769                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5770                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5771
5772                 $virtdev_hash->{$virtdev} = {
5773                     format => $format,
5774                     storeid => $storeid,
5775                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5776                     is_cloudinit => 1,
5777                 };
5778             }
5779         }
5780     }
5781
5782     return $virtdev_hash;
5783 };
5784
5785 # Helper to allocate and activate all volumes required for a restore
5786 #
5787 # $storecfg: Storage configuration
5788 # $virtdev_hash: as returned by parse_backup_hints()
5789 #
5790 # Returns: { $virtdev => $volid }
5791 my $restore_allocate_devices = sub {
5792     my ($storecfg, $virtdev_hash, $vmid) = @_;
5793
5794     my $map = {};
5795     foreach my $virtdev (sort keys %$virtdev_hash) {
5796         my $d = $virtdev_hash->{$virtdev};
5797         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5798         my $storeid = $d->{storeid};
5799         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5800
5801         # test if requested format is supported
5802         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5803         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5804         $d->{format} = $defFormat if !$supported;
5805
5806         my $name;
5807         if ($d->{is_cloudinit}) {
5808             $name = "vm-$vmid-cloudinit";
5809             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5810             if ($scfg->{path}) {
5811                 $name .= ".$d->{format}";
5812             }
5813         }
5814
5815         my $volid = PVE::Storage::vdisk_alloc(
5816             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5817
5818         print STDERR "new volume ID is '$volid'\n";
5819         $d->{volid} = $volid;
5820
5821         PVE::Storage::activate_volumes($storecfg, [$volid]);
5822
5823         $map->{$virtdev} = $volid;
5824     }
5825
5826     return $map;
5827 };
5828
5829 my $restore_update_config_line = sub {
5830     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5831
5832     return if $line =~ m/^\#qmdump\#/;
5833     return if $line =~ m/^\#vzdump\#/;
5834     return if $line =~ m/^lock:/;
5835     return if $line =~ m/^unused\d+:/;
5836     return if $line =~ m/^parent:/;
5837
5838     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5839     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5840         # try to convert old 1.X settings
5841         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5842         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5843             my ($model, $macaddr) = split(/\=/, $devconfig);
5844             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5845             my $net = {
5846                 model => $model,
5847                 bridge => "vmbr$ind",
5848                 macaddr => $macaddr,
5849             };
5850             my $netstr = print_net($net);
5851
5852             print $outfd "net$cookie->{netcount}: $netstr\n";
5853             $cookie->{netcount}++;
5854         }
5855     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5856         my ($id, $netstr) = ($1, $2);
5857         my $net = parse_net($netstr);
5858         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5859         $netstr = print_net($net);
5860         print $outfd "$id: $netstr\n";
5861     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5862         my $virtdev = $1;
5863         my $value = $3;
5864         my $di = parse_drive($virtdev, $value);
5865         if (defined($di->{backup}) && !$di->{backup}) {
5866             print $outfd "#$line";
5867         } elsif ($map->{$virtdev}) {
5868             delete $di->{format}; # format can change on restore
5869             $di->{file} = $map->{$virtdev};
5870             $value = print_drive($di);
5871             print $outfd "$virtdev: $value\n";
5872         } else {
5873             print $outfd $line;
5874         }
5875     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5876         my $vmgenid = $1;
5877         if ($vmgenid ne '0') {
5878             # always generate a new vmgenid if there was a valid one setup
5879             $vmgenid = generate_uuid();
5880         }
5881         print $outfd "vmgenid: $vmgenid\n";
5882     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5883         my ($uuid, $uuid_str);
5884         UUID::generate($uuid);
5885         UUID::unparse($uuid, $uuid_str);
5886         my $smbios1 = parse_smbios1($2);
5887         $smbios1->{uuid} = $uuid_str;
5888         print $outfd $1.print_smbios1($smbios1)."\n";
5889     } else {
5890         print $outfd $line;
5891     }
5892 };
5893
5894 my $restore_deactivate_volumes = sub {
5895     my ($storecfg, $devinfo) = @_;
5896
5897     my $vollist = [];
5898     foreach my $devname (keys %$devinfo) {
5899         my $volid = $devinfo->{$devname}->{volid};
5900         push @$vollist, $volid if $volid;
5901     }
5902
5903     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5904 };
5905
5906 my $restore_destroy_volumes = sub {
5907     my ($storecfg, $devinfo) = @_;
5908
5909     foreach my $devname (keys %$devinfo) {
5910         my $volid = $devinfo->{$devname}->{volid};
5911         next if !$volid;
5912         eval {
5913             if ($volid =~ m|^/|) {
5914                 unlink $volid || die 'unlink failed\n';
5915             } else {
5916                 PVE::Storage::vdisk_free($storecfg, $volid);
5917             }
5918             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5919         };
5920         print STDERR "unable to cleanup '$volid' - $@" if $@;
5921     }
5922 };
5923
5924 sub scan_volids {
5925     my ($cfg, $vmid) = @_;
5926
5927     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5928
5929     my $volid_hash = {};
5930     foreach my $storeid (keys %$info) {
5931         foreach my $item (@{$info->{$storeid}}) {
5932             next if !($item->{volid} && $item->{size});
5933             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5934             $volid_hash->{$item->{volid}} = $item;
5935         }
5936     }
5937
5938     return $volid_hash;
5939 }
5940
5941 sub update_disk_config {
5942     my ($vmid, $conf, $volid_hash) = @_;
5943
5944     my $changes;
5945     my $prefix = "VM $vmid";
5946
5947     # used and unused disks
5948     my $referenced = {};
5949
5950     # Note: it is allowed to define multiple storages with same path (alias), so
5951     # we need to check both 'volid' and real 'path' (two different volid can point
5952     # to the same path).
5953
5954     my $referencedpath = {};
5955
5956     # update size info
5957     PVE::QemuConfig->foreach_volume($conf, sub {
5958         my ($opt, $drive) = @_;
5959
5960         my $volid = $drive->{file};
5961         return if !$volid;
5962         my $volume = $volid_hash->{$volid};
5963
5964         # mark volid as "in-use" for next step
5965         $referenced->{$volid} = 1;
5966         if ($volume && (my $path = $volume->{path})) {
5967             $referencedpath->{$path} = 1;
5968         }
5969
5970         return if drive_is_cdrom($drive);
5971         return if !$volume;
5972
5973         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5974         if (defined($updated)) {
5975             $changes = 1;
5976             $conf->{$opt} = print_drive($updated);
5977             print "$prefix ($opt): $msg\n";
5978         }
5979     });
5980
5981     # remove 'unusedX' entry if volume is used
5982     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5983         my ($opt, $drive) = @_;
5984
5985         my $volid = $drive->{file};
5986         return if !$volid;
5987
5988         my $path;
5989         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5990         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5991             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5992             $changes = 1;
5993             delete $conf->{$opt};
5994         }
5995
5996         $referenced->{$volid} = 1;
5997         $referencedpath->{$path} = 1 if $path;
5998     });
5999
6000     foreach my $volid (sort keys %$volid_hash) {
6001         next if $volid =~ m/vm-$vmid-state-/;
6002         next if $referenced->{$volid};
6003         my $path = $volid_hash->{$volid}->{path};
6004         next if !$path; # just to be sure
6005         next if $referencedpath->{$path};
6006         $changes = 1;
6007         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6008         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6009         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6010     }
6011
6012     return $changes;
6013 }
6014
6015 sub rescan {
6016     my ($vmid, $nolock, $dryrun) = @_;
6017
6018     my $cfg = PVE::Storage::config();
6019
6020     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6021     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6022     foreach my $stor (keys %{$cfg->{ids}}) {
6023         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6024     }
6025
6026     print "rescan volumes...\n";
6027     my $volid_hash = scan_volids($cfg, $vmid);
6028
6029     my $updatefn =  sub {
6030         my ($vmid) = @_;
6031
6032         my $conf = PVE::QemuConfig->load_config($vmid);
6033
6034         PVE::QemuConfig->check_lock($conf);
6035
6036         my $vm_volids = {};
6037         foreach my $volid (keys %$volid_hash) {
6038             my $info = $volid_hash->{$volid};
6039             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6040         }
6041
6042         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6043
6044         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6045     };
6046
6047     if (defined($vmid)) {
6048         if ($nolock) {
6049             &$updatefn($vmid);
6050         } else {
6051             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6052         }
6053     } else {
6054         my $vmlist = config_list();
6055         foreach my $vmid (keys %$vmlist) {
6056             if ($nolock) {
6057                 &$updatefn($vmid);
6058             } else {
6059                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6060             }
6061         }
6062     }
6063 }
6064
6065 sub restore_proxmox_backup_archive {
6066     my ($archive, $vmid, $user, $options) = @_;
6067
6068     my $storecfg = PVE::Storage::config();
6069
6070     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6071     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6072
6073     my $fingerprint = $scfg->{fingerprint};
6074     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6075
6076     my $repo = PVE::PBSClient::get_repository($scfg);
6077
6078     # This is only used for `pbs-restore`!
6079     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6080     local $ENV{PBS_PASSWORD} = $password;
6081     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6082
6083     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6084         PVE::Storage::parse_volname($storecfg, $archive);
6085
6086     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6087
6088     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6089
6090     my $tmpdir = "/var/tmp/vzdumptmp$$";
6091     rmtree $tmpdir;
6092     mkpath $tmpdir;
6093
6094     my $conffile = PVE::QemuConfig->config_file($vmid);
6095     my $tmpfn = "$conffile.$$.tmp";
6096      # disable interrupts (always do cleanups)
6097     local $SIG{INT} =
6098         local $SIG{TERM} =
6099         local $SIG{QUIT} =
6100         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6101
6102     # Note: $oldconf is undef if VM does not exists
6103     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6104     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6105
6106     my $rpcenv = PVE::RPCEnvironment::get();
6107     my $devinfo = {};
6108
6109     eval {
6110         # enable interrupts
6111         local $SIG{INT} =
6112             local $SIG{TERM} =
6113             local $SIG{QUIT} =
6114             local $SIG{HUP} =
6115             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6116
6117         my $cfgfn = "$tmpdir/qemu-server.conf";
6118         my $firewall_config_fn = "$tmpdir/fw.conf";
6119         my $index_fn = "$tmpdir/index.json";
6120
6121         my $cmd = "restore";
6122
6123         my $param = [$pbs_backup_name, "index.json", $index_fn];
6124         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6125         my $index = PVE::Tools::file_get_contents($index_fn);
6126         $index = decode_json($index);
6127
6128         # print Dumper($index);
6129         foreach my $info (@{$index->{files}}) {
6130             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6131                 my $devname = $1;
6132                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6133                     $devinfo->{$devname}->{size} = $1;
6134                 } else {
6135                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6136                 }
6137             }
6138         }
6139
6140         my $is_qemu_server_backup = scalar(
6141             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6142         );
6143         if (!$is_qemu_server_backup) {
6144             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6145         }
6146         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6147
6148         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6149         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6150
6151         if ($has_firewall_config) {
6152             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6153             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6154
6155             my $pve_firewall_dir = '/etc/pve/firewall';
6156             mkdir $pve_firewall_dir; # make sure the dir exists
6157             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6158         }
6159
6160         my $fh = IO::File->new($cfgfn, "r") ||
6161             die "unable to read qemu-server.conf - $!\n";
6162
6163         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6164
6165         # fixme: rate limit?
6166
6167         # create empty/temp config
6168         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6169
6170         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6171
6172         # allocate volumes
6173         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6174
6175         foreach my $virtdev (sort keys %$virtdev_hash) {
6176             my $d = $virtdev_hash->{$virtdev};
6177             next if $d->{is_cloudinit}; # no need to restore cloudinit
6178
6179             my $volid = $d->{volid};
6180
6181             my $path = PVE::Storage::path($storecfg, $volid);
6182
6183             # This is the ONLY user of the PBS_ env vars set on top of this function!
6184             my $pbs_restore_cmd = [
6185                 '/usr/bin/pbs-restore',
6186                 '--repository', $repo,
6187                 $pbs_backup_name,
6188                 "$d->{devname}.img.fidx",
6189                 $path,
6190                 '--verbose',
6191                 ];
6192
6193             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6194             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6195
6196             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6197                 push @$pbs_restore_cmd, '--skip-zero';
6198             }
6199
6200             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6201             print "restore proxmox backup image: $dbg_cmdstring\n";
6202             run_command($pbs_restore_cmd);
6203         }
6204
6205         $fh->seek(0, 0) || die "seek failed - $!\n";
6206
6207         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6208
6209         my $cookie = { netcount => 0 };
6210         while (defined(my $line = <$fh>)) {
6211             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6212         }
6213
6214         $fh->close();
6215         $outfd->close();
6216     };
6217     my $err = $@;
6218
6219     $restore_deactivate_volumes->($storecfg, $devinfo);
6220
6221     rmtree $tmpdir;
6222
6223     if ($err) {
6224         unlink $tmpfn;
6225         $restore_destroy_volumes->($storecfg, $devinfo);
6226         die $err;
6227     }
6228
6229     rename($tmpfn, $conffile) ||
6230         die "unable to commit configuration file '$conffile'\n";
6231
6232     PVE::Cluster::cfs_update(); # make sure we read new file
6233
6234     eval { rescan($vmid, 1); };
6235     warn $@ if $@;
6236 }
6237
6238 sub restore_vma_archive {
6239     my ($archive, $vmid, $user, $opts, $comp) = @_;
6240
6241     my $readfrom = $archive;
6242
6243     my $cfg = PVE::Storage::config();
6244     my $commands = [];
6245     my $bwlimit = $opts->{bwlimit};
6246
6247     my $dbg_cmdstring = '';
6248     my $add_pipe = sub {
6249         my ($cmd) = @_;
6250         push @$commands, $cmd;
6251         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6252         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6253         $readfrom = '-';
6254     };
6255
6256     my $input = undef;
6257     if ($archive eq '-') {
6258         $input = '<&STDIN';
6259     } else {
6260         # If we use a backup from a PVE defined storage we also consider that
6261         # storage's rate limit:
6262         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6263         if (defined($volid)) {
6264             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6265             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6266             if ($readlimit) {
6267                 print STDERR "applying read rate limit: $readlimit\n";
6268                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6269                 $add_pipe->($cstream);
6270             }
6271         }
6272     }
6273
6274     if ($comp) {
6275         my $info = PVE::Storage::decompressor_info('vma', $comp);
6276         my $cmd = $info->{decompressor};
6277         push @$cmd, $readfrom;
6278         $add_pipe->($cmd);
6279     }
6280
6281     my $tmpdir = "/var/tmp/vzdumptmp$$";
6282     rmtree $tmpdir;
6283
6284     # disable interrupts (always do cleanups)
6285     local $SIG{INT} =
6286         local $SIG{TERM} =
6287         local $SIG{QUIT} =
6288         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6289
6290     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6291     POSIX::mkfifo($mapfifo, 0600);
6292     my $fifofh;
6293     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6294
6295     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6296
6297     my $oldtimeout;
6298     my $timeout = 5;
6299
6300     my $devinfo = {};
6301
6302     my $rpcenv = PVE::RPCEnvironment::get();
6303
6304     my $conffile = PVE::QemuConfig->config_file($vmid);
6305     my $tmpfn = "$conffile.$$.tmp";
6306
6307     # Note: $oldconf is undef if VM does not exist
6308     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6309     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6310
6311     my %storage_limits;
6312
6313     my $print_devmap = sub {
6314         my $cfgfn = "$tmpdir/qemu-server.conf";
6315
6316         # we can read the config - that is already extracted
6317         my $fh = IO::File->new($cfgfn, "r") ||
6318             die "unable to read qemu-server.conf - $!\n";
6319
6320         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6321         if (-f $fwcfgfn) {
6322             my $pve_firewall_dir = '/etc/pve/firewall';
6323             mkdir $pve_firewall_dir; # make sure the dir exists
6324             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6325         }
6326
6327         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6328
6329         foreach my $key (keys %storage_limits) {
6330             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6331             next if !$limit;
6332             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6333             $storage_limits{$key} = $limit * 1024;
6334         }
6335
6336         foreach my $devname (keys %$devinfo) {
6337             die "found no device mapping information for device '$devname'\n"
6338                 if !$devinfo->{$devname}->{virtdev};
6339         }
6340
6341         # create empty/temp config
6342         if ($oldconf) {
6343             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6344             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6345         }
6346
6347         # allocate volumes
6348         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6349
6350         # print restore information to $fifofh
6351         foreach my $virtdev (sort keys %$virtdev_hash) {
6352             my $d = $virtdev_hash->{$virtdev};
6353             next if $d->{is_cloudinit}; # no need to restore cloudinit
6354
6355             my $storeid = $d->{storeid};
6356             my $volid = $d->{volid};
6357
6358             my $map_opts = '';
6359             if (my $limit = $storage_limits{$storeid}) {
6360                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6361             }
6362
6363             my $write_zeros = 1;
6364             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6365                 $write_zeros = 0;
6366             }
6367
6368             my $path = PVE::Storage::path($cfg, $volid);
6369
6370             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6371
6372             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6373         }
6374
6375         $fh->seek(0, 0) || die "seek failed - $!\n";
6376
6377         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6378
6379         my $cookie = { netcount => 0 };
6380         while (defined(my $line = <$fh>)) {
6381             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6382         }
6383
6384         $fh->close();
6385         $outfd->close();
6386     };
6387
6388     eval {
6389         # enable interrupts
6390         local $SIG{INT} =
6391             local $SIG{TERM} =
6392             local $SIG{QUIT} =
6393             local $SIG{HUP} =
6394             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6395         local $SIG{ALRM} = sub { die "got timeout\n"; };
6396
6397         $oldtimeout = alarm($timeout);
6398
6399         my $parser = sub {
6400             my $line = shift;
6401
6402             print "$line\n";
6403
6404             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6405                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6406                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6407             } elsif ($line =~ m/^CTIME: /) {
6408                 # we correctly received the vma config, so we can disable
6409                 # the timeout now for disk allocation (set to 10 minutes, so
6410                 # that we always timeout if something goes wrong)
6411                 alarm(600);
6412                 &$print_devmap();
6413                 print $fifofh "done\n";
6414                 my $tmp = $oldtimeout || 0;
6415                 $oldtimeout = undef;
6416                 alarm($tmp);
6417                 close($fifofh);
6418                 $fifofh = undef;
6419             }
6420         };
6421
6422         print "restore vma archive: $dbg_cmdstring\n";
6423         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6424     };
6425     my $err = $@;
6426
6427     alarm($oldtimeout) if $oldtimeout;
6428
6429     $restore_deactivate_volumes->($cfg, $devinfo);
6430
6431     close($fifofh) if $fifofh;
6432     unlink $mapfifo;
6433     rmtree $tmpdir;
6434
6435     if ($err) {
6436         unlink $tmpfn;
6437         $restore_destroy_volumes->($cfg, $devinfo);
6438         die $err;
6439     }
6440
6441     rename($tmpfn, $conffile) ||
6442         die "unable to commit configuration file '$conffile'\n";
6443
6444     PVE::Cluster::cfs_update(); # make sure we read new file
6445
6446     eval { rescan($vmid, 1); };
6447     warn $@ if $@;
6448 }
6449
6450 sub restore_tar_archive {
6451     my ($archive, $vmid, $user, $opts) = @_;
6452
6453     if ($archive ne '-') {
6454         my $firstfile = tar_archive_read_firstfile($archive);
6455         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6456             if $firstfile ne 'qemu-server.conf';
6457     }
6458
6459     my $storecfg = PVE::Storage::config();
6460
6461     # avoid zombie disks when restoring over an existing VM -> cleanup first
6462     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6463     # skiplock=1 because qmrestore has set the 'create' lock itself already
6464     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6465     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6466
6467     my $tocmd = "/usr/lib/qemu-server/qmextract";
6468
6469     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6470     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6471     $tocmd .= ' --prealloc' if $opts->{prealloc};
6472     $tocmd .= ' --info' if $opts->{info};
6473
6474     # tar option "xf" does not autodetect compression when read from STDIN,
6475     # so we pipe to zcat
6476     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6477         PVE::Tools::shellquote("--to-command=$tocmd");
6478
6479     my $tmpdir = "/var/tmp/vzdumptmp$$";
6480     mkpath $tmpdir;
6481
6482     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6483     local $ENV{VZDUMP_VMID} = $vmid;
6484     local $ENV{VZDUMP_USER} = $user;
6485
6486     my $conffile = PVE::QemuConfig->config_file($vmid);
6487     my $tmpfn = "$conffile.$$.tmp";
6488
6489     # disable interrupts (always do cleanups)
6490     local $SIG{INT} =
6491         local $SIG{TERM} =
6492         local $SIG{QUIT} =
6493         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6494
6495     eval {
6496         # enable interrupts
6497         local $SIG{INT} =
6498             local $SIG{TERM} =
6499             local $SIG{QUIT} =
6500             local $SIG{HUP} =
6501             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6502
6503         if ($archive eq '-') {
6504             print "extracting archive from STDIN\n";
6505             run_command($cmd, input => "<&STDIN");
6506         } else {
6507             print "extracting archive '$archive'\n";
6508             run_command($cmd);
6509         }
6510
6511         return if $opts->{info};
6512
6513         # read new mapping
6514         my $map = {};
6515         my $statfile = "$tmpdir/qmrestore.stat";
6516         if (my $fd = IO::File->new($statfile, "r")) {
6517             while (defined (my $line = <$fd>)) {
6518                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6519                     $map->{$1} = $2 if $1;
6520                 } else {
6521                     print STDERR "unable to parse line in statfile - $line\n";
6522                 }
6523             }
6524             $fd->close();
6525         }
6526
6527         my $confsrc = "$tmpdir/qemu-server.conf";
6528
6529         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6530
6531         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6532
6533         my $cookie = { netcount => 0 };
6534         while (defined (my $line = <$srcfd>)) {
6535             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6536         }
6537
6538         $srcfd->close();
6539         $outfd->close();
6540     };
6541     if (my $err = $@) {
6542         unlink $tmpfn;
6543         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6544         die $err;
6545     }
6546
6547     rmtree $tmpdir;
6548
6549     rename $tmpfn, $conffile ||
6550         die "unable to commit configuration file '$conffile'\n";
6551
6552     PVE::Cluster::cfs_update(); # make sure we read new file
6553
6554     eval { rescan($vmid, 1); };
6555     warn $@ if $@;
6556 };
6557
6558 sub foreach_storage_used_by_vm {
6559     my ($conf, $func) = @_;
6560
6561     my $sidhash = {};
6562
6563     PVE::QemuConfig->foreach_volume($conf, sub {
6564         my ($ds, $drive) = @_;
6565         return if drive_is_cdrom($drive);
6566
6567         my $volid = $drive->{file};
6568
6569         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6570         $sidhash->{$sid} = $sid if $sid;
6571     });
6572
6573     foreach my $sid (sort keys %$sidhash) {
6574         &$func($sid);
6575     }
6576 }
6577
6578 my $qemu_snap_storage = {
6579     rbd => 1,
6580 };
6581 sub do_snapshots_with_qemu {
6582     my ($storecfg, $volid) = @_;
6583
6584     my $storage_name = PVE::Storage::parse_volume_id($volid);
6585     my $scfg = $storecfg->{ids}->{$storage_name};
6586     die "could not find storage '$storage_name'\n" if !defined($scfg);
6587
6588     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6589         return 1;
6590     }
6591
6592     if ($volid =~ m/\.(qcow2|qed)$/){
6593         return 1;
6594     }
6595
6596     return;
6597 }
6598
6599 sub qga_check_running {
6600     my ($vmid, $nowarn) = @_;
6601
6602     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6603     if ($@) {
6604         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6605         return 0;
6606     }
6607     return 1;
6608 }
6609
6610 sub template_create {
6611     my ($vmid, $conf, $disk) = @_;
6612
6613     my $storecfg = PVE::Storage::config();
6614
6615     PVE::QemuConfig->foreach_volume($conf, sub {
6616         my ($ds, $drive) = @_;
6617
6618         return if drive_is_cdrom($drive);
6619         return if $disk && $ds ne $disk;
6620
6621         my $volid = $drive->{file};
6622         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6623
6624         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6625         $drive->{file} = $voliddst;
6626         $conf->{$ds} = print_drive($drive);
6627         PVE::QemuConfig->write_config($vmid, $conf);
6628     });
6629 }
6630
6631 sub convert_iscsi_path {
6632     my ($path) = @_;
6633
6634     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6635         my $portal = $1;
6636         my $target = $2;
6637         my $lun = $3;
6638
6639         my $initiator_name = get_initiator_name();
6640
6641         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6642                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6643     }
6644
6645     die "cannot convert iscsi path '$path', unkown format\n";
6646 }
6647
6648 sub qemu_img_convert {
6649     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6650
6651     my $storecfg = PVE::Storage::config();
6652     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6653     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6654
6655     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6656
6657     my $cachemode;
6658     my $src_path;
6659     my $src_is_iscsi = 0;
6660     my $src_format;
6661
6662     if ($src_storeid) {
6663         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6664         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6665         $src_format = qemu_img_format($src_scfg, $src_volname);
6666         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6667         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6668         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6669     } elsif (-f $src_volid) {
6670         $src_path = $src_volid;
6671         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6672             $src_format = $1;
6673         }
6674     }
6675
6676     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6677
6678     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6679     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6680     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6681     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6682
6683     my $cmd = [];
6684     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6685     push @$cmd, '-l', "snapshot.name=$snapname"
6686         if $snapname && $src_format && $src_format eq "qcow2";
6687     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6688     push @$cmd, '-T', $cachemode if defined($cachemode);
6689
6690     if ($src_is_iscsi) {
6691         push @$cmd, '--image-opts';
6692         $src_path = convert_iscsi_path($src_path);
6693     } elsif ($src_format) {
6694         push @$cmd, '-f', $src_format;
6695     }
6696
6697     if ($dst_is_iscsi) {
6698         push @$cmd, '--target-image-opts';
6699         $dst_path = convert_iscsi_path($dst_path);
6700     } else {
6701         push @$cmd, '-O', $dst_format;
6702     }
6703
6704     push @$cmd, $src_path;
6705
6706     if (!$dst_is_iscsi && $is_zero_initialized) {
6707         push @$cmd, "zeroinit:$dst_path";
6708     } else {
6709         push @$cmd, $dst_path;
6710     }
6711
6712     my $parser = sub {
6713         my $line = shift;
6714         if($line =~ m/\((\S+)\/100\%\)/){
6715             my $percent = $1;
6716             my $transferred = int($size * $percent / 100);
6717             my $remaining = $size - $transferred;
6718
6719             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6720         }
6721
6722     };
6723
6724     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6725     my $err = $@;
6726     die "copy failed: $err" if $err;
6727 }
6728
6729 sub qemu_img_format {
6730     my ($scfg, $volname) = @_;
6731
6732     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6733         return $1;
6734     } else {
6735         return "raw";
6736     }
6737 }
6738
6739 sub qemu_drive_mirror {
6740     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6741
6742     $jobs = {} if !$jobs;
6743
6744     my $qemu_target;
6745     my $format;
6746     $jobs->{"drive-$drive"} = {};
6747
6748     if ($dst_volid =~ /^nbd:/) {
6749         $qemu_target = $dst_volid;
6750         $format = "nbd";
6751     } else {
6752         my $storecfg = PVE::Storage::config();
6753         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6754
6755         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6756
6757         $format = qemu_img_format($dst_scfg, $dst_volname);
6758
6759         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6760
6761         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6762     }
6763
6764     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6765     $opts->{format} = $format if $format;
6766
6767     if (defined($src_bitmap)) {
6768         $opts->{sync} = 'incremental';
6769         $opts->{bitmap} = $src_bitmap;
6770         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6771     }
6772
6773     if (defined($bwlimit)) {
6774         $opts->{speed} = $bwlimit * 1024;
6775         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6776     } else {
6777         print "drive mirror is starting for drive-$drive\n";
6778     }
6779
6780     # if a job already runs for this device we get an error, catch it for cleanup
6781     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6782     if (my $err = $@) {
6783         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6784         warn "$@\n" if $@;
6785         die "mirroring error: $err\n";
6786     }
6787
6788     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6789 }
6790
6791 # $completion can be either
6792 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6793 # 'cancel': wait until all jobs are ready, block-job-cancel them
6794 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6795 sub qemu_drive_mirror_monitor {
6796     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6797
6798     $completion //= 'complete';
6799
6800     eval {
6801         my $err_complete = 0;
6802
6803         while (1) {
6804             die "storage migration timed out\n" if $err_complete > 300;
6805
6806             my $stats = mon_cmd($vmid, "query-block-jobs");
6807
6808             my $running_mirror_jobs = {};
6809             foreach my $stat (@$stats) {
6810                 next if $stat->{type} ne 'mirror';
6811                 $running_mirror_jobs->{$stat->{device}} = $stat;
6812             }
6813
6814             my $readycounter = 0;
6815
6816             foreach my $job (keys %$jobs) {
6817
6818                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6819                     print "$job : finished\n";
6820                     delete $jobs->{$job};
6821                     next;
6822                 }
6823
6824                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6825
6826                 my $busy = $running_mirror_jobs->{$job}->{busy};
6827                 my $ready = $running_mirror_jobs->{$job}->{ready};
6828                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6829                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6830                     my $remaining = $total - $transferred;
6831                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6832
6833                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6834                 }
6835
6836                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6837             }
6838
6839             last if scalar(keys %$jobs) == 0;
6840
6841             if ($readycounter == scalar(keys %$jobs)) {
6842                 print "all mirroring jobs are ready \n";
6843                 last if $completion eq 'skip'; #do the complete later
6844
6845                 if ($vmiddst && $vmiddst != $vmid) {
6846                     my $agent_running = $qga && qga_check_running($vmid);
6847                     if ($agent_running) {
6848                         print "freeze filesystem\n";
6849                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6850                     } else {
6851                         print "suspend vm\n";
6852                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6853                     }
6854
6855                     # if we clone a disk for a new target vm, we don't switch the disk
6856                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6857
6858                     if ($agent_running) {
6859                         print "unfreeze filesystem\n";
6860                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6861                     } else {
6862                         print "resume vm\n";
6863                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6864                     }
6865
6866                     last;
6867                 } else {
6868
6869                     foreach my $job (keys %$jobs) {
6870                         # try to switch the disk if source and destination are on the same guest
6871                         print "$job: Completing block job...\n";
6872
6873                         my $op;
6874                         if ($completion eq 'complete') {
6875                             $op = 'block-job-complete';
6876                         } elsif ($completion eq 'cancel') {
6877                             $op = 'block-job-cancel';
6878                         } else {
6879                             die "invalid completion value: $completion\n";
6880                         }
6881                         eval { mon_cmd($vmid, $op, device => $job) };
6882                         if ($@ =~ m/cannot be completed/) {
6883                             print "$job: Block job cannot be completed, try again.\n";
6884                             $err_complete++;
6885                         }else {
6886                             print "$job: Completed successfully.\n";
6887                             $jobs->{$job}->{complete} = 1;
6888                         }
6889                     }
6890                 }
6891             }
6892             sleep 1;
6893         }
6894     };
6895     my $err = $@;
6896
6897     if ($err) {
6898         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6899         die "mirroring error: $err";
6900     }
6901
6902 }
6903
6904 sub qemu_blockjobs_cancel {
6905     my ($vmid, $jobs) = @_;
6906
6907     foreach my $job (keys %$jobs) {
6908         print "$job: Cancelling block job\n";
6909         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6910         $jobs->{$job}->{cancel} = 1;
6911     }
6912
6913     while (1) {
6914         my $stats = mon_cmd($vmid, "query-block-jobs");
6915
6916         my $running_jobs = {};
6917         foreach my $stat (@$stats) {
6918             $running_jobs->{$stat->{device}} = $stat;
6919         }
6920
6921         foreach my $job (keys %$jobs) {
6922
6923             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6924                 print "$job: Done.\n";
6925                 delete $jobs->{$job};
6926             }
6927         }
6928
6929         last if scalar(keys %$jobs) == 0;
6930
6931         sleep 1;
6932     }
6933 }
6934
6935 sub clone_disk {
6936     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6937         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6938
6939     my $newvolid;
6940
6941     if (!$full) {
6942         print "create linked clone of drive $drivename ($drive->{file})\n";
6943         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6944         push @$newvollist, $newvolid;
6945     } else {
6946
6947         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6948         $storeid = $storage if $storage;
6949
6950         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6951
6952         print "create full clone of drive $drivename ($drive->{file})\n";
6953         my $name = undef;
6954         my $size = undef;
6955         if (drive_is_cloudinit($drive)) {
6956             $name = "vm-$newvmid-cloudinit";
6957             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6958             if ($scfg->{path}) {
6959                 $name .= ".$dst_format";
6960             }
6961             $snapname = undef;
6962             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6963         } elsif ($drivename eq 'efidisk0') {
6964             $size = get_efivars_size($conf);
6965         } else {
6966             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
6967         }
6968         $newvolid = PVE::Storage::vdisk_alloc(
6969             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
6970         );
6971         push @$newvollist, $newvolid;
6972
6973         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6974
6975         if (drive_is_cloudinit($drive)) {
6976             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
6977             # if this is the case, we have to complete any block-jobs still there from
6978             # previous drive-mirrors
6979             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
6980                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
6981             }
6982             goto no_data_clone;
6983         }
6984
6985         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6986         if (!$running || $snapname) {
6987             # TODO: handle bwlimits
6988             if ($drivename eq 'efidisk0') {
6989                 # the relevant data on the efidisk may be smaller than the source
6990                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6991                 # that is given by the OVMF_VARS.fd
6992                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6993                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6994
6995                 # better for Ceph if block size is not too small, see bug #3324
6996                 my $bs = 1024*1024;
6997
6998                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
6999                     "if=$src_path", "of=$dst_path"]);
7000             } else {
7001                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7002             }
7003         } else {
7004
7005             my $kvmver = get_running_qemu_version ($vmid);
7006             if (!min_version($kvmver, 2, 7)) {
7007                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7008                     if $drive->{iothread};
7009             }
7010
7011             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7012                 $completion, $qga, $bwlimit);
7013         }
7014     }
7015
7016 no_data_clone:
7017     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7018
7019     my $disk = $drive;
7020     $disk->{format} = undef;
7021     $disk->{file} = $newvolid;
7022     $disk->{size} = $size if defined($size);
7023
7024     return $disk;
7025 }
7026
7027 sub get_running_qemu_version {
7028     my ($vmid) = @_;
7029     my $res = mon_cmd($vmid, "query-version");
7030     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7031 }
7032
7033 sub qemu_use_old_bios_files {
7034     my ($machine_type) = @_;
7035
7036     return if !$machine_type;
7037
7038     my $use_old_bios_files = undef;
7039
7040     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7041         $machine_type = $1;
7042         $use_old_bios_files = 1;
7043     } else {
7044         my $version = extract_version($machine_type, kvm_user_version());
7045         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7046         # load new efi bios files on migration. So this hack is required to allow
7047         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7048         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7049         $use_old_bios_files = !min_version($version, 2, 4);
7050     }
7051
7052     return ($use_old_bios_files, $machine_type);
7053 }
7054
7055 sub get_efivars_size {
7056     my ($conf) = @_;
7057     my $arch = get_vm_arch($conf);
7058     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7059     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7060     return -s $ovmf_vars;
7061 }
7062
7063 sub update_efidisk_size {
7064     my ($conf) = @_;
7065
7066     return if !defined($conf->{efidisk0});
7067
7068     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7069     $disk->{size} = get_efivars_size($conf);
7070     $conf->{efidisk0} = print_drive($disk);
7071
7072     return;
7073 }
7074
7075 sub create_efidisk($$$$$) {
7076     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7077
7078     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7079     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7080
7081     my $vars_size_b = -s $ovmf_vars;
7082     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7083     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7084     PVE::Storage::activate_volumes($storecfg, [$volid]);
7085
7086     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7087     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7088
7089     return ($volid, $size/1024);
7090 }
7091
7092 sub vm_iothreads_list {
7093     my ($vmid) = @_;
7094
7095     my $res = mon_cmd($vmid, 'query-iothreads');
7096
7097     my $iothreads = {};
7098     foreach my $iothread (@$res) {
7099         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7100     }
7101
7102     return $iothreads;
7103 }
7104
7105 sub scsihw_infos {
7106     my ($conf, $drive) = @_;
7107
7108     my $maxdev = 0;
7109
7110     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7111         $maxdev = 7;
7112     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7113         $maxdev = 1;
7114     } else {
7115         $maxdev = 256;
7116     }
7117
7118     my $controller = int($drive->{index} / $maxdev);
7119     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7120         ? "virtioscsi"
7121         : "scsihw";
7122
7123     return ($maxdev, $controller, $controller_prefix);
7124 }
7125
7126 sub windows_version {
7127     my ($ostype) = @_;
7128
7129     return 0 if !$ostype;
7130
7131     my $winversion = 0;
7132
7133     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7134         $winversion = 5;
7135     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7136         $winversion = 6;
7137     } elsif ($ostype =~ m/^win(\d+)$/) {
7138         $winversion = $1;
7139     }
7140
7141     return $winversion;
7142 }
7143
7144 sub resolve_dst_disk_format {
7145         my ($storecfg, $storeid, $src_volname, $format) = @_;
7146         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7147
7148         if (!$format) {
7149             # if no target format is specified, use the source disk format as hint
7150             if ($src_volname) {
7151                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7152                 $format = qemu_img_format($scfg, $src_volname);
7153             } else {
7154                 return $defFormat;
7155             }
7156         }
7157
7158         # test if requested format is supported - else use default
7159         my $supported = grep { $_ eq $format } @$validFormats;
7160         $format = $defFormat if !$supported;
7161         return $format;
7162 }
7163
7164 # NOTE: if this logic changes, please update docs & possibly gui logic
7165 sub find_vmstate_storage {
7166     my ($conf, $storecfg) = @_;
7167
7168     # first, return storage from conf if set
7169     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7170
7171     my ($target, $shared, $local);
7172
7173     foreach_storage_used_by_vm($conf, sub {
7174         my ($sid) = @_;
7175         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7176         my $dst = $scfg->{shared} ? \$shared : \$local;
7177         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7178     });
7179
7180     # second, use shared storage where VM has at least one disk
7181     # third, use local storage where VM has at least one disk
7182     # fall back to local storage
7183     $target = $shared // $local // 'local';
7184
7185     return $target;
7186 }
7187
7188 sub generate_uuid {
7189     my ($uuid, $uuid_str);
7190     UUID::generate($uuid);
7191     UUID::unparse($uuid, $uuid_str);
7192     return $uuid_str;
7193 }
7194
7195 sub generate_smbios1_uuid {
7196     return "uuid=".generate_uuid();
7197 }
7198
7199 sub nbd_stop {
7200     my ($vmid) = @_;
7201
7202     mon_cmd($vmid, 'nbd-server-stop');
7203 }
7204
7205 sub create_reboot_request {
7206     my ($vmid) = @_;
7207     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7208         or die "failed to create reboot trigger file: $!\n";
7209     close($fh);
7210 }
7211
7212 sub clear_reboot_request {
7213     my ($vmid) = @_;
7214     my $path = "/run/qemu-server/$vmid.reboot";
7215     my $res = 0;
7216
7217     $res = unlink($path);
7218     die "could not remove reboot request for $vmid: $!"
7219         if !$res && $! != POSIX::ENOENT;
7220
7221     return $res;
7222 }
7223
7224 sub bootorder_from_legacy {
7225     my ($conf, $bootcfg) = @_;
7226
7227     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7228     my $bootindex_hash = {};
7229     my $i = 1;
7230     foreach my $o (split(//, $boot)) {
7231         $bootindex_hash->{$o} = $i*100;
7232         $i++;
7233     }
7234
7235     my $bootorder = {};
7236
7237     PVE::QemuConfig->foreach_volume($conf, sub {
7238         my ($ds, $drive) = @_;
7239
7240         if (drive_is_cdrom ($drive, 1)) {
7241             if ($bootindex_hash->{d}) {
7242                 $bootorder->{$ds} = $bootindex_hash->{d};
7243                 $bootindex_hash->{d} += 1;
7244             }
7245         } elsif ($bootindex_hash->{c}) {
7246             $bootorder->{$ds} = $bootindex_hash->{c}
7247                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7248             $bootindex_hash->{c} += 1;
7249         }
7250     });
7251
7252     if ($bootindex_hash->{n}) {
7253         for (my $i = 0; $i < $MAX_NETS; $i++) {
7254             my $netname = "net$i";
7255             next if !$conf->{$netname};
7256             $bootorder->{$netname} = $bootindex_hash->{n};
7257             $bootindex_hash->{n} += 1;
7258         }
7259     }
7260
7261     return $bootorder;
7262 }
7263
7264 # Generate default device list for 'boot: order=' property. Matches legacy
7265 # default boot order, but with explicit device names. This is important, since
7266 # the fallback for when neither 'order' nor the old format is specified relies
7267 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7268 sub get_default_bootdevices {
7269     my ($conf) = @_;
7270
7271     my @ret = ();
7272
7273     # harddisk
7274     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7275     push @ret, $first if $first;
7276
7277     # cdrom
7278     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7279     push @ret, $first if $first;
7280
7281     # network
7282     for (my $i = 0; $i < $MAX_NETS; $i++) {
7283         my $netname = "net$i";
7284         next if !$conf->{$netname};
7285         push @ret, $netname;
7286         last;
7287     }
7288
7289     return \@ret;
7290 }
7291
7292 sub device_bootorder {
7293     my ($conf) = @_;
7294
7295     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7296
7297     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7298
7299     my $bootorder = {};
7300     if (!defined($boot) || $boot->{legacy}) {
7301         $bootorder = bootorder_from_legacy($conf, $boot);
7302     } elsif ($boot->{order}) {
7303         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7304         for my $dev (PVE::Tools::split_list($boot->{order})) {
7305             $bootorder->{$dev} = $i++;
7306         }
7307     }
7308
7309     return $bootorder;
7310 }
7311
7312 # bash completion helper
7313
7314 sub complete_backup_archives {
7315     my ($cmdname, $pname, $cvalue) = @_;
7316
7317     my $cfg = PVE::Storage::config();
7318
7319     my $storeid;
7320
7321     if ($cvalue =~ m/^([^:]+):/) {
7322         $storeid = $1;
7323     }
7324
7325     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7326
7327     my $res = [];
7328     foreach my $id (keys %$data) {
7329         foreach my $item (@{$data->{$id}}) {
7330             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7331             push @$res, $item->{volid} if defined($item->{volid});
7332         }
7333     }
7334
7335     return $res;
7336 }
7337
7338 my $complete_vmid_full = sub {
7339     my ($running) = @_;
7340
7341     my $idlist = vmstatus();
7342
7343     my $res = [];
7344
7345     foreach my $id (keys %$idlist) {
7346         my $d = $idlist->{$id};
7347         if (defined($running)) {
7348             next if $d->{template};
7349             next if $running && $d->{status} ne 'running';
7350             next if !$running && $d->{status} eq 'running';
7351         }
7352         push @$res, $id;
7353
7354     }
7355     return $res;
7356 };
7357
7358 sub complete_vmid {
7359     return &$complete_vmid_full();
7360 }
7361
7362 sub complete_vmid_stopped {
7363     return &$complete_vmid_full(0);
7364 }
7365
7366 sub complete_vmid_running {
7367     return &$complete_vmid_full(1);
7368 }
7369
7370 sub complete_storage {
7371
7372     my $cfg = PVE::Storage::config();
7373     my $ids = $cfg->{ids};
7374
7375     my $res = [];
7376     foreach my $sid (keys %$ids) {
7377         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7378         next if !$ids->{$sid}->{content}->{images};
7379         push @$res, $sid;
7380     }
7381
7382     return $res;
7383 }
7384
7385 sub complete_migration_storage {
7386     my ($cmd, $param, $current_value, $all_args) = @_;
7387
7388     my $targetnode = @$all_args[1];
7389
7390     my $cfg = PVE::Storage::config();
7391     my $ids = $cfg->{ids};
7392
7393     my $res = [];
7394     foreach my $sid (keys %$ids) {
7395         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7396         next if !$ids->{$sid}->{content}->{images};
7397         push @$res, $sid;
7398     }
7399
7400     return $res;
7401 }
7402
7403 sub vm_is_paused {
7404     my ($vmid) = @_;
7405     my $qmpstatus = eval {
7406         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7407         mon_cmd($vmid, "query-status");
7408     };
7409     warn "$@\n" if $@;
7410     return $qmpstatus && $qmpstatus->{status} eq "paused";
7411 }
7412
7413 1;