PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 970 dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $usb_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string', format => 'pve-qm-usb-device',
1005         format_description => 'HOSTUSBDEVICE|spice',
1006         description => <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1008
1009  'bus-port(.port)*' (decimal numbers) or
1010  'vendor_id:product_id' (hexadeciaml numbers) or
1011  'spice'
1012
1013 You can use the 'lsusb -t' command to list existing usb devices.
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1017
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1019 EODESCR
1020     },
1021     usb3 => {
1022         optional => 1,
1023         type => 'boolean',
1024         description => "Specifies whether if given host option is a USB3 device or port.",
1025         default => 0,
1026     },
1027 };
1028
1029 my $usbdesc = {
1030     optional => 1,
1031     type => 'string', format => $usb_fmt,
1032     description => "Configure an USB device (n is 0 to 4).",
1033 };
1034 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1035
1036 my $serialdesc = {
1037         optional => 1,
1038         type => 'string',
1039         pattern => '(/dev/.+|socket)',
1040         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1041         verbose_description =>  <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1045
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052
1053 my $paralleldesc= {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1057         description =>  "Map host parallel devices (n is 0 to 2).",
1058         verbose_description =>  <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1078 }
1079
1080 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1081     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1082 }
1083
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1085     $confdesc->{"usb$i"} = $usbdesc;
1086 }
1087
1088 my $boot_fmt = {
1089     legacy => {
1090         optional => 1,
1091         default_key => 1,
1092         type => 'string',
1093         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094                      . " Deprecated, use 'order=' instead.",
1095         pattern => '[acdn]{1,4}',
1096         format_description => "[acdn]{1,4}",
1097
1098         # note: this is also the fallback if boot: is not given at all
1099         default => 'cdn',
1100     },
1101     order => {
1102         optional => 1,
1103         type => 'string',
1104         format => 'pve-qm-bootdev-list',
1105         format_description => "device[;device...]",
1106         description => <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1108
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1112
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1116
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1118 EODESC
1119     },
1120 };
1121 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1122
1123 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1124 sub verify_bootdev {
1125     my ($dev, $noerr) = @_;
1126
1127     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1128
1129     my $check = sub {
1130         my ($base) = @_;
1131         return 0 if $dev !~ m/^$base\d+$/;
1132         return 0 if !$confdesc->{$dev};
1133         return 1;
1134     };
1135
1136     return $dev if $check->("net");
1137     return $dev if $check->("usb");
1138     return $dev if $check->("hostpci");
1139
1140     return if $noerr;
1141     die "invalid boot device '$dev'\n";
1142 }
1143
1144 sub print_bootorder {
1145     my ($devs) = @_;
1146     return "" if !@$devs;
1147     my $data = { order => join(';', @$devs) };
1148     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1149 }
1150
1151 my $kvm_api_version = 0;
1152
1153 sub kvm_version {
1154     return $kvm_api_version if $kvm_api_version;
1155
1156     open my $fh, '<', '/dev/kvm' or return;
1157
1158     # 0xae00 => KVM_GET_API_VERSION
1159     $kvm_api_version = ioctl($fh, 0xae00, 0);
1160     close($fh);
1161
1162     return $kvm_api_version;
1163 }
1164
1165 my $kvm_user_version = {};
1166 my $kvm_mtime = {};
1167
1168 sub kvm_user_version {
1169     my ($binary) = @_;
1170
1171     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1172     my $st = stat($binary);
1173
1174     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176         $cachedmtime == $st->mtime;
1177
1178     $kvm_user_version->{$binary} = 'unknown';
1179     $kvm_mtime->{$binary} = $st->mtime;
1180
1181     my $code = sub {
1182         my $line = shift;
1183         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184             $kvm_user_version->{$binary} = $2;
1185         }
1186     };
1187
1188     eval { run_command([$binary, '--version'], outfunc => $code); };
1189     warn $@ if $@;
1190
1191     return $kvm_user_version->{$binary};
1192
1193 }
1194 my sub extract_version {
1195     my ($machine_type, $version) = @_;
1196     $version = kvm_user_version() if !defined($version);
1197     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1198 }
1199
1200 sub kernel_has_vhost_net {
1201     return -c '/dev/vhost-net';
1202 }
1203
1204 sub option_exists {
1205     my $key = shift;
1206     return defined($confdesc->{$key});
1207 }
1208
1209 my $cdrom_path;
1210 sub get_cdrom_path {
1211
1212     return  $cdrom_path if $cdrom_path;
1213
1214     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1215     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1216     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1217 }
1218
1219 sub get_iso_path {
1220     my ($storecfg, $vmid, $cdrom) = @_;
1221
1222     if ($cdrom eq 'cdrom') {
1223         return get_cdrom_path();
1224     } elsif ($cdrom eq 'none') {
1225         return '';
1226     } elsif ($cdrom =~ m|^/|) {
1227         return $cdrom;
1228     } else {
1229         return PVE::Storage::path($storecfg, $cdrom);
1230     }
1231 }
1232
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id {
1235     my ($vmid, $file, $media) = @_;
1236
1237      if (!($file eq 'none' || $file eq 'cdrom' ||
1238           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1239
1240         return if $file =~ m|/|;
1241
1242         if ($media && $media eq 'cdrom') {
1243             $file = "local:iso/$file";
1244         } else {
1245             $file = "local:$vmid/$file";
1246         }
1247     }
1248
1249     return $file;
1250 }
1251
1252 sub verify_media_type {
1253     my ($opt, $vtype, $media) = @_;
1254
1255     return if !$media;
1256
1257     my $etype;
1258     if ($media eq 'disk') {
1259         $etype = 'images';
1260     } elsif ($media eq 'cdrom') {
1261         $etype = 'iso';
1262     } else {
1263         die "internal error";
1264     }
1265
1266     return if ($vtype eq $etype);
1267
1268     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1269 }
1270
1271 sub cleanup_drive_path {
1272     my ($opt, $storecfg, $drive) = @_;
1273
1274     # try to convert filesystem paths to volume IDs
1275
1276     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1277         ($drive->{file} !~ m|^/dev/.+|) &&
1278         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1279         ($drive->{file} !~ m/^\d+$/)) {
1280         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1281         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1282             if !$vtype;
1283         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1284         verify_media_type($opt, $vtype, $drive->{media});
1285         $drive->{file} = $volid;
1286     }
1287
1288     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1289 }
1290
1291 sub parse_hotplug_features {
1292     my ($data) = @_;
1293
1294     my $res = {};
1295
1296     return $res if $data eq '0';
1297
1298     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1299
1300     foreach my $feature (PVE::Tools::split_list($data)) {
1301         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1302             $res->{$1} = 1;
1303         } else {
1304             die "invalid hotplug feature '$feature'\n";
1305         }
1306     }
1307     return $res;
1308 }
1309
1310 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1311 sub pve_verify_hotplug_features {
1312     my ($value, $noerr) = @_;
1313
1314     return $value if parse_hotplug_features($value);
1315
1316     return if $noerr;
1317
1318     die "unable to parse hotplug option\n";
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub print_tabletdevice_full {
1383     my ($conf, $arch) = @_;
1384
1385     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1386
1387     # we use uhci for old VMs because tablet driver was buggy in older qemu
1388     my $usbbus;
1389     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1390         $usbbus = 'ehci';
1391     } else {
1392         $usbbus = 'uhci';
1393     }
1394
1395     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1396 }
1397
1398 sub print_keyboarddevice_full {
1399     my ($conf, $arch, $machine) = @_;
1400
1401     return if $arch ne 'aarch64';
1402
1403     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1404 }
1405
1406 sub print_drivedevice_full {
1407     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1408
1409     my $device = '';
1410     my $maxdev = 0;
1411
1412     my $drive_id = "$drive->{interface}$drive->{index}";
1413     if ($drive->{interface} eq 'virtio') {
1414         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1415         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1417     } elsif ($drive->{interface} eq 'scsi') {
1418
1419         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1420         my $unit = $drive->{index} % $maxdev;
1421         my $devicetype = 'hd';
1422         my $path = '';
1423         if (drive_is_cdrom($drive)) {
1424             $devicetype = 'cd';
1425         } else {
1426             if ($drive->{file} =~ m|^/|) {
1427                 $path = $drive->{file};
1428                 if (my $info = path_is_scsi($path)) {
1429                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1430                         $devicetype = 'block';
1431                     } elsif ($info->{type} == 1) { # tape
1432                         $devicetype = 'generic';
1433                     }
1434                 }
1435             } else {
1436                  $path = PVE::Storage::path($storecfg, $drive->{file});
1437             }
1438
1439             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440             my $version = extract_version($machine_type, kvm_user_version());
1441             if ($path =~ m/^iscsi\:\/\// &&
1442                !min_version($version, 4, 1)) {
1443                 $devicetype = 'generic';
1444             }
1445         }
1446
1447         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1448             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1449         } else {
1450             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451                 .",lun=$drive->{index}";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456             $device .= ",rotation_rate=1";
1457         }
1458         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1459
1460     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1461         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1462         my $controller = int($drive->{index} / $maxdev);
1463         my $unit = $drive->{index} % $maxdev;
1464         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1465
1466         $device = "ide-$devicetype";
1467         if ($drive->{interface} eq 'ide') {
1468             $device .= ",bus=ide.$controller,unit=$unit";
1469         } else {
1470             $device .= ",bus=ahci$controller.$unit";
1471         }
1472         $device .= ",drive=drive-$drive_id,id=$drive_id";
1473
1474         if ($devicetype eq 'hd') {
1475             if (my $model = $drive->{model}) {
1476                 $model = URI::Escape::uri_unescape($model);
1477                 $device .= ",model=$model";
1478             }
1479             if ($drive->{ssd}) {
1480                 $device .= ",rotation_rate=1";
1481             }
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484     } elsif ($drive->{interface} eq 'usb') {
1485         die "implement me";
1486         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1487     } else {
1488         die "unsupported interface type";
1489     }
1490
1491     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1492
1493     if (my $serial = $drive->{serial}) {
1494         $serial = URI::Escape::uri_unescape($serial);
1495         $device .= ",serial=$serial";
1496     }
1497
1498
1499     return $device;
1500 }
1501
1502 sub get_initiator_name {
1503     my $initiator;
1504
1505     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1506     while (defined(my $line = <$fh>)) {
1507         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1508         $initiator = $1;
1509         last;
1510     }
1511     $fh->close();
1512
1513     return $initiator;
1514 }
1515
1516 sub print_drive_commandline_full {
1517     my ($storecfg, $vmid, $drive) = @_;
1518
1519     my $path;
1520     my $volid = $drive->{file};
1521     my $format;
1522
1523     if (drive_is_cdrom($drive)) {
1524         $path = get_iso_path($storecfg, $vmid, $volid);
1525     } else {
1526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1527         if ($storeid) {
1528             $path = PVE::Storage::path($storecfg, $volid);
1529             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1530             $format = qemu_img_format($scfg, $volname);
1531         } else {
1532             $path = $volid;
1533             $format = "raw";
1534         }
1535    }
1536
1537     my $opts = '';
1538     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539     foreach my $o (@qemu_drive_options) {
1540         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1541     }
1542
1543     # snapshot only accepts on|off
1544     if (defined($drive->{snapshot})) {
1545         my $v = $drive->{snapshot} ? 'on' : 'off';
1546         $opts .= ",snapshot=$v";
1547     }
1548
1549     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1550         my ($dir, $qmpname) = @$type;
1551         if (my $v = $drive->{"mbps$dir"}) {
1552             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1553         }
1554         if (my $v = $drive->{"mbps${dir}_max"}) {
1555             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1556         }
1557         if (my $v = $drive->{"bps${dir}_max_length"}) {
1558             $opts .= ",throttling.bps$qmpname-max-length=$v";
1559         }
1560         if (my $v = $drive->{"iops${dir}"}) {
1561             $opts .= ",throttling.iops$qmpname=$v";
1562         }
1563         if (my $v = $drive->{"iops${dir}_max"}) {
1564             $opts .= ",throttling.iops$qmpname-max=$v";
1565         }
1566         if (my $v = $drive->{"iops${dir}_max_length"}) {
1567             $opts .= ",throttling.iops$qmpname-max-length=$v";
1568         }
1569     }
1570
1571     $opts .= ",format=$format" if $format && !$drive->{format};
1572
1573     my $cache_direct = 0;
1574
1575     if (my $cache = $drive->{cache}) {
1576         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577     } elsif (!drive_is_cdrom($drive)) {
1578         $opts .= ",cache=none";
1579         $cache_direct = 1;
1580     }
1581
1582     # aio native works only with O_DIRECT
1583     if (!$drive->{aio}) {
1584         if($cache_direct) {
1585             $opts .= ",aio=native";
1586         } else {
1587             $opts .= ",aio=threads";
1588         }
1589     }
1590
1591     if (!drive_is_cdrom($drive)) {
1592         my $detectzeroes;
1593         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1594             $detectzeroes = 'off';
1595         } elsif ($drive->{discard}) {
1596             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1597         } else {
1598             # This used to be our default with discard not being specified:
1599             $detectzeroes = 'on';
1600         }
1601         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1602     }
1603
1604     my $pathinfo = $path ? "file=$path," : '';
1605
1606     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1607 }
1608
1609 sub print_netdevice_full {
1610     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1611
1612     my $device = $net->{model};
1613     if ($net->{model} eq 'virtio') {
1614          $device = 'virtio-net-pci';
1615      };
1616
1617     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1618     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1620         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621         # and out of each queue plus one config interrupt and control vector queue
1622         my $vectors = $net->{queues} * 2 + 2;
1623         $tmpstr .= ",vectors=$vectors,mq=on";
1624     }
1625     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1626
1627     if (my $mtu = $net->{mtu}) {
1628         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1629             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1630             if ($mtu == 1) {
1631                  $mtu = $bridge_mtu;
1632             } elsif ($mtu < 576) {
1633                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634             } elsif ($mtu > $bridge_mtu) {
1635                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1636             }
1637             $tmpstr .= ",host_mtu=$mtu";
1638         } else {
1639             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1640         }
1641     }
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     if (is_native($arch)) {
1680         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1681     }
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1691     } else {
1692         $netdev = "type=user,id=$netid,hostname=$vmname";
1693     }
1694
1695     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1696
1697     return $netdev;
1698 }
1699
1700 my $vga_map = {
1701     'cirrus' => 'cirrus-vga',
1702     'std' => 'VGA',
1703     'vmware' => 'vmware-svga',
1704     'virtio' => 'virtio-vga',
1705 };
1706
1707 sub print_vga_device {
1708     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1709
1710     my $type = $vga_map->{$vga->{type}};
1711     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712         $type = 'virtio-gpu';
1713     }
1714     my $vgamem_mb = $vga->{memory};
1715
1716     my $max_outputs = '';
1717     if ($qxlnum) {
1718         $type = $id ? 'qxl' : 'qxl-vga';
1719
1720         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1721             # set max outputs so linux can have up to 4 qxl displays with one device
1722             if (min_version($machine_version, 4, 1)) {
1723                 $max_outputs = ",max_outputs=4";
1724             }
1725         }
1726     }
1727
1728     die "no devicetype for $vga->{type}\n" if !$type;
1729
1730     my $memory = "";
1731     if ($vgamem_mb) {
1732         if ($vga->{type} eq 'virtio') {
1733             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1734             $memory = ",max_hostmem=$bytes";
1735         } elsif ($qxlnum) {
1736             # from https://www.spice-space.org/multiple-monitors.html
1737             $memory = ",vgamem_mb=$vga->{memory}";
1738             my $ram = $vgamem_mb * 4;
1739             my $vram = $vgamem_mb * 2;
1740             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1741         } else {
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743         }
1744     } elsif ($qxlnum && $id) {
1745         $memory = ",ram_size=67108864,vram_size=33554432";
1746     }
1747
1748     my $edidoff = "";
1749     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1750         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1751     }
1752
1753     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1754     my $vgaid = "vga" . ($id // '');
1755     my $pciaddr;
1756
1757     if ($q35 && $vgaid eq 'vga') {
1758         # the first display uses pcie.0 bus on q35 machines
1759         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1760     } else {
1761         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1762     }
1763
1764     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1765 }
1766
1767 sub parse_number_sets {
1768     my ($set) = @_;
1769     my $res = [];
1770     foreach my $part (split(/;/, $set)) {
1771         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773             push @$res, [ $1, $2 ];
1774         } else {
1775             die "invalid range: $part\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 sub parse_numa {
1782     my ($data) = @_;
1783
1784     my $res = parse_property_string($numa_fmt, $data);
1785     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1786     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1787     return $res;
1788 }
1789
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1791 sub parse_net {
1792     my ($data) = @_;
1793
1794     my $res = eval { parse_property_string($net_fmt, $data) };
1795     if ($@) {
1796         warn $@;
1797         return;
1798     }
1799     if (!defined($res->{macaddr})) {
1800         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1801         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1802     }
1803     return $res;
1804 }
1805
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig {
1808     my ($data) = @_;
1809
1810     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1811     if ($@) {
1812         warn $@;
1813         return;
1814     }
1815
1816     if ($res->{gw} && !$res->{ip}) {
1817         warn 'gateway specified without specifying an IP address';
1818         return;
1819     }
1820     if ($res->{gw6} && !$res->{ip6}) {
1821         warn 'IPv6 gateway specified without specifying an IPv6 address';
1822         return;
1823     }
1824     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1825         warn 'gateway specified together with DHCP';
1826         return;
1827     }
1828     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1829         # gw6 + auto/dhcp
1830         warn "IPv6 gateway specified together with $res->{ip6} address";
1831         return;
1832     }
1833
1834     if (!$res->{ip} && !$res->{ip6}) {
1835         return { ip => 'dhcp', ip6 => 'dhcp' };
1836     }
1837
1838     return $res;
1839 }
1840
1841 sub print_net {
1842     my $net = shift;
1843
1844     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1845 }
1846
1847 sub add_random_macs {
1848     my ($settings) = @_;
1849
1850     foreach my $opt (keys %$settings) {
1851         next if $opt !~ m/^net(\d+)$/;
1852         my $net = parse_net($settings->{$opt});
1853         next if !$net;
1854         $settings->{$opt} = print_net($net);
1855     }
1856 }
1857
1858 sub vm_is_volid_owner {
1859     my ($storecfg, $vmid, $volid) = @_;
1860
1861     if ($volid !~  m|^/|) {
1862         my ($path, $owner);
1863         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1864         if ($owner && ($owner == $vmid)) {
1865             return 1;
1866         }
1867     }
1868
1869     return;
1870 }
1871
1872 sub vmconfig_register_unused_drive {
1873     my ($storecfg, $vmid, $conf, $drive) = @_;
1874
1875     if (drive_is_cloudinit($drive)) {
1876         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1877         warn $@ if $@;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         my $volid = $drive->{file};
1880         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1881             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1882         }
1883     }
1884 }
1885
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1887 my $smbios1_fmt = {
1888     uuid => {
1889         type => 'string',
1890         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891         format_description => 'UUID',
1892         description => "Set SMBIOS1 UUID.",
1893         optional => 1,
1894     },
1895     version => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 version.",
1900         optional => 1,
1901     },
1902     serial => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 serial number.",
1907         optional => 1,
1908     },
1909     manufacturer => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 manufacturer.",
1914         optional => 1,
1915     },
1916     product => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 product ID.",
1921         optional => 1,
1922     },
1923     sku => {
1924         type => 'string',
1925         pattern => '[A-Za-z0-9+\/]+={0,2}',
1926         format_description => 'Base64 encoded string',
1927         description => "Set SMBIOS1 SKU string.",
1928         optional => 1,
1929     },
1930     family => {
1931         type => 'string',
1932         pattern => '[A-Za-z0-9+\/]+={0,2}',
1933         format_description => 'Base64 encoded string',
1934         description => "Set SMBIOS1 family string.",
1935         optional => 1,
1936     },
1937     base64 => {
1938         type => 'boolean',
1939         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1940         optional => 1,
1941     },
1942 };
1943
1944 sub parse_smbios1 {
1945     my ($data) = @_;
1946
1947     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1948     warn $@ if $@;
1949     return $res;
1950 }
1951
1952 sub print_smbios1 {
1953     my ($smbios1) = @_;
1954     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1958
1959 sub parse_watchdog {
1960     my ($value) = @_;
1961
1962     return if !$value;
1963
1964     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1965     warn $@ if $@;
1966     return $res;
1967 }
1968
1969 sub parse_guest_agent {
1970     my ($conf) = @_;
1971
1972     return {} if !defined($conf->{agent});
1973
1974     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1975     warn $@ if $@;
1976
1977     # if the agent is disabled ignore the other potentially set properties
1978     return {} if !$res->{enabled};
1979     return $res;
1980 }
1981
1982 sub get_qga_key {
1983     my ($conf, $key) = @_;
1984     return undef if !defined($conf->{agent});
1985
1986     my $agent = parse_guest_agent($conf);
1987     return $agent->{$key};
1988 }
1989
1990 sub parse_vga {
1991     my ($value) = @_;
1992
1993     return {} if !$value;
1994     my $res = eval { parse_property_string($vga_fmt, $value) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub parse_rng {
2000     my ($value) = @_;
2001
2002     return if !$value;
2003
2004     my $res = eval { parse_property_string($rng_fmt, $value) };
2005     warn $@ if $@;
2006     return $res;
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2010 sub verify_usb_device {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if parse_usb_device($value);
2014
2015     return if $noerr;
2016
2017     die "unable to parse usb device\n";
2018 }
2019
2020 # add JSON properties for create and set function
2021 sub json_config_properties {
2022     my $prop = shift;
2023
2024     foreach my $opt (keys %$confdesc) {
2025         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027         $prop->{$opt} = $confdesc->{$opt};
2028     }
2029
2030     return $prop;
2031 }
2032
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties {
2035
2036     return dclone($confdesc_cloudinit);
2037 }
2038
2039 sub check_type {
2040     my ($key, $value) = @_;
2041
2042     die "unknown setting '$key'\n" if !$confdesc->{$key};
2043
2044     my $type = $confdesc->{$key}->{type};
2045
2046     if (!defined($value)) {
2047         die "got undefined value\n";
2048     }
2049
2050     if ($value =~ m/[\n\r]/) {
2051         die "property contains a line feed\n";
2052     }
2053
2054     if ($type eq 'boolean') {
2055         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057         die "type check ('boolean') failed - got '$value'\n";
2058     } elsif ($type eq 'integer') {
2059         return int($1) if $value =~ m/^(\d+)$/;
2060         die "type check ('integer') failed - got '$value'\n";
2061     } elsif ($type eq 'number') {
2062         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063         die "type check ('number') failed - got '$value'\n";
2064     } elsif ($type eq 'string') {
2065         if (my $fmt = $confdesc->{$key}->{format}) {
2066             PVE::JSONSchema::check_format($fmt, $value);
2067             return $value;
2068         }
2069         $value =~ s/^\"(.*)\"$/$1/;
2070         return $value;
2071     } else {
2072         die "internal error"
2073     }
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     if ($conf->{template}) {
2084         # check if any base image is still used by a linked clone
2085         PVE::QemuConfig->foreach_volume($conf, sub {
2086                 my ($ds, $drive) = @_;
2087                 return if drive_is_cdrom($drive);
2088
2089                 my $volid = $drive->{file};
2090                 return if !$volid || $volid =~ m|^/|;
2091
2092                 die "base volume '$volid' is still in use by linked cloned\n"
2093                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2094
2095         });
2096     }
2097
2098     # only remove disks owned by this VM (referenced in the config)
2099     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2100         my ($ds, $drive) = @_;
2101         return if drive_is_cdrom($drive, 1);
2102
2103         my $volid = $drive->{file};
2104         return if !$volid || $volid =~ m|^/|;
2105
2106         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2107         return if !$path || !$owner || ($owner != $vmid);
2108
2109         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2110         warn "Could not remove disk '$volid', check manually: $@" if $@;
2111     });
2112
2113     if ($purge_unreferenced) { # also remove unreferenced disk
2114         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2115         PVE::Storage::foreach_volid($vmdisks, sub {
2116             my ($volid, $sid, $volname, $d) = @_;
2117             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2118             warn $@ if $@;
2119         });
2120     }
2121
2122     if (defined $replacement_conf) {
2123         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2124     } else {
2125         PVE::QemuConfig->destroy_config($vmid);
2126     }
2127 }
2128
2129 sub parse_vm_config {
2130     my ($filename, $raw) = @_;
2131
2132     return if !defined($raw);
2133
2134     my $res = {
2135         digest => Digest::SHA::sha1_hex($raw),
2136         snapshots => {},
2137         pending => {},
2138     };
2139
2140     $filename =~ m|/qemu-server/(\d+)\.conf$|
2141         || die "got strange filename '$filename'";
2142
2143     my $vmid = $1;
2144
2145     my $conf = $res;
2146     my $descr;
2147     my $section = '';
2148
2149     my @lines = split(/\n/, $raw);
2150     foreach my $line (@lines) {
2151         next if $line =~ m/^\s*$/;
2152
2153         if ($line =~ m/^\[PENDING\]\s*$/i) {
2154             $section = 'pending';
2155             if (defined($descr)) {
2156                 $descr =~ s/\s+$//;
2157                 $conf->{description} = $descr;
2158             }
2159             $descr = undef;
2160             $conf = $res->{$section} = {};
2161             next;
2162
2163         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2164             $section = $1;
2165             if (defined($descr)) {
2166                 $descr =~ s/\s+$//;
2167                 $conf->{description} = $descr;
2168             }
2169             $descr = undef;
2170             $conf = $res->{snapshots}->{$section} = {};
2171             next;
2172         }
2173
2174         if ($line =~ m/^\#(.*)\s*$/) {
2175             $descr = '' if !defined($descr);
2176             $descr .= PVE::Tools::decode_text($1) . "\n";
2177             next;
2178         }
2179
2180         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181             $descr = '' if !defined($descr);
2182             $descr .= PVE::Tools::decode_text($2);
2183         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184             $conf->{snapstate} = $1;
2185         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2186             my $key = $1;
2187             my $value = $2;
2188             $conf->{$key} = $value;
2189         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2190             my $value = $1;
2191             if ($section eq 'pending') {
2192                 $conf->{delete} = $value; # we parse this later
2193             } else {
2194                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2195             }
2196         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2197             my $key = $1;
2198             my $value = $2;
2199             eval { $value = check_type($key, $value); };
2200             if ($@) {
2201                 warn "vm $vmid - unable to parse value of '$key' - $@";
2202             } else {
2203                 $key = 'ide2' if $key eq 'cdrom';
2204                 my $fmt = $confdesc->{$key}->{format};
2205                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206                     my $v = parse_drive($key, $value);
2207                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2208                         $v->{file} = $volid;
2209                         $value = print_drive($v);
2210                     } else {
2211                         warn "vm $vmid - unable to parse value of '$key'\n";
2212                         next;
2213                     }
2214                 }
2215
2216                 $conf->{$key} = $value;
2217             }
2218         } else {
2219             warn "vm $vmid - unable to parse config: $line\n";
2220         }
2221     }
2222
2223     if (defined($descr)) {
2224         $descr =~ s/\s+$//;
2225         $conf->{description} = $descr;
2226     }
2227     delete $res->{snapstate}; # just to be sure
2228
2229     return $res;
2230 }
2231
2232 sub write_vm_config {
2233     my ($filename, $conf) = @_;
2234
2235     delete $conf->{snapstate}; # just to be sure
2236
2237     if ($conf->{cdrom}) {
2238         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2239         $conf->{ide2} = $conf->{cdrom};
2240         delete $conf->{cdrom};
2241     }
2242
2243     # we do not use 'smp' any longer
2244     if ($conf->{sockets}) {
2245         delete $conf->{smp};
2246     } elsif ($conf->{smp}) {
2247         $conf->{sockets} = $conf->{smp};
2248         delete $conf->{cores};
2249         delete $conf->{smp};
2250     }
2251
2252     my $used_volids = {};
2253
2254     my $cleanup_config = sub {
2255         my ($cref, $pending, $snapname) = @_;
2256
2257         foreach my $key (keys %$cref) {
2258             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2259                 $key eq 'snapstate' || $key eq 'pending';
2260             my $value = $cref->{$key};
2261             if ($key eq 'delete') {
2262                 die "propertry 'delete' is only allowed in [PENDING]\n"
2263                     if !$pending;
2264                 # fixme: check syntax?
2265                 next;
2266             }
2267             eval { $value = check_type($key, $value); };
2268             die "unable to parse value of '$key' - $@" if $@;
2269
2270             $cref->{$key} = $value;
2271
2272             if (!$snapname && is_valid_drivename($key)) {
2273                 my $drive = parse_drive($key, $value);
2274                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2275             }
2276         }
2277     };
2278
2279     &$cleanup_config($conf);
2280
2281     &$cleanup_config($conf->{pending}, 1);
2282
2283     foreach my $snapname (keys %{$conf->{snapshots}}) {
2284         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2285         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2286     }
2287
2288     # remove 'unusedX' settings if we re-add a volume
2289     foreach my $key (keys %$conf) {
2290         my $value = $conf->{$key};
2291         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2292             delete $conf->{$key};
2293         }
2294     }
2295
2296     my $generate_raw_config = sub {
2297         my ($conf, $pending) = @_;
2298
2299         my $raw = '';
2300
2301         # add description as comment to top of file
2302         if (defined(my $descr = $conf->{description})) {
2303             if ($descr) {
2304                 foreach my $cl (split(/\n/, $descr)) {
2305                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2306                 }
2307             } else {
2308                 $raw .= "#\n" if $pending;
2309             }
2310         }
2311
2312         foreach my $key (sort keys %$conf) {
2313             next if $key =~ /^(digest|description|pending|snapshots)$/;
2314             $raw .= "$key: $conf->{$key}\n";
2315         }
2316         return $raw;
2317     };
2318
2319     my $raw = &$generate_raw_config($conf);
2320
2321     if (scalar(keys %{$conf->{pending}})){
2322         $raw .= "\n[PENDING]\n";
2323         $raw .= &$generate_raw_config($conf->{pending}, 1);
2324     }
2325
2326     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2327         $raw .= "\n[$snapname]\n";
2328         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2329     }
2330
2331     return $raw;
2332 }
2333
2334 sub load_defaults {
2335
2336     my $res = {};
2337
2338     # we use static defaults from our JSON schema configuration
2339     foreach my $key (keys %$confdesc) {
2340         if (defined(my $default = $confdesc->{$key}->{default})) {
2341             $res->{$key} = $default;
2342         }
2343     }
2344
2345     return $res;
2346 }
2347
2348 sub config_list {
2349     my $vmlist = PVE::Cluster::get_vmlist();
2350     my $res = {};
2351     return $res if !$vmlist || !$vmlist->{ids};
2352     my $ids = $vmlist->{ids};
2353     my $nodename = nodename();
2354
2355     foreach my $vmid (keys %$ids) {
2356         my $d = $ids->{$vmid};
2357         next if !$d->{node} || $d->{node} ne $nodename;
2358         next if !$d->{type} || $d->{type} ne 'qemu';
2359         $res->{$vmid}->{exists} = 1;
2360     }
2361     return $res;
2362 }
2363
2364 # test if VM uses local resources (to prevent migration)
2365 sub check_local_resources {
2366     my ($conf, $noerr) = @_;
2367
2368     my @loc_res = ();
2369
2370     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2371     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2372
2373     push @loc_res, "ivshmem" if $conf->{ivshmem};
2374
2375     foreach my $k (keys %$conf) {
2376         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2377         # sockets are safe: they will recreated be on the target side post-migrate
2378         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2379         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2380     }
2381
2382     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2383
2384     return \@loc_res;
2385 }
2386
2387 # check if used storages are available on all nodes (use by migrate)
2388 sub check_storage_availability {
2389     my ($storecfg, $conf, $node) = @_;
2390
2391     PVE::QemuConfig->foreach_volume($conf, sub {
2392         my ($ds, $drive) = @_;
2393
2394         my $volid = $drive->{file};
2395         return if !$volid;
2396
2397         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2398         return if !$sid;
2399
2400         # check if storage is available on both nodes
2401         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2402         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2403    });
2404 }
2405
2406 # list nodes where all VM images are available (used by has_feature API)
2407 sub shared_nodes {
2408     my ($conf, $storecfg) = @_;
2409
2410     my $nodelist = PVE::Cluster::get_nodelist();
2411     my $nodehash = { map { $_ => 1 } @$nodelist };
2412     my $nodename = nodename();
2413
2414     PVE::QemuConfig->foreach_volume($conf, sub {
2415         my ($ds, $drive) = @_;
2416
2417         my $volid = $drive->{file};
2418         return if !$volid;
2419
2420         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2421         if ($storeid) {
2422             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2423             if ($scfg->{disable}) {
2424                 $nodehash = {};
2425             } elsif (my $avail = $scfg->{nodes}) {
2426                 foreach my $node (keys %$nodehash) {
2427                     delete $nodehash->{$node} if !$avail->{$node};
2428                 }
2429             } elsif (!$scfg->{shared}) {
2430                 foreach my $node (keys %$nodehash) {
2431                     delete $nodehash->{$node} if $node ne $nodename
2432                 }
2433             }
2434         }
2435     });
2436
2437     return $nodehash
2438 }
2439
2440 sub check_local_storage_availability {
2441     my ($conf, $storecfg) = @_;
2442
2443     my $nodelist = PVE::Cluster::get_nodelist();
2444     my $nodehash = { map { $_ => {} } @$nodelist };
2445
2446     PVE::QemuConfig->foreach_volume($conf, sub {
2447         my ($ds, $drive) = @_;
2448
2449         my $volid = $drive->{file};
2450         return if !$volid;
2451
2452         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2453         if ($storeid) {
2454             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2455
2456             if ($scfg->{disable}) {
2457                 foreach my $node (keys %$nodehash) {
2458                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2459                 }
2460             } elsif (my $avail = $scfg->{nodes}) {
2461                 foreach my $node (keys %$nodehash) {
2462                     if (!$avail->{$node}) {
2463                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2464                     }
2465                 }
2466             }
2467         }
2468     });
2469
2470     foreach my $node (values %$nodehash) {
2471         if (my $unavail = $node->{unavailable_storages}) {
2472             $node->{unavailable_storages} = [ sort keys %$unavail ];
2473         }
2474     }
2475
2476     return $nodehash
2477 }
2478
2479 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2480 sub check_running {
2481     my ($vmid, $nocheck, $node) = @_;
2482
2483     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2484     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2485 }
2486
2487 sub vzlist {
2488
2489     my $vzlist = config_list();
2490
2491     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2492
2493     while (defined(my $de = $fd->read)) {
2494         next if $de !~ m/^(\d+)\.pid$/;
2495         my $vmid = $1;
2496         next if !defined($vzlist->{$vmid});
2497         if (my $pid = check_running($vmid)) {
2498             $vzlist->{$vmid}->{pid} = $pid;
2499         }
2500     }
2501
2502     return $vzlist;
2503 }
2504
2505 our $vmstatus_return_properties = {
2506     vmid => get_standard_option('pve-vmid'),
2507     status => {
2508         description => "Qemu process status.",
2509         type => 'string',
2510         enum => ['stopped', 'running'],
2511     },
2512     maxmem => {
2513         description => "Maximum memory in bytes.",
2514         type => 'integer',
2515         optional => 1,
2516         renderer => 'bytes',
2517     },
2518     maxdisk => {
2519         description => "Root disk size in bytes.",
2520         type => 'integer',
2521         optional => 1,
2522         renderer => 'bytes',
2523     },
2524     name => {
2525         description => "VM name.",
2526         type => 'string',
2527         optional => 1,
2528     },
2529     qmpstatus => {
2530         description => "Qemu QMP agent status.",
2531         type => 'string',
2532         optional => 1,
2533     },
2534     pid => {
2535         description => "PID of running qemu process.",
2536         type => 'integer',
2537         optional => 1,
2538     },
2539     uptime => {
2540         description => "Uptime.",
2541         type => 'integer',
2542         optional => 1,
2543         renderer => 'duration',
2544     },
2545     cpus => {
2546         description => "Maximum usable CPUs.",
2547         type => 'number',
2548         optional => 1,
2549     },
2550     lock => {
2551         description => "The current config lock, if any.",
2552         type => 'string',
2553         optional => 1,
2554     },
2555     tags => {
2556         description  => "The current configured tags, if any",
2557         type => 'string',
2558         optional => 1,
2559     },
2560 };
2561
2562 my $last_proc_pid_stat;
2563
2564 # get VM status information
2565 # This must be fast and should not block ($full == false)
2566 # We only query KVM using QMP if $full == true (this can be slow)
2567 sub vmstatus {
2568     my ($opt_vmid, $full) = @_;
2569
2570     my $res = {};
2571
2572     my $storecfg = PVE::Storage::config();
2573
2574     my $list = vzlist();
2575     my $defaults = load_defaults();
2576
2577     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2578
2579     my $cpucount = $cpuinfo->{cpus} || 1;
2580
2581     foreach my $vmid (keys %$list) {
2582         next if $opt_vmid && ($vmid ne $opt_vmid);
2583
2584         my $conf = PVE::QemuConfig->load_config($vmid);
2585
2586         my $d = { vmid => $vmid };
2587         $d->{pid} = $list->{$vmid}->{pid};
2588
2589         # fixme: better status?
2590         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2591
2592         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2593         if (defined($size)) {
2594             $d->{disk} = 0; # no info available
2595             $d->{maxdisk} = $size;
2596         } else {
2597             $d->{disk} = 0;
2598             $d->{maxdisk} = 0;
2599         }
2600
2601         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2602             * ($conf->{cores} || $defaults->{cores});
2603         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2604         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2605
2606         $d->{name} = $conf->{name} || "VM $vmid";
2607         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2608             : $defaults->{memory}*(1024*1024);
2609
2610         if ($conf->{balloon}) {
2611             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2612             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2613                 : $defaults->{shares};
2614         }
2615
2616         $d->{uptime} = 0;
2617         $d->{cpu} = 0;
2618         $d->{mem} = 0;
2619
2620         $d->{netout} = 0;
2621         $d->{netin} = 0;
2622
2623         $d->{diskread} = 0;
2624         $d->{diskwrite} = 0;
2625
2626         $d->{template} = PVE::QemuConfig->is_template($conf);
2627
2628         $d->{serial} = 1 if conf_has_serial($conf);
2629         $d->{lock} = $conf->{lock} if $conf->{lock};
2630         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2631
2632         $res->{$vmid} = $d;
2633     }
2634
2635     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2636     foreach my $dev (keys %$netdev) {
2637         next if $dev !~ m/^tap([1-9]\d*)i/;
2638         my $vmid = $1;
2639         my $d = $res->{$vmid};
2640         next if !$d;
2641
2642         $d->{netout} += $netdev->{$dev}->{receive};
2643         $d->{netin} += $netdev->{$dev}->{transmit};
2644
2645         if ($full) {
2646             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2647             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2648         }
2649
2650     }
2651
2652     my $ctime = gettimeofday;
2653
2654     foreach my $vmid (keys %$list) {
2655
2656         my $d = $res->{$vmid};
2657         my $pid = $d->{pid};
2658         next if !$pid;
2659
2660         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2661         next if !$pstat; # not running
2662
2663         my $used = $pstat->{utime} + $pstat->{stime};
2664
2665         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2666
2667         if ($pstat->{vsize}) {
2668             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2669         }
2670
2671         my $old = $last_proc_pid_stat->{$pid};
2672         if (!$old) {
2673             $last_proc_pid_stat->{$pid} = {
2674                 time => $ctime,
2675                 used => $used,
2676                 cpu => 0,
2677             };
2678             next;
2679         }
2680
2681         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2682
2683         if ($dtime > 1000) {
2684             my $dutime = $used -  $old->{used};
2685
2686             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2687             $last_proc_pid_stat->{$pid} = {
2688                 time => $ctime,
2689                 used => $used,
2690                 cpu => $d->{cpu},
2691             };
2692         } else {
2693             $d->{cpu} = $old->{cpu};
2694         }
2695     }
2696
2697     return $res if !$full;
2698
2699     my $qmpclient = PVE::QMPClient->new();
2700
2701     my $ballooncb = sub {
2702         my ($vmid, $resp) = @_;
2703
2704         my $info = $resp->{'return'};
2705         return if !$info->{max_mem};
2706
2707         my $d = $res->{$vmid};
2708
2709         # use memory assigned to VM
2710         $d->{maxmem} = $info->{max_mem};
2711         $d->{balloon} = $info->{actual};
2712
2713         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2714             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2715             $d->{freemem} = $info->{free_mem};
2716         }
2717
2718         $d->{ballooninfo} = $info;
2719     };
2720
2721     my $blockstatscb = sub {
2722         my ($vmid, $resp) = @_;
2723         my $data = $resp->{'return'} || [];
2724         my $totalrdbytes = 0;
2725         my $totalwrbytes = 0;
2726
2727         for my $blockstat (@$data) {
2728             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2729             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2730
2731             $blockstat->{device} =~ s/drive-//;
2732             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2733         }
2734         $res->{$vmid}->{diskread} = $totalrdbytes;
2735         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2736     };
2737
2738     my $statuscb = sub {
2739         my ($vmid, $resp) = @_;
2740
2741         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2742         # this fails if ballon driver is not loaded, so this must be
2743         # the last commnand (following command are aborted if this fails).
2744         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2745
2746         my $status = 'unknown';
2747         if (!defined($status = $resp->{'return'}->{status})) {
2748             warn "unable to get VM status\n";
2749             return;
2750         }
2751
2752         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2753     };
2754
2755     foreach my $vmid (keys %$list) {
2756         next if $opt_vmid && ($vmid ne $opt_vmid);
2757         next if !$res->{$vmid}->{pid}; # not running
2758         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2759     }
2760
2761     $qmpclient->queue_execute(undef, 2);
2762
2763     foreach my $vmid (keys %$list) {
2764         next if $opt_vmid && ($vmid ne $opt_vmid);
2765         next if !$res->{$vmid}->{pid}; #not running
2766
2767         # we can't use the $qmpclient since it might have already aborted on
2768         # 'query-balloon', but this might also fail for older versions...
2769         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2770         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2771     }
2772
2773     foreach my $vmid (keys %$list) {
2774         next if $opt_vmid && ($vmid ne $opt_vmid);
2775         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2776     }
2777
2778     return $res;
2779 }
2780
2781 sub conf_has_serial {
2782     my ($conf) = @_;
2783
2784     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2785         if ($conf->{"serial$i"}) {
2786             return 1;
2787         }
2788     }
2789
2790     return 0;
2791 }
2792
2793 sub conf_has_audio {
2794     my ($conf, $id) = @_;
2795
2796     $id //= 0;
2797     my $audio = $conf->{"audio$id"};
2798     return if !defined($audio);
2799
2800     my $audioproperties = parse_property_string($audio_fmt, $audio);
2801     my $audiodriver = $audioproperties->{driver} // 'spice';
2802
2803     return {
2804         dev => $audioproperties->{device},
2805         dev_id => "audiodev$id",
2806         backend => $audiodriver,
2807         backend_id => "$audiodriver-backend${id}",
2808     };
2809 }
2810
2811 sub audio_devs {
2812     my ($audio, $audiopciaddr, $machine_version) = @_;
2813
2814     my $devs = [];
2815
2816     my $id = $audio->{dev_id};
2817     my $audiodev = "";
2818     if (min_version($machine_version, 4, 2)) {
2819         $audiodev = ",audiodev=$audio->{backend_id}";
2820     }
2821
2822     if ($audio->{dev} eq 'AC97') {
2823         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2824     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2825         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2826         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2827         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2828     } else {
2829         die "unkown audio device '$audio->{dev}', implement me!";
2830     }
2831
2832     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2833
2834     return $devs;
2835 }
2836
2837 sub vga_conf_has_spice {
2838     my ($vga) = @_;
2839
2840     my $vgaconf = parse_vga($vga);
2841     my $vgatype = $vgaconf->{type};
2842     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2843
2844     return $1 || 1;
2845 }
2846
2847 sub is_native($) {
2848     my ($arch) = @_;
2849     return get_host_arch() eq $arch;
2850 }
2851
2852 sub get_vm_arch {
2853     my ($conf) = @_;
2854     return $conf->{arch} // get_host_arch();
2855 }
2856
2857 my $default_machines = {
2858     x86_64 => 'pc',
2859     aarch64 => 'virt',
2860 };
2861
2862 sub get_vm_machine {
2863     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2864
2865     my $machine = $forcemachine || $conf->{machine};
2866
2867     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2868         $arch //= 'x86_64';
2869         $machine ||= $default_machines->{$arch};
2870         if ($add_pve_version) {
2871             $kvmversion //= kvm_user_version();
2872             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2873             $machine .= "+pve$pvever";
2874         }
2875     }
2876
2877     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2878         # for version-pinned machines that do not include a pve-version (e.g.
2879         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2880         $machine .= '+pve0';
2881     }
2882
2883     return $machine;
2884 }
2885
2886 sub get_ovmf_files($) {
2887     my ($arch) = @_;
2888
2889     my $ovmf = $OVMF->{$arch}
2890         or die "no OVMF images known for architecture '$arch'\n";
2891
2892     return @$ovmf;
2893 }
2894
2895 my $Arch2Qemu = {
2896     aarch64 => '/usr/bin/qemu-system-aarch64',
2897     x86_64 => '/usr/bin/qemu-system-x86_64',
2898 };
2899 sub get_command_for_arch($) {
2900     my ($arch) = @_;
2901     return '/usr/bin/kvm' if is_native($arch);
2902
2903     my $cmd = $Arch2Qemu->{$arch}
2904         or die "don't know how to emulate architecture '$arch'\n";
2905     return $cmd;
2906 }
2907
2908 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2909 # to use in a QEMU command line (-cpu element), first array_intersect the result
2910 # of query_supported_ with query_understood_. This is necessary because:
2911 #
2912 # a) query_understood_ returns flags the host cannot use and
2913 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2914 #    flags, but CPU settings - with most of them being flags. Those settings
2915 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2916 #
2917 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2918 # expensive. If you need the value returned from this, you can get it much
2919 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2920 # $accel being 'kvm' or 'tcg'.
2921 #
2922 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2923 # changes, automatically populating pmxcfs.
2924 #
2925 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2926 # since kvm and tcg machines support different flags
2927 #
2928 sub query_supported_cpu_flags {
2929     my ($arch) = @_;
2930
2931     $arch //= get_host_arch();
2932     my $default_machine = $default_machines->{$arch};
2933
2934     my $flags = {};
2935
2936     # FIXME: Once this is merged, the code below should work for ARM as well:
2937     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2938     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2939         $arch eq "aarch64";
2940
2941     my $kvm_supported = defined(kvm_version());
2942     my $qemu_cmd = get_command_for_arch($arch);
2943     my $fakevmid = -1;
2944     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2945
2946     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2947     my $query_supported_run_qemu = sub {
2948         my ($kvm) = @_;
2949
2950         my $flags = {};
2951         my $cmd = [
2952             $qemu_cmd,
2953             '-machine', $default_machine,
2954             '-display', 'none',
2955             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2956             '-mon', 'chardev=qmp,mode=control',
2957             '-pidfile', $pidfile,
2958             '-S', '-daemonize'
2959         ];
2960
2961         if (!$kvm) {
2962             push @$cmd, '-accel', 'tcg';
2963         }
2964
2965         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2966         die "QEMU flag querying VM exited with code " . $rc if $rc;
2967
2968         eval {
2969             my $cmd_result = mon_cmd(
2970                 $fakevmid,
2971                 'query-cpu-model-expansion',
2972                 type => 'full',
2973                 model => { name => 'host' }
2974             );
2975
2976             my $props = $cmd_result->{model}->{props};
2977             foreach my $prop (keys %$props) {
2978                 next if $props->{$prop} ne '1';
2979                 # QEMU returns some flags multiple times, with '_', '.' or '-'
2980                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
2981                 # We only keep those with underscores, to match /proc/cpuinfo
2982                 $prop =~ s/\.|-/_/g;
2983                 $flags->{$prop} = 1;
2984             }
2985         };
2986         my $err = $@;
2987
2988         # force stop with 10 sec timeout and 'nocheck'
2989         # always stop, even if QMP failed
2990         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
2991
2992         die $err if $err;
2993
2994         return [ sort keys %$flags ];
2995     };
2996
2997     # We need to query QEMU twice, since KVM and TCG have different supported flags
2998     PVE::QemuConfig->lock_config($fakevmid, sub {
2999         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3000         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3001
3002         if ($kvm_supported) {
3003             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3004             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3005         }
3006     });
3007
3008     return $flags;
3009 }
3010
3011 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3012 my $understood_cpu_flag_dir = "/usr/share/kvm";
3013 sub query_understood_cpu_flags {
3014     my $arch = get_host_arch();
3015     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3016
3017     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3018         if ! -e $filepath;
3019
3020     my $raw = file_get_contents($filepath);
3021     $raw =~ s/^\s+|\s+$//g;
3022     my @flags = split(/\s+/, $raw);
3023
3024     return \@flags;
3025 }
3026
3027 sub config_to_command {
3028     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3029
3030     my $cmd = [];
3031     my $globalFlags = [];
3032     my $machineFlags = [];
3033     my $rtcFlags = [];
3034     my $devices = [];
3035     my $pciaddr = '';
3036     my $bridges = {};
3037     my $ostype = $conf->{ostype};
3038     my $winversion = windows_version($ostype);
3039     my $kvm = $conf->{kvm};
3040     my $nodename = nodename();
3041
3042     my $arch = get_vm_arch($conf);
3043     my $kvm_binary = get_command_for_arch($arch);
3044     my $kvmver = kvm_user_version($kvm_binary);
3045
3046     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3047         $kvmver //= "undefined";
3048         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3049     }
3050
3051     my $add_pve_version = min_version($kvmver, 4, 1);
3052
3053     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3054     my $machine_version = extract_version($machine_type, $kvmver);
3055     $kvm //= 1 if is_native($arch);
3056
3057     $machine_version =~ m/(\d+)\.(\d+)/;
3058     my ($machine_major, $machine_minor) = ($1, $2);
3059
3060     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3061         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3062     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3063         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3064             ." please upgrade node '$nodename'\n"
3065     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3066         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3067         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3068             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3069             ." node '$nodename'\n";
3070     }
3071
3072     # if a specific +pve version is required for a feature, use $version_guard
3073     # instead of min_version to allow machines to be run with the minimum
3074     # required version
3075     my $required_pve_version = 0;
3076     my $version_guard = sub {
3077         my ($major, $minor, $pve) = @_;
3078         return 0 if !min_version($machine_version, $major, $minor, $pve);
3079         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3080         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3081         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3082         return 1;
3083     };
3084
3085     if ($kvm && !defined kvm_version()) {
3086         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3087             ." or enable in BIOS.\n";
3088     }
3089
3090     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3091     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3092     my $use_old_bios_files = undef;
3093     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3094
3095     my $cpuunits = defined($conf->{cpuunits}) ?
3096             $conf->{cpuunits} : $defaults->{cpuunits};
3097
3098     push @$cmd, $kvm_binary;
3099
3100     push @$cmd, '-id', $vmid;
3101
3102     my $vmname = $conf->{name} || "vm$vmid";
3103
3104     push @$cmd, '-name', $vmname;
3105
3106     push @$cmd, '-no-shutdown';
3107
3108     my $use_virtio = 0;
3109
3110     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3111     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3112     push @$cmd, '-mon', "chardev=qmp,mode=control";
3113
3114     if (min_version($machine_version, 2, 12)) {
3115         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3116         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3117     }
3118
3119     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3120
3121     push @$cmd, '-daemonize';
3122
3123     if ($conf->{smbios1}) {
3124         my $smbios_conf = parse_smbios1($conf->{smbios1});
3125         if ($smbios_conf->{base64}) {
3126             # Do not pass base64 flag to qemu
3127             delete $smbios_conf->{base64};
3128             my $smbios_string = "";
3129             foreach my $key (keys %$smbios_conf) {
3130                 my $value;
3131                 if ($key eq "uuid") {
3132                     $value = $smbios_conf->{uuid}
3133                 } else {
3134                     $value = decode_base64($smbios_conf->{$key});
3135                 }
3136                 # qemu accepts any binary data, only commas need escaping by double comma
3137                 $value =~ s/,/,,/g;
3138                 $smbios_string .= "," . $key . "=" . $value if $value;
3139             }
3140             push @$cmd, '-smbios', "type=1" . $smbios_string;
3141         } else {
3142             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3143         }
3144     }
3145
3146     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3147         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3148         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3149
3150         my ($path, $format);
3151         if (my $efidisk = $conf->{efidisk0}) {
3152             my $d = parse_drive('efidisk0', $efidisk);
3153             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3154             $format = $d->{format};
3155             if ($storeid) {
3156                 $path = PVE::Storage::path($storecfg, $d->{file});
3157                 if (!defined($format)) {
3158                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3159                     $format = qemu_img_format($scfg, $volname);
3160                 }
3161             } else {
3162                 $path = $d->{file};
3163                 die "efidisk format must be specified\n"
3164                     if !defined($format);
3165             }
3166         } else {
3167             warn "no efidisk configured! Using temporary efivars disk.\n";
3168             $path = "/tmp/$vmid-ovmf.fd";
3169             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3170             $format = 'raw';
3171         }
3172
3173         my $size_str = "";
3174
3175         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3176             $size_str = ",size=" . (-s $ovmf_vars);
3177         }
3178
3179         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3180         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3181     }
3182
3183     # load q35 config
3184     if ($q35) {
3185         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3186         if (min_version($machine_version, 4, 0)) {
3187             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3188         } else {
3189             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3190         }
3191     }
3192
3193     if ($conf->{vmgenid}) {
3194         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3195     }
3196
3197     # add usb controllers
3198     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3199         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3200     push @$devices, @usbcontrollers if @usbcontrollers;
3201     my $vga = parse_vga($conf->{vga});
3202
3203     my $qxlnum = vga_conf_has_spice($conf->{vga});
3204     $vga->{type} = 'qxl' if $qxlnum;
3205
3206     if (!$vga->{type}) {
3207         if ($arch eq 'aarch64') {
3208             $vga->{type} = 'virtio';
3209         } elsif (min_version($machine_version, 2, 9)) {
3210             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3211         } else {
3212             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3213         }
3214     }
3215
3216     # enable absolute mouse coordinates (needed by vnc)
3217     my $tablet;
3218     if (defined($conf->{tablet})) {
3219         $tablet = $conf->{tablet};
3220     } else {
3221         $tablet = $defaults->{tablet};
3222         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3223         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3224     }
3225
3226     if ($tablet) {
3227         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3228         my $kbd = print_keyboarddevice_full($conf, $arch);
3229         push @$devices, '-device', $kbd if defined($kbd);
3230     }
3231
3232     my $bootorder = device_bootorder($conf);
3233
3234     # host pci device passthrough
3235     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3236         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3237
3238     # usb devices
3239     my $usb_dev_features = {};
3240     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3241
3242     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3243         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3244     push @$devices, @usbdevices if @usbdevices;
3245
3246     # serial devices
3247     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3248         if (my $path = $conf->{"serial$i"}) {
3249             if ($path eq 'socket') {
3250                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3251                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3252                 # On aarch64, serial0 is the UART device. Qemu only allows
3253                 # connecting UART devices via the '-serial' command line, as
3254                 # the device has a fixed slot on the hardware...
3255                 if ($arch eq 'aarch64' && $i == 0) {
3256                     push @$devices, '-serial', "chardev:serial$i";
3257                 } else {
3258                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3259                 }
3260             } else {
3261                 die "no such serial device\n" if ! -c $path;
3262                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3263                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3264             }
3265         }
3266     }
3267
3268     # parallel devices
3269     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3270         if (my $path = $conf->{"parallel$i"}) {
3271             die "no such parallel device\n" if ! -c $path;
3272             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3273             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3274             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3275         }
3276     }
3277
3278     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3279         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3280         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3281         push @$devices, @$audio_devs;
3282     }
3283
3284     my $sockets = 1;
3285     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3286     $sockets = $conf->{sockets} if  $conf->{sockets};
3287
3288     my $cores = $conf->{cores} || 1;
3289
3290     my $maxcpus = $sockets * $cores;
3291
3292     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3293
3294     my $allowed_vcpus = $cpuinfo->{cpus};
3295
3296     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3297         if ($allowed_vcpus < $maxcpus);
3298
3299     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3300
3301         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3302         for (my $i = 2; $i <= $vcpus; $i++)  {
3303             my $cpustr = print_cpu_device($conf,$i);
3304             push @$cmd, '-device', $cpustr;
3305         }
3306
3307     } else {
3308
3309         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3310     }
3311     push @$cmd, '-nodefaults';
3312
3313     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3314
3315     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3316
3317     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3318
3319     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3320         push @$devices, '-device', print_vga_device(
3321             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3322         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3323         push @$cmd,  '-vnc', "unix:$socket,password";
3324     } else {
3325         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3326         push @$cmd, '-nographic';
3327     }
3328
3329     # time drift fix
3330     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3331     my $useLocaltime = $conf->{localtime};
3332
3333     if ($winversion >= 5) { # windows
3334         $useLocaltime = 1 if !defined($conf->{localtime});
3335
3336         # use time drift fix when acpi is enabled
3337         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3338             $tdf = 1 if !defined($conf->{tdf});
3339         }
3340     }
3341
3342     if ($winversion >= 6) {
3343         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3344         push @$cmd, '-no-hpet';
3345     }
3346
3347     push @$rtcFlags, 'driftfix=slew' if $tdf;
3348
3349     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3350         push @$rtcFlags, "base=$conf->{startdate}";
3351     } elsif ($useLocaltime) {
3352         push @$rtcFlags, 'base=localtime';
3353     }
3354
3355     if ($forcecpu) {
3356         push @$cmd, '-cpu', $forcecpu;
3357     } else {
3358         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3359     }
3360
3361     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3362
3363     push @$cmd, '-S' if $conf->{freeze};
3364
3365     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3366
3367     my $guest_agent = parse_guest_agent($conf);
3368
3369     if ($guest_agent->{enabled}) {
3370         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3371         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3372
3373         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3374             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3375             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3376             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3377         } elsif ($guest_agent->{type} eq 'isa') {
3378             push @$devices, '-device', "isa-serial,chardev=qga0";
3379         }
3380     }
3381
3382     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3383     if ($rng && $version_guard->(4, 1, 2)) {
3384         check_rng_source($rng->{source});
3385
3386         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3387         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3388         my $limiter_str = "";
3389         if ($max_bytes) {
3390             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3391         }
3392
3393         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3394         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3395         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3396     }
3397
3398     my $spice_port;
3399
3400     if ($qxlnum) {
3401         if ($qxlnum > 1) {
3402             if ($winversion){
3403                 for (my $i = 1; $i < $qxlnum; $i++){
3404                     push @$devices, '-device', print_vga_device(
3405                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3406                 }
3407             } else {
3408                 # assume other OS works like Linux
3409                 my ($ram, $vram) = ("134217728", "67108864");
3410                 if ($vga->{memory}) {
3411                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3412                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3413                 }
3414                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3415                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3416             }
3417         }
3418
3419         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3420
3421         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3422         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3423         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3424
3425         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3426         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3427         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3428
3429         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3430         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3431
3432         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3433         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3434         if ($spice_enhancement->{foldersharing}) {
3435             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3436             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3437         }
3438
3439         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3440         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3441             if $spice_enhancement->{videostreaming};
3442
3443         push @$devices, '-spice', "$spice_opts";
3444     }
3445
3446     # enable balloon by default, unless explicitly disabled
3447     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3448         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3449         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3450     }
3451
3452     if ($conf->{watchdog}) {
3453         my $wdopts = parse_watchdog($conf->{watchdog});
3454         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3455         my $watchdog = $wdopts->{model} || 'i6300esb';
3456         push @$devices, '-device', "$watchdog$pciaddr";
3457         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3458     }
3459
3460     my $vollist = [];
3461     my $scsicontroller = {};
3462     my $ahcicontroller = {};
3463     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3464
3465     # Add iscsi initiator name if available
3466     if (my $initiator = get_initiator_name()) {
3467         push @$devices, '-iscsi', "initiator-name=$initiator";
3468     }
3469
3470     PVE::QemuConfig->foreach_volume($conf, sub {
3471         my ($ds, $drive) = @_;
3472
3473         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3474             push @$vollist, $drive->{file};
3475         }
3476
3477         # ignore efidisk here, already added in bios/fw handling code above
3478         return if $drive->{interface} eq 'efidisk';
3479
3480         $use_virtio = 1 if $ds =~ m/^virtio/;
3481
3482         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3483
3484         if ($drive->{interface} eq 'virtio'){
3485            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3486         }
3487
3488         if ($drive->{interface} eq 'scsi') {
3489
3490             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3491
3492             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3493                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3494
3495             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3496             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3497
3498             my $iothread = '';
3499             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3500                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3501                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3502             } elsif ($drive->{iothread}) {
3503                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3504             }
3505
3506             my $queues = '';
3507             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3508                 $queues = ",num_queues=$drive->{queues}";
3509             }
3510
3511             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3512                 if !$scsicontroller->{$controller};
3513             $scsicontroller->{$controller}=1;
3514         }
3515
3516         if ($drive->{interface} eq 'sata') {
3517             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3518             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3519             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3520                 if !$ahcicontroller->{$controller};
3521             $ahcicontroller->{$controller}=1;
3522         }
3523
3524         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3525         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3526
3527         push @$devices, '-drive',$drive_cmd;
3528         push @$devices, '-device', print_drivedevice_full(
3529             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3530     });
3531
3532     for (my $i = 0; $i < $MAX_NETS; $i++) {
3533         my $netname = "net$i";
3534
3535         next if !$conf->{$netname};
3536         my $d = parse_net($conf->{$netname});
3537         next if !$d;
3538
3539         $use_virtio = 1 if $d->{model} eq 'virtio';
3540
3541         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3542
3543         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3544         push @$devices, '-netdev', $netdevfull;
3545
3546         my $netdevicefull = print_netdevice_full(
3547             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3548
3549         push @$devices, '-device', $netdevicefull;
3550     }
3551
3552     if ($conf->{ivshmem}) {
3553         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3554
3555         my $bus;
3556         if ($q35) {
3557             $bus = print_pcie_addr("ivshmem");
3558         } else {
3559             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3560         }
3561
3562         my $ivshmem_name = $ivshmem->{name} // $vmid;
3563         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3564
3565         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3566         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3567             .",size=$ivshmem->{size}M";
3568     }
3569
3570     # pci.4 is nested in pci.1
3571     $bridges->{1} = 1 if $bridges->{4};
3572
3573     if (!$q35) {
3574         # add pci bridges
3575         if (min_version($machine_version, 2, 3)) {
3576            $bridges->{1} = 1;
3577            $bridges->{2} = 1;
3578         }
3579
3580         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3581
3582     }
3583
3584     for my $k (sort {$b cmp $a} keys %$bridges) {
3585         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3586
3587         my $k_name = $k;
3588         if ($k == 2 && $legacy_igd) {
3589             $k_name = "$k-igd";
3590         }
3591         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3592
3593         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3594         if ($q35) {
3595             # add after -readconfig pve-q35.cfg
3596             splice @$devices, 2, 0, '-device', $devstr;
3597         } else {
3598             unshift @$devices, '-device', $devstr if $k > 0;
3599         }
3600     }
3601
3602     if (!$kvm) {
3603         push @$machineFlags, 'accel=tcg';
3604     }
3605
3606     my $machine_type_min = $machine_type;
3607     if ($add_pve_version) {
3608         $machine_type_min =~ s/\+pve\d+$//;
3609         $machine_type_min .= "+pve$required_pve_version";
3610     }
3611     push @$machineFlags, "type=${machine_type_min}";
3612
3613     push @$cmd, @$devices;
3614     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3615     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3616     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3617
3618     if (my $vmstate = $conf->{vmstate}) {
3619         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3620         push @$vollist, $vmstate;
3621         push @$cmd, '-loadstate', $statepath;
3622         print "activating and using '$vmstate' as vmstate\n";
3623     }
3624
3625     # add custom args
3626     if ($conf->{args}) {
3627         my $aa = PVE::Tools::split_args($conf->{args});
3628         push @$cmd, @$aa;
3629     }
3630
3631     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3632 }
3633
3634 sub check_rng_source {
3635     my ($source) = @_;
3636
3637     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3638     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3639         if ! -e $source;
3640
3641     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3642     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3643         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3644         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3645         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3646             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3647             ." to the host.\n";
3648     }
3649 }
3650
3651 sub spice_port {
3652     my ($vmid) = @_;
3653
3654     my $res = mon_cmd($vmid, 'query-spice');
3655
3656     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3657 }
3658
3659 sub vm_devices_list {
3660     my ($vmid) = @_;
3661
3662     my $res = mon_cmd($vmid, 'query-pci');
3663     my $devices_to_check = [];
3664     my $devices = {};
3665     foreach my $pcibus (@$res) {
3666         push @$devices_to_check, @{$pcibus->{devices}},
3667     }
3668
3669     while (@$devices_to_check) {
3670         my $to_check = [];
3671         for my $d (@$devices_to_check) {
3672             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3673             next if !$d->{'pci_bridge'};
3674
3675             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3676             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3677         }
3678         $devices_to_check = $to_check;
3679     }
3680
3681     my $resblock = mon_cmd($vmid, 'query-block');
3682     foreach my $block (@$resblock) {
3683         if($block->{device} =~ m/^drive-(\S+)/){
3684                 $devices->{$1} = 1;
3685         }
3686     }
3687
3688     my $resmice = mon_cmd($vmid, 'query-mice');
3689     foreach my $mice (@$resmice) {
3690         if ($mice->{name} eq 'QEMU HID Tablet') {
3691             $devices->{tablet} = 1;
3692             last;
3693         }
3694     }
3695
3696     # for usb devices there is no query-usb
3697     # but we can iterate over the entries in
3698     # qom-list path=/machine/peripheral
3699     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3700     foreach my $per (@$resperipheral) {
3701         if ($per->{name} =~ m/^usb\d+$/) {
3702             $devices->{$per->{name}} = 1;
3703         }
3704     }
3705
3706     return $devices;
3707 }
3708
3709 sub vm_deviceplug {
3710     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3711
3712     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3713
3714     my $devices_list = vm_devices_list($vmid);
3715     return 1 if defined($devices_list->{$deviceid});
3716
3717     # add PCI bridge if we need it for the device
3718     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3719
3720     if ($deviceid eq 'tablet') {
3721
3722         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3723
3724     } elsif ($deviceid eq 'keyboard') {
3725
3726         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3727
3728     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3729
3730         die "usb hotplug currently not reliable\n";
3731         # since we can't reliably hot unplug all added usb devices and usb
3732         # passthrough breaks live migration we disable usb hotplugging for now
3733         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3734
3735     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3736
3737         qemu_iothread_add($vmid, $deviceid, $device);
3738
3739         qemu_driveadd($storecfg, $vmid, $device);
3740         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3741
3742         qemu_deviceadd($vmid, $devicefull);
3743         eval { qemu_deviceaddverify($vmid, $deviceid); };
3744         if (my $err = $@) {
3745             eval { qemu_drivedel($vmid, $deviceid); };
3746             warn $@ if $@;
3747             die $err;
3748         }
3749
3750     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3751
3752
3753         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3754         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3755         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3756
3757         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3758
3759         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3760             qemu_iothread_add($vmid, $deviceid, $device);
3761             $devicefull .= ",iothread=iothread-$deviceid";
3762         }
3763
3764         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3765             $devicefull .= ",num_queues=$device->{queues}";
3766         }
3767
3768         qemu_deviceadd($vmid, $devicefull);
3769         qemu_deviceaddverify($vmid, $deviceid);
3770
3771     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3772
3773         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3774         qemu_driveadd($storecfg, $vmid, $device);
3775
3776         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3777         eval { qemu_deviceadd($vmid, $devicefull); };
3778         if (my $err = $@) {
3779             eval { qemu_drivedel($vmid, $deviceid); };
3780             warn $@ if $@;
3781             die $err;
3782         }
3783
3784     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3785
3786         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3787
3788         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3789         my $use_old_bios_files = undef;
3790         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3791
3792         my $netdevicefull = print_netdevice_full(
3793             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3794         qemu_deviceadd($vmid, $netdevicefull);
3795         eval {
3796             qemu_deviceaddverify($vmid, $deviceid);
3797             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3798         };
3799         if (my $err = $@) {
3800             eval { qemu_netdevdel($vmid, $deviceid); };
3801             warn $@ if $@;
3802             die $err;
3803         }
3804
3805     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3806
3807         my $bridgeid = $2;
3808         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3809         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3810
3811         qemu_deviceadd($vmid, $devicefull);
3812         qemu_deviceaddverify($vmid, $deviceid);
3813
3814     } else {
3815         die "can't hotplug device '$deviceid'\n";
3816     }
3817
3818     return 1;
3819 }
3820
3821 # fixme: this should raise exceptions on error!
3822 sub vm_deviceunplug {
3823     my ($vmid, $conf, $deviceid) = @_;
3824
3825     my $devices_list = vm_devices_list($vmid);
3826     return 1 if !defined($devices_list->{$deviceid});
3827
3828     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3829     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3830
3831     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3832
3833         qemu_devicedel($vmid, $deviceid);
3834
3835     } elsif ($deviceid =~ m/^usb\d+$/) {
3836
3837         die "usb hotplug currently not reliable\n";
3838         # when unplugging usb devices this way, there may be remaining usb
3839         # controllers/hubs so we disable it for now
3840         #qemu_devicedel($vmid, $deviceid);
3841         #qemu_devicedelverify($vmid, $deviceid);
3842
3843     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3844
3845         qemu_devicedel($vmid, $deviceid);
3846         qemu_devicedelverify($vmid, $deviceid);
3847         qemu_drivedel($vmid, $deviceid);
3848         qemu_iothread_del($conf, $vmid, $deviceid);
3849
3850     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3851
3852         qemu_devicedel($vmid, $deviceid);
3853         qemu_devicedelverify($vmid, $deviceid);
3854         qemu_iothread_del($conf, $vmid, $deviceid);
3855
3856     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3857
3858         qemu_devicedel($vmid, $deviceid);
3859         qemu_drivedel($vmid, $deviceid);
3860         qemu_deletescsihw($conf, $vmid, $deviceid);
3861
3862     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3863
3864         qemu_devicedel($vmid, $deviceid);
3865         qemu_devicedelverify($vmid, $deviceid);
3866         qemu_netdevdel($vmid, $deviceid);
3867
3868     } else {
3869         die "can't unplug device '$deviceid'\n";
3870     }
3871
3872     return 1;
3873 }
3874
3875 sub qemu_deviceadd {
3876     my ($vmid, $devicefull) = @_;
3877
3878     $devicefull = "driver=".$devicefull;
3879     my %options =  split(/[=,]/, $devicefull);
3880
3881     mon_cmd($vmid, "device_add" , %options);
3882 }
3883
3884 sub qemu_devicedel {
3885     my ($vmid, $deviceid) = @_;
3886
3887     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3888 }
3889
3890 sub qemu_iothread_add {
3891     my($vmid, $deviceid, $device) = @_;
3892
3893     if ($device->{iothread}) {
3894         my $iothreads = vm_iothreads_list($vmid);
3895         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3896     }
3897 }
3898
3899 sub qemu_iothread_del {
3900     my($conf, $vmid, $deviceid) = @_;
3901
3902     my $confid = $deviceid;
3903     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3904         $confid = 'scsi' . $1;
3905     }
3906     my $device = parse_drive($confid, $conf->{$confid});
3907     if ($device->{iothread}) {
3908         my $iothreads = vm_iothreads_list($vmid);
3909         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3910     }
3911 }
3912
3913 sub qemu_objectadd {
3914     my($vmid, $objectid, $qomtype) = @_;
3915
3916     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3917
3918     return 1;
3919 }
3920
3921 sub qemu_objectdel {
3922     my($vmid, $objectid) = @_;
3923
3924     mon_cmd($vmid, "object-del", id => $objectid);
3925
3926     return 1;
3927 }
3928
3929 sub qemu_driveadd {
3930     my ($storecfg, $vmid, $device) = @_;
3931
3932     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3933     $drive =~ s/\\/\\\\/g;
3934     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3935
3936     # If the command succeeds qemu prints: "OK"
3937     return 1 if $ret =~ m/OK/s;
3938
3939     die "adding drive failed: $ret\n";
3940 }
3941
3942 sub qemu_drivedel {
3943     my($vmid, $deviceid) = @_;
3944
3945     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3946     $ret =~ s/^\s+//;
3947
3948     return 1 if $ret eq "";
3949
3950     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3951     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3952
3953     die "deleting drive $deviceid failed : $ret\n";
3954 }
3955
3956 sub qemu_deviceaddverify {
3957     my ($vmid, $deviceid) = @_;
3958
3959     for (my $i = 0; $i <= 5; $i++) {
3960          my $devices_list = vm_devices_list($vmid);
3961          return 1 if defined($devices_list->{$deviceid});
3962          sleep 1;
3963     }
3964
3965     die "error on hotplug device '$deviceid'\n";
3966 }
3967
3968
3969 sub qemu_devicedelverify {
3970     my ($vmid, $deviceid) = @_;
3971
3972     # need to verify that the device is correctly removed as device_del
3973     # is async and empty return is not reliable
3974
3975     for (my $i = 0; $i <= 5; $i++) {
3976          my $devices_list = vm_devices_list($vmid);
3977          return 1 if !defined($devices_list->{$deviceid});
3978          sleep 1;
3979     }
3980
3981     die "error on hot-unplugging device '$deviceid'\n";
3982 }
3983
3984 sub qemu_findorcreatescsihw {
3985     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
3986
3987     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3988
3989     my $scsihwid="$controller_prefix$controller";
3990     my $devices_list = vm_devices_list($vmid);
3991
3992     if(!defined($devices_list->{$scsihwid})) {
3993         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
3994     }
3995
3996     return 1;
3997 }
3998
3999 sub qemu_deletescsihw {
4000     my ($conf, $vmid, $opt) = @_;
4001
4002     my $device = parse_drive($opt, $conf->{$opt});
4003
4004     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4005         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4006         return 1;
4007     }
4008
4009     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4010
4011     my $devices_list = vm_devices_list($vmid);
4012     foreach my $opt (keys %{$devices_list}) {
4013         if (is_valid_drivename($opt)) {
4014             my $drive = parse_drive($opt, $conf->{$opt});
4015             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4016                 return 1;
4017             }
4018         }
4019     }
4020
4021     my $scsihwid="scsihw$controller";
4022
4023     vm_deviceunplug($vmid, $conf, $scsihwid);
4024
4025     return 1;
4026 }
4027
4028 sub qemu_add_pci_bridge {
4029     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4030
4031     my $bridges = {};
4032
4033     my $bridgeid;
4034
4035     print_pci_addr($device, $bridges, $arch, $machine_type);
4036
4037     while (my ($k, $v) = each %$bridges) {
4038         $bridgeid = $k;
4039     }
4040     return 1 if !defined($bridgeid) || $bridgeid < 1;
4041
4042     my $bridge = "pci.$bridgeid";
4043     my $devices_list = vm_devices_list($vmid);
4044
4045     if (!defined($devices_list->{$bridge})) {
4046         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4047     }
4048
4049     return 1;
4050 }
4051
4052 sub qemu_set_link_status {
4053     my ($vmid, $device, $up) = @_;
4054
4055     mon_cmd($vmid, "set_link", name => $device,
4056                up => $up ? JSON::true : JSON::false);
4057 }
4058
4059 sub qemu_netdevadd {
4060     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4061
4062     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4063     my %options =  split(/[=,]/, $netdev);
4064
4065     if (defined(my $vhost = $options{vhost})) {
4066         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4067     }
4068
4069     if (defined(my $queues = $options{queues})) {
4070         $options{queues} = $queues + 0;
4071     }
4072
4073     mon_cmd($vmid, "netdev_add",  %options);
4074     return 1;
4075 }
4076
4077 sub qemu_netdevdel {
4078     my ($vmid, $deviceid) = @_;
4079
4080     mon_cmd($vmid, "netdev_del", id => $deviceid);
4081 }
4082
4083 sub qemu_usb_hotplug {
4084     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4085
4086     return if !$device;
4087
4088     # remove the old one first
4089     vm_deviceunplug($vmid, $conf, $deviceid);
4090
4091     # check if xhci controller is necessary and available
4092     if ($device->{usb3}) {
4093
4094         my $devicelist = vm_devices_list($vmid);
4095
4096         if (!$devicelist->{xhci}) {
4097             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4098             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4099         }
4100     }
4101     my $d = parse_usb_device($device->{host});
4102     $d->{usb3} = $device->{usb3};
4103
4104     # add the new one
4105     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4106 }
4107
4108 sub qemu_cpu_hotplug {
4109     my ($vmid, $conf, $vcpus) = @_;
4110
4111     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4112
4113     my $sockets = 1;
4114     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4115     $sockets = $conf->{sockets} if  $conf->{sockets};
4116     my $cores = $conf->{cores} || 1;
4117     my $maxcpus = $sockets * $cores;
4118
4119     $vcpus = $maxcpus if !$vcpus;
4120
4121     die "you can't add more vcpus than maxcpus\n"
4122         if $vcpus > $maxcpus;
4123
4124     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4125
4126     if ($vcpus < $currentvcpus) {
4127
4128         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4129
4130             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4131                 qemu_devicedel($vmid, "cpu$i");
4132                 my $retry = 0;
4133                 my $currentrunningvcpus = undef;
4134                 while (1) {
4135                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4136                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4137                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4138                     $retry++;
4139                     sleep 1;
4140                 }
4141                 #update conf after each succesfull cpu unplug
4142                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4143                 PVE::QemuConfig->write_config($vmid, $conf);
4144             }
4145         } else {
4146             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4147         }
4148
4149         return;
4150     }
4151
4152     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4153     die "vcpus in running vm does not match its configuration\n"
4154         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4155
4156     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4157
4158         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4159             my $cpustr = print_cpu_device($conf, $i);
4160             qemu_deviceadd($vmid, $cpustr);
4161
4162             my $retry = 0;
4163             my $currentrunningvcpus = undef;
4164             while (1) {
4165                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4166                 last if scalar(@{$currentrunningvcpus}) == $i;
4167                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4168                 sleep 1;
4169                 $retry++;
4170             }
4171             #update conf after each succesfull cpu hotplug
4172             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4173             PVE::QemuConfig->write_config($vmid, $conf);
4174         }
4175     } else {
4176
4177         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4178             mon_cmd($vmid, "cpu-add", id => int($i));
4179         }
4180     }
4181 }
4182
4183 sub qemu_block_set_io_throttle {
4184     my ($vmid, $deviceid,
4185         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4186         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4187         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4188         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4189
4190     return if !check_running($vmid) ;
4191
4192     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4193         bps => int($bps),
4194         bps_rd => int($bps_rd),
4195         bps_wr => int($bps_wr),
4196         iops => int($iops),
4197         iops_rd => int($iops_rd),
4198         iops_wr => int($iops_wr),
4199         bps_max => int($bps_max),
4200         bps_rd_max => int($bps_rd_max),
4201         bps_wr_max => int($bps_wr_max),
4202         iops_max => int($iops_max),
4203         iops_rd_max => int($iops_rd_max),
4204         iops_wr_max => int($iops_wr_max),
4205         bps_max_length => int($bps_max_length),
4206         bps_rd_max_length => int($bps_rd_max_length),
4207         bps_wr_max_length => int($bps_wr_max_length),
4208         iops_max_length => int($iops_max_length),
4209         iops_rd_max_length => int($iops_rd_max_length),
4210         iops_wr_max_length => int($iops_wr_max_length),
4211     );
4212
4213 }
4214
4215 sub qemu_block_resize {
4216     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4217
4218     my $running = check_running($vmid);
4219
4220     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4221
4222     return if !$running;
4223
4224     my $padding = (1024 - $size % 1024) % 1024;
4225     $size = $size + $padding;
4226
4227     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4228
4229 }
4230
4231 sub qemu_volume_snapshot {
4232     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4233
4234     my $running = check_running($vmid);
4235
4236     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4237         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4238     } else {
4239         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4240     }
4241 }
4242
4243 sub qemu_volume_snapshot_delete {
4244     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4245
4246     my $running = check_running($vmid);
4247
4248     if($running) {
4249
4250         $running = undef;
4251         my $conf = PVE::QemuConfig->load_config($vmid);
4252         PVE::QemuConfig->foreach_volume($conf, sub {
4253             my ($ds, $drive) = @_;
4254             $running = 1 if $drive->{file} eq $volid;
4255         });
4256     }
4257
4258     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4259         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4260     } else {
4261         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4262     }
4263 }
4264
4265 sub set_migration_caps {
4266     my ($vmid) = @_;
4267
4268     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4269
4270     my $cap_ref = [];
4271
4272     my $enabled_cap = {
4273         "auto-converge" => 1,
4274         "xbzrle" => 1,
4275         "x-rdma-pin-all" => 0,
4276         "zero-blocks" => 0,
4277         "compress" => 0,
4278         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4279     };
4280
4281     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4282
4283     for my $supported_capability (@$supported_capabilities) {
4284         push @$cap_ref, {
4285             capability => $supported_capability->{capability},
4286             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4287         };
4288     }
4289
4290     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4291 }
4292
4293 sub foreach_volid {
4294     my ($conf, $func, @param) = @_;
4295
4296     my $volhash = {};
4297
4298     my $test_volid = sub {
4299         my ($key, $drive, $snapname) = @_;
4300
4301         my $volid = $drive->{file};
4302         return if !$volid;
4303
4304         $volhash->{$volid}->{cdrom} //= 1;
4305         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4306
4307         my $replicate = $drive->{replicate} // 1;
4308         $volhash->{$volid}->{replicate} //= 0;
4309         $volhash->{$volid}->{replicate} = 1 if $replicate;
4310
4311         $volhash->{$volid}->{shared} //= 0;
4312         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4313
4314         $volhash->{$volid}->{referenced_in_config} //= 0;
4315         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4316
4317         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4318             if defined($snapname);
4319
4320         my $size = $drive->{size};
4321         $volhash->{$volid}->{size} //= $size if $size;
4322
4323         $volhash->{$volid}->{is_vmstate} //= 0;
4324         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4325
4326         $volhash->{$volid}->{is_unused} //= 0;
4327         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4328     };
4329
4330     my $include_opts = {
4331         extra_keys => ['vmstate'],
4332         include_unused => 1,
4333     };
4334
4335     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4336     foreach my $snapname (keys %{$conf->{snapshots}}) {
4337         my $snap = $conf->{snapshots}->{$snapname};
4338         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4339     }
4340
4341     foreach my $volid (keys %$volhash) {
4342         &$func($volid, $volhash->{$volid}, @param);
4343     }
4344 }
4345
4346 my $fast_plug_option = {
4347     'lock' => 1,
4348     'name' => 1,
4349     'onboot' => 1,
4350     'shares' => 1,
4351     'startup' => 1,
4352     'description' => 1,
4353     'protection' => 1,
4354     'vmstatestorage' => 1,
4355     'hookscript' => 1,
4356     'tags' => 1,
4357 };
4358
4359 # hotplug changes in [PENDING]
4360 # $selection hash can be used to only apply specified options, for
4361 # example: { cores => 1 } (only apply changed 'cores')
4362 # $errors ref is used to return error messages
4363 sub vmconfig_hotplug_pending {
4364     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4365
4366     my $defaults = load_defaults();
4367     my $arch = get_vm_arch($conf);
4368     my $machine_type = get_vm_machine($conf, undef, $arch);
4369
4370     # commit values which do not have any impact on running VM first
4371     # Note: those option cannot raise errors, we we do not care about
4372     # $selection and always apply them.
4373
4374     my $add_error = sub {
4375         my ($opt, $msg) = @_;
4376         $errors->{$opt} = "hotplug problem - $msg";
4377     };
4378
4379     my $changes = 0;
4380     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4381         if ($fast_plug_option->{$opt}) {
4382             $conf->{$opt} = $conf->{pending}->{$opt};
4383             delete $conf->{pending}->{$opt};
4384             $changes = 1;
4385         }
4386     }
4387
4388     if ($changes) {
4389         PVE::QemuConfig->write_config($vmid, $conf);
4390     }
4391
4392     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4393
4394     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4395     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4396     foreach my $opt (sort keys %$pending_delete_hash) {
4397         next if $selection && !$selection->{$opt};
4398         my $force = $pending_delete_hash->{$opt}->{force};
4399         eval {
4400             if ($opt eq 'hotplug') {
4401                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4402             } elsif ($opt eq 'tablet') {
4403                 die "skip\n" if !$hotplug_features->{usb};
4404                 if ($defaults->{tablet}) {
4405                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4406                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4407                         if $arch eq 'aarch64';
4408                 } else {
4409                     vm_deviceunplug($vmid, $conf, 'tablet');
4410                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4411                 }
4412             } elsif ($opt =~ m/^usb\d+/) {
4413                 die "skip\n";
4414                 # since we cannot reliably hot unplug usb devices we are disabling it
4415                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4416                 #vm_deviceunplug($vmid, $conf, $opt);
4417             } elsif ($opt eq 'vcpus') {
4418                 die "skip\n" if !$hotplug_features->{cpu};
4419                 qemu_cpu_hotplug($vmid, $conf, undef);
4420             } elsif ($opt eq 'balloon') {
4421                 # enable balloon device is not hotpluggable
4422                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4423                 # here we reset the ballooning value to memory
4424                 my $balloon = $conf->{memory} || $defaults->{memory};
4425                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4426             } elsif ($fast_plug_option->{$opt}) {
4427                 # do nothing
4428             } elsif ($opt =~ m/^net(\d+)$/) {
4429                 die "skip\n" if !$hotplug_features->{network};
4430                 vm_deviceunplug($vmid, $conf, $opt);
4431             } elsif (is_valid_drivename($opt)) {
4432                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4433                 vm_deviceunplug($vmid, $conf, $opt);
4434                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4435             } elsif ($opt =~ m/^memory$/) {
4436                 die "skip\n" if !$hotplug_features->{memory};
4437                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4438             } elsif ($opt eq 'cpuunits') {
4439                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4440             } elsif ($opt eq 'cpulimit') {
4441                 $cgroup->change_cpu_quota(-1, 100000);
4442             } else {
4443                 die "skip\n";
4444             }
4445         };
4446         if (my $err = $@) {
4447             &$add_error($opt, $err) if $err ne "skip\n";
4448         } else {
4449             delete $conf->{$opt};
4450             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4451         }
4452     }
4453
4454     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4455     $apply_pending_cloudinit = sub {
4456         return if $apply_pending_cloudinit_done; # once is enough
4457         $apply_pending_cloudinit_done = 1; # once is enough
4458
4459         my ($key, $value) = @_;
4460
4461         my @cloudinit_opts = keys %$confdesc_cloudinit;
4462         foreach my $opt (keys %{$conf->{pending}}) {
4463             next if !grep { $_ eq $opt } @cloudinit_opts;
4464             $conf->{$opt} = delete $conf->{pending}->{$opt};
4465         }
4466
4467         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4468         foreach my $opt (sort keys %$pending_delete_hash) {
4469             next if !grep { $_ eq $opt } @cloudinit_opts;
4470             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4471             delete $conf->{$opt};
4472         }
4473
4474         my $new_conf = { %$conf };
4475         $new_conf->{$key} = $value;
4476         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4477     };
4478
4479     foreach my $opt (keys %{$conf->{pending}}) {
4480         next if $selection && !$selection->{$opt};
4481         my $value = $conf->{pending}->{$opt};
4482         eval {
4483             if ($opt eq 'hotplug') {
4484                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4485             } elsif ($opt eq 'tablet') {
4486                 die "skip\n" if !$hotplug_features->{usb};
4487                 if ($value == 1) {
4488                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4489                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4490                         if $arch eq 'aarch64';
4491                 } elsif ($value == 0) {
4492                     vm_deviceunplug($vmid, $conf, 'tablet');
4493                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4494                 }
4495             } elsif ($opt =~ m/^usb\d+$/) {
4496                 die "skip\n";
4497                 # since we cannot reliably hot unplug usb devices we disable it for now
4498                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4499                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4500                 #die "skip\n" if !$d;
4501                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4502             } elsif ($opt eq 'vcpus') {
4503                 die "skip\n" if !$hotplug_features->{cpu};
4504                 qemu_cpu_hotplug($vmid, $conf, $value);
4505             } elsif ($opt eq 'balloon') {
4506                 # enable/disable balloning device is not hotpluggable
4507                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4508                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4509                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4510
4511                 # allow manual ballooning if shares is set to zero
4512                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4513                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4514                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4515                 }
4516             } elsif ($opt =~ m/^net(\d+)$/) {
4517                 # some changes can be done without hotplug
4518                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4519                                     $vmid, $opt, $value, $arch, $machine_type);
4520             } elsif (is_valid_drivename($opt)) {
4521                 die "skip\n" if $opt eq 'efidisk0';
4522                 # some changes can be done without hotplug
4523                 my $drive = parse_drive($opt, $value);
4524                 if (drive_is_cloudinit($drive)) {
4525                     &$apply_pending_cloudinit($opt, $value);
4526                 }
4527                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4528                                      $vmid, $opt, $value, $arch, $machine_type);
4529             } elsif ($opt =~ m/^memory$/) { #dimms
4530                 die "skip\n" if !$hotplug_features->{memory};
4531                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4532             } elsif ($opt eq 'cpuunits') {
4533                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4534             } elsif ($opt eq 'cpulimit') {
4535                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4536                 $cgroup->change_cpu_quota($cpulimit, 100000);
4537             } else {
4538                 die "skip\n";  # skip non-hot-pluggable options
4539             }
4540         };
4541         if (my $err = $@) {
4542             &$add_error($opt, $err) if $err ne "skip\n";
4543         } else {
4544             $conf->{$opt} = $value;
4545             delete $conf->{pending}->{$opt};
4546         }
4547     }
4548
4549     PVE::QemuConfig->write_config($vmid, $conf);
4550 }
4551
4552 sub try_deallocate_drive {
4553     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4554
4555     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4556         my $volid = $drive->{file};
4557         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4558             my $sid = PVE::Storage::parse_volume_id($volid);
4559             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4560
4561             # check if the disk is really unused
4562             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4563                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4564             PVE::Storage::vdisk_free($storecfg, $volid);
4565             return 1;
4566         } else {
4567             # If vm is not owner of this disk remove from config
4568             return 1;
4569         }
4570     }
4571
4572     return;
4573 }
4574
4575 sub vmconfig_delete_or_detach_drive {
4576     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4577
4578     my $drive = parse_drive($opt, $conf->{$opt});
4579
4580     my $rpcenv = PVE::RPCEnvironment::get();
4581     my $authuser = $rpcenv->get_user();
4582
4583     if ($force) {
4584         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4585         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4586     } else {
4587         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4588     }
4589 }
4590
4591
4592
4593 sub vmconfig_apply_pending {
4594     my ($vmid, $conf, $storecfg, $errors) = @_;
4595
4596     my $add_apply_error = sub {
4597         my ($opt, $msg) = @_;
4598         my $err_msg = "unable to apply pending change $opt : $msg";
4599         $errors->{$opt} = $err_msg;
4600         warn $err_msg;
4601     };
4602
4603     # cold plug
4604
4605     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4606     foreach my $opt (sort keys %$pending_delete_hash) {
4607         my $force = $pending_delete_hash->{$opt}->{force};
4608         eval {
4609             if ($opt =~ m/^unused/) {
4610                 die "internal error";
4611             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4612                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4613             }
4614         };
4615         if (my $err = $@) {
4616             $add_apply_error->($opt, $err);
4617         } else {
4618             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4619             delete $conf->{$opt};
4620         }
4621     }
4622
4623     PVE::QemuConfig->cleanup_pending($conf);
4624
4625     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4626         next if $opt eq 'delete'; # just to be sure
4627         eval {
4628             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4629                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4630             }
4631         };
4632         if (my $err = $@) {
4633             $add_apply_error->($opt, $err);
4634         } else {
4635             $conf->{$opt} = delete $conf->{pending}->{$opt};
4636         }
4637     }
4638
4639     # write all changes at once to avoid unnecessary i/o
4640     PVE::QemuConfig->write_config($vmid, $conf);
4641 }
4642
4643 sub vmconfig_update_net {
4644     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4645
4646     my $newnet = parse_net($value);
4647
4648     if ($conf->{$opt}) {
4649         my $oldnet = parse_net($conf->{$opt});
4650
4651         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4652             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4653             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4654             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4655
4656             # for non online change, we try to hot-unplug
4657             die "skip\n" if !$hotplug;
4658             vm_deviceunplug($vmid, $conf, $opt);
4659         } else {
4660
4661             die "internal error" if $opt !~ m/net(\d+)/;
4662             my $iface = "tap${vmid}i$1";
4663
4664             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4665                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4666                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4667                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4668                 PVE::Network::tap_unplug($iface);
4669
4670                 if ($have_sdn) {
4671                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4672                 } else {
4673                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4674                 }
4675             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4676                 # Rate can be applied on its own but any change above needs to
4677                 # include the rate in tap_plug since OVS resets everything.
4678                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4679             }
4680
4681             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4682                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4683             }
4684
4685             return 1;
4686         }
4687     }
4688
4689     if ($hotplug) {
4690         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4691     } else {
4692         die "skip\n";
4693     }
4694 }
4695
4696 sub vmconfig_update_disk {
4697     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4698
4699     my $drive = parse_drive($opt, $value);
4700
4701     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4702         my $media = $drive->{media} || 'disk';
4703         my $oldmedia = $old_drive->{media} || 'disk';
4704         die "unable to change media type\n" if $media ne $oldmedia;
4705
4706         if (!drive_is_cdrom($old_drive)) {
4707
4708             if ($drive->{file} ne $old_drive->{file}) {
4709
4710                 die "skip\n" if !$hotplug;
4711
4712                 # unplug and register as unused
4713                 vm_deviceunplug($vmid, $conf, $opt);
4714                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4715
4716             } else {
4717                 # update existing disk
4718
4719                 # skip non hotpluggable value
4720                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4721                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4722                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4723                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4724                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4725                     die "skip\n";
4726                 }
4727
4728                 # apply throttle
4729                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4730                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4731                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4732                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4733                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4734                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4735                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4736                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4737                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4738                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4739                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4740                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4741                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4742                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4743                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4744                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4745                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4746                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4747
4748                     qemu_block_set_io_throttle(
4749                         $vmid,"drive-$opt",
4750                         ($drive->{mbps} || 0)*1024*1024,
4751                         ($drive->{mbps_rd} || 0)*1024*1024,
4752                         ($drive->{mbps_wr} || 0)*1024*1024,
4753                          $drive->{iops} || 0,
4754                          $drive->{iops_rd} || 0,
4755                          $drive->{iops_wr} || 0,
4756                         ($drive->{mbps_max} || 0)*1024*1024,
4757                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4758                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4759                          $drive->{iops_max} || 0,
4760                          $drive->{iops_rd_max} || 0,
4761                          $drive->{iops_wr_max} || 0,
4762                          $drive->{bps_max_length} || 1,
4763                          $drive->{bps_rd_max_length} || 1,
4764                          $drive->{bps_wr_max_length} || 1,
4765                          $drive->{iops_max_length} || 1,
4766                          $drive->{iops_rd_max_length} || 1,
4767                          $drive->{iops_wr_max_length} || 1,
4768                     );
4769
4770                 }
4771
4772                 return 1;
4773             }
4774
4775         } else { # cdrom
4776
4777             if ($drive->{file} eq 'none') {
4778                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4779                 if (drive_is_cloudinit($old_drive)) {
4780                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4781                 }
4782             } else {
4783                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4784
4785                 # force eject if locked
4786                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4787
4788                 if ($path) {
4789                     mon_cmd($vmid, "blockdev-change-medium",
4790                         id => "$opt", filename => "$path");
4791                 }
4792             }
4793
4794             return 1;
4795         }
4796     }
4797
4798     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4799     # hotplug new disks
4800     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4801     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4802 }
4803
4804 # called in locked context by incoming migration
4805 sub vm_migrate_get_nbd_disks {
4806     my ($storecfg, $conf, $replicated_volumes) = @_;
4807
4808     my $local_volumes = {};
4809     PVE::QemuConfig->foreach_volume($conf, sub {
4810         my ($ds, $drive) = @_;
4811
4812         return if drive_is_cdrom($drive);
4813
4814         my $volid = $drive->{file};
4815
4816         return if !$volid;
4817
4818         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4819
4820         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4821         return if $scfg->{shared};
4822
4823         # replicated disks re-use existing state via bitmap
4824         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4825         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4826     });
4827     return $local_volumes;
4828 }
4829
4830 # called in locked context by incoming migration
4831 sub vm_migrate_alloc_nbd_disks {
4832     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4833
4834     my $format = undef;
4835
4836     my $nbd = {};
4837     foreach my $opt (sort keys %$source_volumes) {
4838         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4839
4840         if ($use_existing) {
4841             $nbd->{$opt}->{drivestr} = print_drive($drive);
4842             $nbd->{$opt}->{volid} = $volid;
4843             $nbd->{$opt}->{replicated} = 1;
4844             next;
4845         }
4846
4847         # If a remote storage is specified and the format of the original
4848         # volume is not available there, fall back to the default format.
4849         # Otherwise use the same format as the original.
4850         if (!$storagemap->{identity}) {
4851             $storeid = map_storage($storagemap, $storeid);
4852             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4853             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4854             my $fileFormat = qemu_img_format($scfg, $volname);
4855             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4856         } else {
4857             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4858             $format = qemu_img_format($scfg, $volname);
4859         }
4860
4861         my $size = $drive->{size} / 1024;
4862         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4863         my $newdrive = $drive;
4864         $newdrive->{format} = $format;
4865         $newdrive->{file} = $newvolid;
4866         my $drivestr = print_drive($newdrive);
4867         $nbd->{$opt}->{drivestr} = $drivestr;
4868         $nbd->{$opt}->{volid} = $newvolid;
4869     }
4870
4871     return $nbd;
4872 }
4873
4874 # see vm_start_nolock for parameters, additionally:
4875 # migrate_opts:
4876 #   storagemap = parsed storage map for allocating NBD disks
4877 sub vm_start {
4878     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4879
4880     return PVE::QemuConfig->lock_config($vmid, sub {
4881         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4882
4883         die "you can't start a vm if it's a template\n"
4884             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4885
4886         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4887         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4888
4889         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4890
4891         if ($has_backup_lock && $running) {
4892             # a backup is currently running, attempt to start the guest in the
4893             # existing QEMU instance
4894             return vm_resume($vmid);
4895         }
4896
4897         PVE::QemuConfig->check_lock($conf)
4898             if !($params->{skiplock} || $has_suspended_lock);
4899
4900         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4901
4902         die "VM $vmid already running\n" if $running;
4903
4904         if (my $storagemap = $migrate_opts->{storagemap}) {
4905             my $replicated = $migrate_opts->{replicated_volumes};
4906             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4907             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4908
4909             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4910                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4911             }
4912         }
4913
4914         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4915     });
4916 }
4917
4918
4919 # params:
4920 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4921 #   skiplock => 0/1, skip checking for config lock
4922 #   skiptemplate => 0/1, skip checking whether VM is template
4923 #   forcemachine => to force Qemu machine (rollback/migration)
4924 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4925 #   timeout => in seconds
4926 #   paused => start VM in paused state (backup)
4927 #   resume => resume from hibernation
4928 # migrate_opts:
4929 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4930 #   migratedfrom => source node
4931 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4932 #   network => CIDR of migration network
4933 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4934 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4935 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4936 sub vm_start_nolock {
4937     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4938
4939     my $statefile = $params->{statefile};
4940     my $resume = $params->{resume};
4941
4942     my $migratedfrom = $migrate_opts->{migratedfrom};
4943     my $migration_type = $migrate_opts->{type};
4944
4945     my $res = {};
4946
4947     # clean up leftover reboot request files
4948     eval { clear_reboot_request($vmid); };
4949     warn $@ if $@;
4950
4951     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4952         vmconfig_apply_pending($vmid, $conf, $storecfg);
4953         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4954     }
4955
4956     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4957
4958     my $defaults = load_defaults();
4959
4960     # set environment variable useful inside network script
4961     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4962
4963     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4964
4965     my $forcemachine = $params->{forcemachine};
4966     my $forcecpu = $params->{forcecpu};
4967     if ($resume) {
4968         # enforce machine and CPU type on suspended vm to ensure HW compatibility
4969         $forcemachine = $conf->{runningmachine};
4970         $forcecpu = $conf->{runningcpu};
4971         print "Resuming suspended VM\n";
4972     }
4973
4974     my ($cmd, $vollist, $spice_port) =
4975         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
4976
4977     my $migration_ip;
4978     my $get_migration_ip = sub {
4979         my ($nodename) = @_;
4980
4981         return $migration_ip if defined($migration_ip);
4982
4983         my $cidr = $migrate_opts->{network};
4984
4985         if (!defined($cidr)) {
4986             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4987             $cidr = $dc_conf->{migration}->{network};
4988         }
4989
4990         if (defined($cidr)) {
4991             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
4992
4993             die "could not get IP: no address configured on local " .
4994                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
4995
4996             die "could not get IP: multiple addresses configured on local " .
4997                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
4998
4999             $migration_ip = @$ips[0];
5000         }
5001
5002         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5003             if !defined($migration_ip);
5004
5005         return $migration_ip;
5006     };
5007
5008     my $migrate_uri;
5009     if ($statefile) {
5010         if ($statefile eq 'tcp') {
5011             my $localip = "localhost";
5012             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5013             my $nodename = nodename();
5014
5015             if (!defined($migration_type)) {
5016                 if (defined($datacenterconf->{migration}->{type})) {
5017                     $migration_type = $datacenterconf->{migration}->{type};
5018                 } else {
5019                     $migration_type = 'secure';
5020                 }
5021             }
5022
5023             if ($migration_type eq 'insecure') {
5024                 $localip = $get_migration_ip->($nodename);
5025                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5026             }
5027
5028             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5029             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5030             $migrate_uri = "tcp:${localip}:${migrate_port}";
5031             push @$cmd, '-incoming', $migrate_uri;
5032             push @$cmd, '-S';
5033
5034         } elsif ($statefile eq 'unix') {
5035             # should be default for secure migrations as a ssh TCP forward
5036             # tunnel is not deterministic reliable ready and fails regurarly
5037             # to set up in time, so use UNIX socket forwards
5038             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5039             unlink $socket_addr;
5040
5041             $migrate_uri = "unix:$socket_addr";
5042
5043             push @$cmd, '-incoming', $migrate_uri;
5044             push @$cmd, '-S';
5045
5046         } elsif (-e $statefile) {
5047             push @$cmd, '-loadstate', $statefile;
5048         } else {
5049             my $statepath = PVE::Storage::path($storecfg, $statefile);
5050             push @$vollist, $statefile;
5051             push @$cmd, '-loadstate', $statepath;
5052         }
5053     } elsif ($params->{paused}) {
5054         push @$cmd, '-S';
5055     }
5056
5057     # host pci devices
5058     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5059       my $d = parse_hostpci($conf->{"hostpci$i"});
5060       next if !$d;
5061       my $pcidevices = $d->{pciid};
5062       foreach my $pcidevice (@$pcidevices) {
5063             my $pciid = $pcidevice->{id};
5064
5065             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5066             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5067             die "no pci device info for device '$pciid'\n" if !$info;
5068
5069             if ($d->{mdev}) {
5070                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5071                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5072             } else {
5073                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5074                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5075                 die "can't reset PCI device '$pciid'\n"
5076                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5077             }
5078       }
5079     }
5080
5081     PVE::Storage::activate_volumes($storecfg, $vollist);
5082
5083     eval {
5084         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5085             outfunc => sub {}, errfunc => sub {});
5086     };
5087     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5088     # timeout should be more than enough here...
5089     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5090
5091     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5092                                               : $defaults->{cpuunits};
5093
5094     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5095     my %run_params = (
5096         timeout => $statefile ? undef : $start_timeout,
5097         umask => 0077,
5098         noerr => 1,
5099     );
5100
5101     # when migrating, prefix QEMU output so other side can pick up any
5102     # errors that might occur and show the user
5103     if ($migratedfrom) {
5104         $run_params{quiet} = 1;
5105         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5106     }
5107
5108     my %properties = (
5109         Slice => 'qemu.slice',
5110         KillMode => 'none'
5111     );
5112
5113     if (PVE::CGroup::cgroup_mode() == 2) {
5114         $properties{CPUWeight} = $cpuunits;
5115     } else {
5116         $properties{CPUShares} = $cpuunits;
5117     }
5118
5119     if (my $cpulimit = $conf->{cpulimit}) {
5120         $properties{CPUQuota} = int($cpulimit * 100);
5121     }
5122     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5123
5124     my $run_qemu = sub {
5125         PVE::Tools::run_fork sub {
5126             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5127
5128             my $exitcode = run_command($cmd, %run_params);
5129             die "QEMU exited with code $exitcode\n" if $exitcode;
5130         };
5131     };
5132
5133     if ($conf->{hugepages}) {
5134
5135         my $code = sub {
5136             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5137             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5138
5139             PVE::QemuServer::Memory::hugepages_mount();
5140             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5141
5142             eval { $run_qemu->() };
5143             if (my $err = $@) {
5144                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5145                     if !$conf->{keephugepages};
5146                 die $err;
5147             }
5148
5149             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5150                 if !$conf->{keephugepages};
5151         };
5152         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5153
5154     } else {
5155         eval { $run_qemu->() };
5156     }
5157
5158     if (my $err = $@) {
5159         # deactivate volumes if start fails
5160         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5161         die "start failed: $err";
5162     }
5163
5164     print "migration listens on $migrate_uri\n" if $migrate_uri;
5165     $res->{migrate_uri} = $migrate_uri;
5166
5167     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5168         eval { mon_cmd($vmid, "cont"); };
5169         warn $@ if $@;
5170     }
5171
5172     #start nbd server for storage migration
5173     if (my $nbd = $migrate_opts->{nbd}) {
5174         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5175
5176         my $migrate_storage_uri;
5177         # nbd_protocol_version > 0 for unix socket support
5178         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5179             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5180             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5181             $migrate_storage_uri = "nbd:unix:$socket_path";
5182         } else {
5183             my $nodename = nodename();
5184             my $localip = $get_migration_ip->($nodename);
5185             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5186             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5187
5188             mon_cmd($vmid, "nbd-server-start", addr => {
5189                 type => 'inet',
5190                 data => {
5191                     host => "${localip}",
5192                     port => "${storage_migrate_port}",
5193                 },
5194             });
5195             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5196             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5197         }
5198
5199         $res->{migrate_storage_uri} = $migrate_storage_uri;
5200
5201         foreach my $opt (sort keys %$nbd) {
5202             my $drivestr = $nbd->{$opt}->{drivestr};
5203             my $volid = $nbd->{$opt}->{volid};
5204             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5205             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5206             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5207             print "re-using replicated volume: $opt - $volid\n"
5208                 if $nbd->{$opt}->{replicated};
5209
5210             $res->{drives}->{$opt} = $nbd->{$opt};
5211             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5212         }
5213     }
5214
5215     if ($migratedfrom) {
5216         eval {
5217             set_migration_caps($vmid);
5218         };
5219         warn $@ if $@;
5220
5221         if ($spice_port) {
5222             print "spice listens on port $spice_port\n";
5223             $res->{spice_port} = $spice_port;
5224             if ($migrate_opts->{spice_ticket}) {
5225                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5226                     $migrate_opts->{spice_ticket});
5227                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5228             }
5229         }
5230
5231     } else {
5232         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5233             if !$statefile && $conf->{balloon};
5234
5235         foreach my $opt (keys %$conf) {
5236             next if $opt !~  m/^net\d+$/;
5237             my $nicconf = parse_net($conf->{$opt});
5238             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5239         }
5240     }
5241
5242     mon_cmd($vmid, 'qom-set',
5243                 path => "machine/peripheral/balloon0",
5244                 property => "guest-stats-polling-interval",
5245                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5246
5247     if ($resume) {
5248         print "Resumed VM, removing state\n";
5249         if (my $vmstate = $conf->{vmstate}) {
5250             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5251             PVE::Storage::vdisk_free($storecfg, $vmstate);
5252         }
5253         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5254         PVE::QemuConfig->write_config($vmid, $conf);
5255     }
5256
5257     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5258
5259     return $res;
5260 }
5261
5262 sub vm_commandline {
5263     my ($storecfg, $vmid, $snapname) = @_;
5264
5265     my $conf = PVE::QemuConfig->load_config($vmid);
5266     my $forcemachine;
5267     my $forcecpu;
5268
5269     if ($snapname) {
5270         my $snapshot = $conf->{snapshots}->{$snapname};
5271         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5272
5273         # check for machine or CPU overrides in snapshot
5274         $forcemachine = $snapshot->{runningmachine};
5275         $forcecpu = $snapshot->{runningcpu};
5276
5277         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5278
5279         $conf = $snapshot;
5280     }
5281
5282     my $defaults = load_defaults();
5283
5284     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5285         $forcemachine, $forcecpu);
5286
5287     return PVE::Tools::cmd2string($cmd);
5288 }
5289
5290 sub vm_reset {
5291     my ($vmid, $skiplock) = @_;
5292
5293     PVE::QemuConfig->lock_config($vmid, sub {
5294
5295         my $conf = PVE::QemuConfig->load_config($vmid);
5296
5297         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5298
5299         mon_cmd($vmid, "system_reset");
5300     });
5301 }
5302
5303 sub get_vm_volumes {
5304     my ($conf) = @_;
5305
5306     my $vollist = [];
5307     foreach_volid($conf, sub {
5308         my ($volid, $attr) = @_;
5309
5310         return if $volid =~ m|^/|;
5311
5312         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5313         return if !$sid;
5314
5315         push @$vollist, $volid;
5316     });
5317
5318     return $vollist;
5319 }
5320
5321 sub vm_stop_cleanup {
5322     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5323
5324     eval {
5325
5326         if (!$keepActive) {
5327             my $vollist = get_vm_volumes($conf);
5328             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5329         }
5330
5331         foreach my $ext (qw(mon qmp pid vnc qga)) {
5332             unlink "/var/run/qemu-server/${vmid}.$ext";
5333         }
5334
5335         if ($conf->{ivshmem}) {
5336             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5337             # just delete it for now, VMs which have this already open do not
5338             # are affected, but new VMs will get a separated one. If this
5339             # becomes an issue we either add some sort of ref-counting or just
5340             # add a "don't delete on stop" flag to the ivshmem format.
5341             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5342         }
5343
5344         foreach my $key (keys %$conf) {
5345             next if $key !~ m/^hostpci(\d+)$/;
5346             my $hostpciindex = $1;
5347             my $d = parse_hostpci($conf->{$key});
5348             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5349
5350             foreach my $pci (@{$d->{pciid}}) {
5351                 my $pciid = $pci->{id};
5352                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5353             }
5354         }
5355
5356         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5357     };
5358     warn $@ if $@; # avoid errors - just warn
5359 }
5360
5361 # call only in locked context
5362 sub _do_vm_stop {
5363     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5364
5365     my $pid = check_running($vmid, $nocheck);
5366     return if !$pid;
5367
5368     my $conf;
5369     if (!$nocheck) {
5370         $conf = PVE::QemuConfig->load_config($vmid);
5371         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5372         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5373             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5374             $timeout = $opts->{down} if $opts->{down};
5375         }
5376         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5377     }
5378
5379     eval {
5380         if ($shutdown) {
5381             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5382                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5383             } else {
5384                 mon_cmd($vmid, "system_powerdown");
5385             }
5386         } else {
5387             mon_cmd($vmid, "quit");
5388         }
5389     };
5390     my $err = $@;
5391
5392     if (!$err) {
5393         $timeout = 60 if !defined($timeout);
5394
5395         my $count = 0;
5396         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5397             $count++;
5398             sleep 1;
5399         }
5400
5401         if ($count >= $timeout) {
5402             if ($force) {
5403                 warn "VM still running - terminating now with SIGTERM\n";
5404                 kill 15, $pid;
5405             } else {
5406                 die "VM quit/powerdown failed - got timeout\n";
5407             }
5408         } else {
5409             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5410             return;
5411         }
5412     } else {
5413        if (!check_running($vmid, $nocheck)) {
5414             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5415             return;
5416        }
5417        if ($force) {
5418             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5419             kill 15, $pid;
5420         } else {
5421             die "VM quit/powerdown failed\n";
5422         }
5423     }
5424
5425     # wait again
5426     $timeout = 10;
5427
5428     my $count = 0;
5429     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5430         $count++;
5431         sleep 1;
5432     }
5433
5434     if ($count >= $timeout) {
5435         warn "VM still running - terminating now with SIGKILL\n";
5436         kill 9, $pid;
5437         sleep 1;
5438     }
5439
5440     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5441 }
5442
5443 # Note: use $nocheck to skip tests if VM configuration file exists.
5444 # We need that when migration VMs to other nodes (files already moved)
5445 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5446 sub vm_stop {
5447     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5448
5449     $force = 1 if !defined($force) && !$shutdown;
5450
5451     if ($migratedfrom){
5452         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5453         kill 15, $pid if $pid;
5454         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5455         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5456         return;
5457     }
5458
5459     PVE::QemuConfig->lock_config($vmid, sub {
5460         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5461    });
5462 }
5463
5464 sub vm_reboot {
5465     my ($vmid, $timeout) = @_;
5466
5467     PVE::QemuConfig->lock_config($vmid, sub {
5468         eval {
5469
5470             # only reboot if running, as qmeventd starts it again on a stop event
5471             return if !check_running($vmid);
5472
5473             create_reboot_request($vmid);
5474
5475             my $storecfg = PVE::Storage::config();
5476             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5477
5478         };
5479         if (my $err = $@) {
5480             # avoid that the next normal shutdown will be confused for a reboot
5481             clear_reboot_request($vmid);
5482             die $err;
5483         }
5484    });
5485 }
5486
5487 # note: if using the statestorage parameter, the caller has to check privileges
5488 sub vm_suspend {
5489     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5490
5491     my $conf;
5492     my $path;
5493     my $storecfg;
5494     my $vmstate;
5495
5496     PVE::QemuConfig->lock_config($vmid, sub {
5497
5498         $conf = PVE::QemuConfig->load_config($vmid);
5499
5500         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5501         PVE::QemuConfig->check_lock($conf)
5502             if !($skiplock || $is_backing_up);
5503
5504         die "cannot suspend to disk during backup\n"
5505             if $is_backing_up && $includestate;
5506
5507         if ($includestate) {
5508             $conf->{lock} = 'suspending';
5509             my $date = strftime("%Y-%m-%d", localtime(time()));
5510             $storecfg = PVE::Storage::config();
5511             if (!$statestorage) {
5512                 $statestorage = find_vmstate_storage($conf, $storecfg);
5513                 # check permissions for the storage
5514                 my $rpcenv = PVE::RPCEnvironment::get();
5515                 if ($rpcenv->{type} ne 'cli') {
5516                     my $authuser = $rpcenv->get_user();
5517                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5518                 }
5519             }
5520
5521
5522             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5523                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5524             $path = PVE::Storage::path($storecfg, $vmstate);
5525             PVE::QemuConfig->write_config($vmid, $conf);
5526         } else {
5527             mon_cmd($vmid, "stop");
5528         }
5529     });
5530
5531     if ($includestate) {
5532         # save vm state
5533         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5534
5535         eval {
5536             mon_cmd($vmid, "savevm-start", statefile => $path);
5537             for(;;) {
5538                 my $state = mon_cmd($vmid, "query-savevm");
5539                 if (!$state->{status}) {
5540                     die "savevm not active\n";
5541                 } elsif ($state->{status} eq 'active') {
5542                     sleep(1);
5543                     next;
5544                 } elsif ($state->{status} eq 'completed') {
5545                     print "State saved, quitting\n";
5546                     last;
5547                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5548                     die "query-savevm failed with error '$state->{error}'\n"
5549                 } else {
5550                     die "query-savevm returned status '$state->{status}'\n";
5551                 }
5552             }
5553         };
5554         my $err = $@;
5555
5556         PVE::QemuConfig->lock_config($vmid, sub {
5557             $conf = PVE::QemuConfig->load_config($vmid);
5558             if ($err) {
5559                 # cleanup, but leave suspending lock, to indicate something went wrong
5560                 eval {
5561                     mon_cmd($vmid, "savevm-end");
5562                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5563                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5564                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5565                     PVE::QemuConfig->write_config($vmid, $conf);
5566                 };
5567                 warn $@ if $@;
5568                 die $err;
5569             }
5570
5571             die "lock changed unexpectedly\n"
5572                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5573
5574             mon_cmd($vmid, "quit");
5575             $conf->{lock} = 'suspended';
5576             PVE::QemuConfig->write_config($vmid, $conf);
5577         });
5578     }
5579 }
5580
5581 sub vm_resume {
5582     my ($vmid, $skiplock, $nocheck) = @_;
5583
5584     PVE::QemuConfig->lock_config($vmid, sub {
5585         my $res = mon_cmd($vmid, 'query-status');
5586         my $resume_cmd = 'cont';
5587         my $reset = 0;
5588
5589         if ($res->{status}) {
5590             return if $res->{status} eq 'running'; # job done, go home
5591             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5592             $reset = 1 if $res->{status} eq 'shutdown';
5593         }
5594
5595         if (!$nocheck) {
5596
5597             my $conf = PVE::QemuConfig->load_config($vmid);
5598
5599             PVE::QemuConfig->check_lock($conf)
5600                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5601         }
5602
5603         if ($reset) {
5604             # required if a VM shuts down during a backup and we get a resume
5605             # request before the backup finishes for example
5606             mon_cmd($vmid, "system_reset");
5607         }
5608         mon_cmd($vmid, $resume_cmd);
5609     });
5610 }
5611
5612 sub vm_sendkey {
5613     my ($vmid, $skiplock, $key) = @_;
5614
5615     PVE::QemuConfig->lock_config($vmid, sub {
5616
5617         my $conf = PVE::QemuConfig->load_config($vmid);
5618
5619         # there is no qmp command, so we use the human monitor command
5620         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5621         die $res if $res ne '';
5622     });
5623 }
5624
5625 # vzdump restore implementaion
5626
5627 sub tar_archive_read_firstfile {
5628     my $archive = shift;
5629
5630     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5631
5632     # try to detect archive type first
5633     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5634         die "unable to open file '$archive'\n";
5635     my $firstfile = <$fh>;
5636     kill 15, $pid;
5637     close $fh;
5638
5639     die "ERROR: archive contaions no data\n" if !$firstfile;
5640     chomp $firstfile;
5641
5642     return $firstfile;
5643 }
5644
5645 sub tar_restore_cleanup {
5646     my ($storecfg, $statfile) = @_;
5647
5648     print STDERR "starting cleanup\n";
5649
5650     if (my $fd = IO::File->new($statfile, "r")) {
5651         while (defined(my $line = <$fd>)) {
5652             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5653                 my $volid = $2;
5654                 eval {
5655                     if ($volid =~ m|^/|) {
5656                         unlink $volid || die 'unlink failed\n';
5657                     } else {
5658                         PVE::Storage::vdisk_free($storecfg, $volid);
5659                     }
5660                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5661                 };
5662                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5663             } else {
5664                 print STDERR "unable to parse line in statfile - $line";
5665             }
5666         }
5667         $fd->close();
5668     }
5669 }
5670
5671 sub restore_file_archive {
5672     my ($archive, $vmid, $user, $opts) = @_;
5673
5674     return restore_vma_archive($archive, $vmid, $user, $opts)
5675         if $archive eq '-';
5676
5677     my $info = PVE::Storage::archive_info($archive);
5678     my $format = $opts->{format} // $info->{format};
5679     my $comp = $info->{compression};
5680
5681     # try to detect archive format
5682     if ($format eq 'tar') {
5683         return restore_tar_archive($archive, $vmid, $user, $opts);
5684     } else {
5685         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5686     }
5687 }
5688
5689 # hepler to remove disks that will not be used after restore
5690 my $restore_cleanup_oldconf = sub {
5691     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5692
5693     PVE::QemuConfig->foreach_volume($oldconf, sub {
5694         my ($ds, $drive) = @_;
5695
5696         return if drive_is_cdrom($drive, 1);
5697
5698         my $volid = $drive->{file};
5699         return if !$volid || $volid =~ m|^/|;
5700
5701         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5702         return if !$path || !$owner || ($owner != $vmid);
5703
5704         # Note: only delete disk we want to restore
5705         # other volumes will become unused
5706         if ($virtdev_hash->{$ds}) {
5707             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5708             if (my $err = $@) {
5709                 warn $err;
5710             }
5711         }
5712     });
5713
5714     # delete vmstate files, after the restore we have no snapshots anymore
5715     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5716         my $snap = $oldconf->{snapshots}->{$snapname};
5717         if ($snap->{vmstate}) {
5718             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5719             if (my $err = $@) {
5720                 warn $err;
5721             }
5722         }
5723     }
5724 };
5725
5726 # Helper to parse vzdump backup device hints
5727 #
5728 # $rpcenv: Environment, used to ckeck storage permissions
5729 # $user: User ID, to check storage permissions
5730 # $storecfg: Storage configuration
5731 # $fh: the file handle for reading the configuration
5732 # $devinfo: should contain device sizes for all backu-up'ed devices
5733 # $options: backup options (pool, default storage)
5734 #
5735 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5736 my $parse_backup_hints = sub {
5737     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5738
5739     my $virtdev_hash = {};
5740
5741     while (defined(my $line = <$fh>)) {
5742         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5743             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5744             die "archive does not contain data for drive '$virtdev'\n"
5745                 if !$devinfo->{$devname};
5746
5747             if (defined($options->{storage})) {
5748                 $storeid = $options->{storage} || 'local';
5749             } elsif (!$storeid) {
5750                 $storeid = 'local';
5751             }
5752             $format = 'raw' if !$format;
5753             $devinfo->{$devname}->{devname} = $devname;
5754             $devinfo->{$devname}->{virtdev} = $virtdev;
5755             $devinfo->{$devname}->{format} = $format;
5756             $devinfo->{$devname}->{storeid} = $storeid;
5757
5758             # check permission on storage
5759             my $pool = $options->{pool}; # todo: do we need that?
5760             if ($user ne 'root@pam') {
5761                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5762             }
5763
5764             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5765         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5766             my $virtdev = $1;
5767             my $drive = parse_drive($virtdev, $2);
5768             if (drive_is_cloudinit($drive)) {
5769                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5770                 $storeid = $options->{storage} if defined ($options->{storage});
5771                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5772                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5773
5774                 $virtdev_hash->{$virtdev} = {
5775                     format => $format,
5776                     storeid => $storeid,
5777                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5778                     is_cloudinit => 1,
5779                 };
5780             }
5781         }
5782     }
5783
5784     return $virtdev_hash;
5785 };
5786
5787 # Helper to allocate and activate all volumes required for a restore
5788 #
5789 # $storecfg: Storage configuration
5790 # $virtdev_hash: as returned by parse_backup_hints()
5791 #
5792 # Returns: { $virtdev => $volid }
5793 my $restore_allocate_devices = sub {
5794     my ($storecfg, $virtdev_hash, $vmid) = @_;
5795
5796     my $map = {};
5797     foreach my $virtdev (sort keys %$virtdev_hash) {
5798         my $d = $virtdev_hash->{$virtdev};
5799         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5800         my $storeid = $d->{storeid};
5801         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5802
5803         # test if requested format is supported
5804         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5805         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5806         $d->{format} = $defFormat if !$supported;
5807
5808         my $name;
5809         if ($d->{is_cloudinit}) {
5810             $name = "vm-$vmid-cloudinit";
5811             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5812             if ($scfg->{path}) {
5813                 $name .= ".$d->{format}";
5814             }
5815         }
5816
5817         my $volid = PVE::Storage::vdisk_alloc(
5818             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5819
5820         print STDERR "new volume ID is '$volid'\n";
5821         $d->{volid} = $volid;
5822
5823         PVE::Storage::activate_volumes($storecfg, [$volid]);
5824
5825         $map->{$virtdev} = $volid;
5826     }
5827
5828     return $map;
5829 };
5830
5831 my $restore_update_config_line = sub {
5832     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5833
5834     return if $line =~ m/^\#qmdump\#/;
5835     return if $line =~ m/^\#vzdump\#/;
5836     return if $line =~ m/^lock:/;
5837     return if $line =~ m/^unused\d+:/;
5838     return if $line =~ m/^parent:/;
5839
5840     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5841     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5842         # try to convert old 1.X settings
5843         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5844         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5845             my ($model, $macaddr) = split(/\=/, $devconfig);
5846             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5847             my $net = {
5848                 model => $model,
5849                 bridge => "vmbr$ind",
5850                 macaddr => $macaddr,
5851             };
5852             my $netstr = print_net($net);
5853
5854             print $outfd "net$cookie->{netcount}: $netstr\n";
5855             $cookie->{netcount}++;
5856         }
5857     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5858         my ($id, $netstr) = ($1, $2);
5859         my $net = parse_net($netstr);
5860         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5861         $netstr = print_net($net);
5862         print $outfd "$id: $netstr\n";
5863     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5864         my $virtdev = $1;
5865         my $value = $3;
5866         my $di = parse_drive($virtdev, $value);
5867         if (defined($di->{backup}) && !$di->{backup}) {
5868             print $outfd "#$line";
5869         } elsif ($map->{$virtdev}) {
5870             delete $di->{format}; # format can change on restore
5871             $di->{file} = $map->{$virtdev};
5872             $value = print_drive($di);
5873             print $outfd "$virtdev: $value\n";
5874         } else {
5875             print $outfd $line;
5876         }
5877     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5878         my $vmgenid = $1;
5879         if ($vmgenid ne '0') {
5880             # always generate a new vmgenid if there was a valid one setup
5881             $vmgenid = generate_uuid();
5882         }
5883         print $outfd "vmgenid: $vmgenid\n";
5884     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5885         my ($uuid, $uuid_str);
5886         UUID::generate($uuid);
5887         UUID::unparse($uuid, $uuid_str);
5888         my $smbios1 = parse_smbios1($2);
5889         $smbios1->{uuid} = $uuid_str;
5890         print $outfd $1.print_smbios1($smbios1)."\n";
5891     } else {
5892         print $outfd $line;
5893     }
5894 };
5895
5896 my $restore_deactivate_volumes = sub {
5897     my ($storecfg, $devinfo) = @_;
5898
5899     my $vollist = [];
5900     foreach my $devname (keys %$devinfo) {
5901         my $volid = $devinfo->{$devname}->{volid};
5902         push @$vollist, $volid if $volid;
5903     }
5904
5905     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5906 };
5907
5908 my $restore_destroy_volumes = sub {
5909     my ($storecfg, $devinfo) = @_;
5910
5911     foreach my $devname (keys %$devinfo) {
5912         my $volid = $devinfo->{$devname}->{volid};
5913         next if !$volid;
5914         eval {
5915             if ($volid =~ m|^/|) {
5916                 unlink $volid || die 'unlink failed\n';
5917             } else {
5918                 PVE::Storage::vdisk_free($storecfg, $volid);
5919             }
5920             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5921         };
5922         print STDERR "unable to cleanup '$volid' - $@" if $@;
5923     }
5924 };
5925
5926 sub scan_volids {
5927     my ($cfg, $vmid) = @_;
5928
5929     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5930
5931     my $volid_hash = {};
5932     foreach my $storeid (keys %$info) {
5933         foreach my $item (@{$info->{$storeid}}) {
5934             next if !($item->{volid} && $item->{size});
5935             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5936             $volid_hash->{$item->{volid}} = $item;
5937         }
5938     }
5939
5940     return $volid_hash;
5941 }
5942
5943 sub update_disk_config {
5944     my ($vmid, $conf, $volid_hash) = @_;
5945
5946     my $changes;
5947     my $prefix = "VM $vmid";
5948
5949     # used and unused disks
5950     my $referenced = {};
5951
5952     # Note: it is allowed to define multiple storages with same path (alias), so
5953     # we need to check both 'volid' and real 'path' (two different volid can point
5954     # to the same path).
5955
5956     my $referencedpath = {};
5957
5958     # update size info
5959     PVE::QemuConfig->foreach_volume($conf, sub {
5960         my ($opt, $drive) = @_;
5961
5962         my $volid = $drive->{file};
5963         return if !$volid;
5964         my $volume = $volid_hash->{$volid};
5965
5966         # mark volid as "in-use" for next step
5967         $referenced->{$volid} = 1;
5968         if ($volume && (my $path = $volume->{path})) {
5969             $referencedpath->{$path} = 1;
5970         }
5971
5972         return if drive_is_cdrom($drive);
5973         return if !$volume;
5974
5975         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
5976         if (defined($updated)) {
5977             $changes = 1;
5978             $conf->{$opt} = print_drive($updated);
5979             print "$prefix ($opt): $msg\n";
5980         }
5981     });
5982
5983     # remove 'unusedX' entry if volume is used
5984     PVE::QemuConfig->foreach_unused_volume($conf, sub {
5985         my ($opt, $drive) = @_;
5986
5987         my $volid = $drive->{file};
5988         return if !$volid;
5989
5990         my $path;
5991         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5992         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5993             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
5994             $changes = 1;
5995             delete $conf->{$opt};
5996         }
5997
5998         $referenced->{$volid} = 1;
5999         $referencedpath->{$path} = 1 if $path;
6000     });
6001
6002     foreach my $volid (sort keys %$volid_hash) {
6003         next if $volid =~ m/vm-$vmid-state-/;
6004         next if $referenced->{$volid};
6005         my $path = $volid_hash->{$volid}->{path};
6006         next if !$path; # just to be sure
6007         next if $referencedpath->{$path};
6008         $changes = 1;
6009         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6010         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6011         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6012     }
6013
6014     return $changes;
6015 }
6016
6017 sub rescan {
6018     my ($vmid, $nolock, $dryrun) = @_;
6019
6020     my $cfg = PVE::Storage::config();
6021
6022     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6023     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6024     foreach my $stor (keys %{$cfg->{ids}}) {
6025         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6026     }
6027
6028     print "rescan volumes...\n";
6029     my $volid_hash = scan_volids($cfg, $vmid);
6030
6031     my $updatefn =  sub {
6032         my ($vmid) = @_;
6033
6034         my $conf = PVE::QemuConfig->load_config($vmid);
6035
6036         PVE::QemuConfig->check_lock($conf);
6037
6038         my $vm_volids = {};
6039         foreach my $volid (keys %$volid_hash) {
6040             my $info = $volid_hash->{$volid};
6041             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6042         }
6043
6044         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6045
6046         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6047     };
6048
6049     if (defined($vmid)) {
6050         if ($nolock) {
6051             &$updatefn($vmid);
6052         } else {
6053             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6054         }
6055     } else {
6056         my $vmlist = config_list();
6057         foreach my $vmid (keys %$vmlist) {
6058             if ($nolock) {
6059                 &$updatefn($vmid);
6060             } else {
6061                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6062             }
6063         }
6064     }
6065 }
6066
6067 sub restore_proxmox_backup_archive {
6068     my ($archive, $vmid, $user, $options) = @_;
6069
6070     my $storecfg = PVE::Storage::config();
6071
6072     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6073     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6074
6075     my $fingerprint = $scfg->{fingerprint};
6076     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6077
6078     my $repo = PVE::PBSClient::get_repository($scfg);
6079
6080     # This is only used for `pbs-restore`!
6081     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6082     local $ENV{PBS_PASSWORD} = $password;
6083     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6084
6085     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6086         PVE::Storage::parse_volname($storecfg, $archive);
6087
6088     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6089
6090     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6091
6092     my $tmpdir = "/var/tmp/vzdumptmp$$";
6093     rmtree $tmpdir;
6094     mkpath $tmpdir;
6095
6096     my $conffile = PVE::QemuConfig->config_file($vmid);
6097     my $tmpfn = "$conffile.$$.tmp";
6098      # disable interrupts (always do cleanups)
6099     local $SIG{INT} =
6100         local $SIG{TERM} =
6101         local $SIG{QUIT} =
6102         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6103
6104     # Note: $oldconf is undef if VM does not exists
6105     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6106     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6107
6108     my $rpcenv = PVE::RPCEnvironment::get();
6109     my $devinfo = {};
6110
6111     eval {
6112         # enable interrupts
6113         local $SIG{INT} =
6114             local $SIG{TERM} =
6115             local $SIG{QUIT} =
6116             local $SIG{HUP} =
6117             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6118
6119         my $cfgfn = "$tmpdir/qemu-server.conf";
6120         my $firewall_config_fn = "$tmpdir/fw.conf";
6121         my $index_fn = "$tmpdir/index.json";
6122
6123         my $cmd = "restore";
6124
6125         my $param = [$pbs_backup_name, "index.json", $index_fn];
6126         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6127         my $index = PVE::Tools::file_get_contents($index_fn);
6128         $index = decode_json($index);
6129
6130         # print Dumper($index);
6131         foreach my $info (@{$index->{files}}) {
6132             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6133                 my $devname = $1;
6134                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6135                     $devinfo->{$devname}->{size} = $1;
6136                 } else {
6137                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6138                 }
6139             }
6140         }
6141
6142         my $is_qemu_server_backup = scalar(
6143             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6144         );
6145         if (!$is_qemu_server_backup) {
6146             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6147         }
6148         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6149
6150         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6151         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6152
6153         if ($has_firewall_config) {
6154             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6155             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6156
6157             my $pve_firewall_dir = '/etc/pve/firewall';
6158             mkdir $pve_firewall_dir; # make sure the dir exists
6159             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6160         }
6161
6162         my $fh = IO::File->new($cfgfn, "r") ||
6163             die "unable to read qemu-server.conf - $!\n";
6164
6165         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6166
6167         # fixme: rate limit?
6168
6169         # create empty/temp config
6170         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6171
6172         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6173
6174         # allocate volumes
6175         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6176
6177         foreach my $virtdev (sort keys %$virtdev_hash) {
6178             my $d = $virtdev_hash->{$virtdev};
6179             next if $d->{is_cloudinit}; # no need to restore cloudinit
6180
6181             my $volid = $d->{volid};
6182
6183             my $path = PVE::Storage::path($storecfg, $volid);
6184
6185             # This is the ONLY user of the PBS_ env vars set on top of this function!
6186             my $pbs_restore_cmd = [
6187                 '/usr/bin/pbs-restore',
6188                 '--repository', $repo,
6189                 $pbs_backup_name,
6190                 "$d->{devname}.img.fidx",
6191                 $path,
6192                 '--verbose',
6193                 ];
6194
6195             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6196             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6197
6198             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6199                 push @$pbs_restore_cmd, '--skip-zero';
6200             }
6201
6202             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6203             print "restore proxmox backup image: $dbg_cmdstring\n";
6204             run_command($pbs_restore_cmd);
6205         }
6206
6207         $fh->seek(0, 0) || die "seek failed - $!\n";
6208
6209         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6210
6211         my $cookie = { netcount => 0 };
6212         while (defined(my $line = <$fh>)) {
6213             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6214         }
6215
6216         $fh->close();
6217         $outfd->close();
6218     };
6219     my $err = $@;
6220
6221     $restore_deactivate_volumes->($storecfg, $devinfo);
6222
6223     rmtree $tmpdir;
6224
6225     if ($err) {
6226         unlink $tmpfn;
6227         $restore_destroy_volumes->($storecfg, $devinfo);
6228         die $err;
6229     }
6230
6231     rename($tmpfn, $conffile) ||
6232         die "unable to commit configuration file '$conffile'\n";
6233
6234     PVE::Cluster::cfs_update(); # make sure we read new file
6235
6236     eval { rescan($vmid, 1); };
6237     warn $@ if $@;
6238 }
6239
6240 sub restore_vma_archive {
6241     my ($archive, $vmid, $user, $opts, $comp) = @_;
6242
6243     my $readfrom = $archive;
6244
6245     my $cfg = PVE::Storage::config();
6246     my $commands = [];
6247     my $bwlimit = $opts->{bwlimit};
6248
6249     my $dbg_cmdstring = '';
6250     my $add_pipe = sub {
6251         my ($cmd) = @_;
6252         push @$commands, $cmd;
6253         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6254         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6255         $readfrom = '-';
6256     };
6257
6258     my $input = undef;
6259     if ($archive eq '-') {
6260         $input = '<&STDIN';
6261     } else {
6262         # If we use a backup from a PVE defined storage we also consider that
6263         # storage's rate limit:
6264         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6265         if (defined($volid)) {
6266             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6267             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6268             if ($readlimit) {
6269                 print STDERR "applying read rate limit: $readlimit\n";
6270                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6271                 $add_pipe->($cstream);
6272             }
6273         }
6274     }
6275
6276     if ($comp) {
6277         my $info = PVE::Storage::decompressor_info('vma', $comp);
6278         my $cmd = $info->{decompressor};
6279         push @$cmd, $readfrom;
6280         $add_pipe->($cmd);
6281     }
6282
6283     my $tmpdir = "/var/tmp/vzdumptmp$$";
6284     rmtree $tmpdir;
6285
6286     # disable interrupts (always do cleanups)
6287     local $SIG{INT} =
6288         local $SIG{TERM} =
6289         local $SIG{QUIT} =
6290         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6291
6292     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6293     POSIX::mkfifo($mapfifo, 0600);
6294     my $fifofh;
6295     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6296
6297     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6298
6299     my $oldtimeout;
6300     my $timeout = 5;
6301
6302     my $devinfo = {};
6303
6304     my $rpcenv = PVE::RPCEnvironment::get();
6305
6306     my $conffile = PVE::QemuConfig->config_file($vmid);
6307     my $tmpfn = "$conffile.$$.tmp";
6308
6309     # Note: $oldconf is undef if VM does not exist
6310     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6311     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6312
6313     my %storage_limits;
6314
6315     my $print_devmap = sub {
6316         my $cfgfn = "$tmpdir/qemu-server.conf";
6317
6318         # we can read the config - that is already extracted
6319         my $fh = IO::File->new($cfgfn, "r") ||
6320             die "unable to read qemu-server.conf - $!\n";
6321
6322         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6323         if (-f $fwcfgfn) {
6324             my $pve_firewall_dir = '/etc/pve/firewall';
6325             mkdir $pve_firewall_dir; # make sure the dir exists
6326             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6327         }
6328
6329         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6330
6331         foreach my $key (keys %storage_limits) {
6332             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6333             next if !$limit;
6334             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6335             $storage_limits{$key} = $limit * 1024;
6336         }
6337
6338         foreach my $devname (keys %$devinfo) {
6339             die "found no device mapping information for device '$devname'\n"
6340                 if !$devinfo->{$devname}->{virtdev};
6341         }
6342
6343         # create empty/temp config
6344         if ($oldconf) {
6345             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6346             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6347         }
6348
6349         # allocate volumes
6350         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6351
6352         # print restore information to $fifofh
6353         foreach my $virtdev (sort keys %$virtdev_hash) {
6354             my $d = $virtdev_hash->{$virtdev};
6355             next if $d->{is_cloudinit}; # no need to restore cloudinit
6356
6357             my $storeid = $d->{storeid};
6358             my $volid = $d->{volid};
6359
6360             my $map_opts = '';
6361             if (my $limit = $storage_limits{$storeid}) {
6362                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6363             }
6364
6365             my $write_zeros = 1;
6366             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6367                 $write_zeros = 0;
6368             }
6369
6370             my $path = PVE::Storage::path($cfg, $volid);
6371
6372             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6373
6374             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6375         }
6376
6377         $fh->seek(0, 0) || die "seek failed - $!\n";
6378
6379         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6380
6381         my $cookie = { netcount => 0 };
6382         while (defined(my $line = <$fh>)) {
6383             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6384         }
6385
6386         $fh->close();
6387         $outfd->close();
6388     };
6389
6390     eval {
6391         # enable interrupts
6392         local $SIG{INT} =
6393             local $SIG{TERM} =
6394             local $SIG{QUIT} =
6395             local $SIG{HUP} =
6396             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6397         local $SIG{ALRM} = sub { die "got timeout\n"; };
6398
6399         $oldtimeout = alarm($timeout);
6400
6401         my $parser = sub {
6402             my $line = shift;
6403
6404             print "$line\n";
6405
6406             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6407                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6408                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6409             } elsif ($line =~ m/^CTIME: /) {
6410                 # we correctly received the vma config, so we can disable
6411                 # the timeout now for disk allocation (set to 10 minutes, so
6412                 # that we always timeout if something goes wrong)
6413                 alarm(600);
6414                 &$print_devmap();
6415                 print $fifofh "done\n";
6416                 my $tmp = $oldtimeout || 0;
6417                 $oldtimeout = undef;
6418                 alarm($tmp);
6419                 close($fifofh);
6420                 $fifofh = undef;
6421             }
6422         };
6423
6424         print "restore vma archive: $dbg_cmdstring\n";
6425         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6426     };
6427     my $err = $@;
6428
6429     alarm($oldtimeout) if $oldtimeout;
6430
6431     $restore_deactivate_volumes->($cfg, $devinfo);
6432
6433     close($fifofh) if $fifofh;
6434     unlink $mapfifo;
6435     rmtree $tmpdir;
6436
6437     if ($err) {
6438         unlink $tmpfn;
6439         $restore_destroy_volumes->($cfg, $devinfo);
6440         die $err;
6441     }
6442
6443     rename($tmpfn, $conffile) ||
6444         die "unable to commit configuration file '$conffile'\n";
6445
6446     PVE::Cluster::cfs_update(); # make sure we read new file
6447
6448     eval { rescan($vmid, 1); };
6449     warn $@ if $@;
6450 }
6451
6452 sub restore_tar_archive {
6453     my ($archive, $vmid, $user, $opts) = @_;
6454
6455     if ($archive ne '-') {
6456         my $firstfile = tar_archive_read_firstfile($archive);
6457         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6458             if $firstfile ne 'qemu-server.conf';
6459     }
6460
6461     my $storecfg = PVE::Storage::config();
6462
6463     # avoid zombie disks when restoring over an existing VM -> cleanup first
6464     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6465     # skiplock=1 because qmrestore has set the 'create' lock itself already
6466     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6467     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6468
6469     my $tocmd = "/usr/lib/qemu-server/qmextract";
6470
6471     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6472     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6473     $tocmd .= ' --prealloc' if $opts->{prealloc};
6474     $tocmd .= ' --info' if $opts->{info};
6475
6476     # tar option "xf" does not autodetect compression when read from STDIN,
6477     # so we pipe to zcat
6478     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6479         PVE::Tools::shellquote("--to-command=$tocmd");
6480
6481     my $tmpdir = "/var/tmp/vzdumptmp$$";
6482     mkpath $tmpdir;
6483
6484     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6485     local $ENV{VZDUMP_VMID} = $vmid;
6486     local $ENV{VZDUMP_USER} = $user;
6487
6488     my $conffile = PVE::QemuConfig->config_file($vmid);
6489     my $tmpfn = "$conffile.$$.tmp";
6490
6491     # disable interrupts (always do cleanups)
6492     local $SIG{INT} =
6493         local $SIG{TERM} =
6494         local $SIG{QUIT} =
6495         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6496
6497     eval {
6498         # enable interrupts
6499         local $SIG{INT} =
6500             local $SIG{TERM} =
6501             local $SIG{QUIT} =
6502             local $SIG{HUP} =
6503             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6504
6505         if ($archive eq '-') {
6506             print "extracting archive from STDIN\n";
6507             run_command($cmd, input => "<&STDIN");
6508         } else {
6509             print "extracting archive '$archive'\n";
6510             run_command($cmd);
6511         }
6512
6513         return if $opts->{info};
6514
6515         # read new mapping
6516         my $map = {};
6517         my $statfile = "$tmpdir/qmrestore.stat";
6518         if (my $fd = IO::File->new($statfile, "r")) {
6519             while (defined (my $line = <$fd>)) {
6520                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6521                     $map->{$1} = $2 if $1;
6522                 } else {
6523                     print STDERR "unable to parse line in statfile - $line\n";
6524                 }
6525             }
6526             $fd->close();
6527         }
6528
6529         my $confsrc = "$tmpdir/qemu-server.conf";
6530
6531         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6532
6533         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6534
6535         my $cookie = { netcount => 0 };
6536         while (defined (my $line = <$srcfd>)) {
6537             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6538         }
6539
6540         $srcfd->close();
6541         $outfd->close();
6542     };
6543     if (my $err = $@) {
6544         unlink $tmpfn;
6545         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6546         die $err;
6547     }
6548
6549     rmtree $tmpdir;
6550
6551     rename $tmpfn, $conffile ||
6552         die "unable to commit configuration file '$conffile'\n";
6553
6554     PVE::Cluster::cfs_update(); # make sure we read new file
6555
6556     eval { rescan($vmid, 1); };
6557     warn $@ if $@;
6558 };
6559
6560 sub foreach_storage_used_by_vm {
6561     my ($conf, $func) = @_;
6562
6563     my $sidhash = {};
6564
6565     PVE::QemuConfig->foreach_volume($conf, sub {
6566         my ($ds, $drive) = @_;
6567         return if drive_is_cdrom($drive);
6568
6569         my $volid = $drive->{file};
6570
6571         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6572         $sidhash->{$sid} = $sid if $sid;
6573     });
6574
6575     foreach my $sid (sort keys %$sidhash) {
6576         &$func($sid);
6577     }
6578 }
6579
6580 my $qemu_snap_storage = {
6581     rbd => 1,
6582 };
6583 sub do_snapshots_with_qemu {
6584     my ($storecfg, $volid) = @_;
6585
6586     my $storage_name = PVE::Storage::parse_volume_id($volid);
6587     my $scfg = $storecfg->{ids}->{$storage_name};
6588     die "could not find storage '$storage_name'\n" if !defined($scfg);
6589
6590     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6591         return 1;
6592     }
6593
6594     if ($volid =~ m/\.(qcow2|qed)$/){
6595         return 1;
6596     }
6597
6598     return;
6599 }
6600
6601 sub qga_check_running {
6602     my ($vmid, $nowarn) = @_;
6603
6604     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6605     if ($@) {
6606         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6607         return 0;
6608     }
6609     return 1;
6610 }
6611
6612 sub template_create {
6613     my ($vmid, $conf, $disk) = @_;
6614
6615     my $storecfg = PVE::Storage::config();
6616
6617     PVE::QemuConfig->foreach_volume($conf, sub {
6618         my ($ds, $drive) = @_;
6619
6620         return if drive_is_cdrom($drive);
6621         return if $disk && $ds ne $disk;
6622
6623         my $volid = $drive->{file};
6624         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6625
6626         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6627         $drive->{file} = $voliddst;
6628         $conf->{$ds} = print_drive($drive);
6629         PVE::QemuConfig->write_config($vmid, $conf);
6630     });
6631 }
6632
6633 sub convert_iscsi_path {
6634     my ($path) = @_;
6635
6636     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6637         my $portal = $1;
6638         my $target = $2;
6639         my $lun = $3;
6640
6641         my $initiator_name = get_initiator_name();
6642
6643         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6644                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6645     }
6646
6647     die "cannot convert iscsi path '$path', unkown format\n";
6648 }
6649
6650 sub qemu_img_convert {
6651     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6652
6653     my $storecfg = PVE::Storage::config();
6654     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6655     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6656
6657     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6658
6659     my $cachemode;
6660     my $src_path;
6661     my $src_is_iscsi = 0;
6662     my $src_format;
6663
6664     if ($src_storeid) {
6665         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6666         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6667         $src_format = qemu_img_format($src_scfg, $src_volname);
6668         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6669         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6670         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6671     } elsif (-f $src_volid) {
6672         $src_path = $src_volid;
6673         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6674             $src_format = $1;
6675         }
6676     }
6677
6678     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6679
6680     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6681     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6682     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6683     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6684
6685     my $cmd = [];
6686     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6687     push @$cmd, '-l', "snapshot.name=$snapname"
6688         if $snapname && $src_format && $src_format eq "qcow2";
6689     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6690     push @$cmd, '-T', $cachemode if defined($cachemode);
6691
6692     if ($src_is_iscsi) {
6693         push @$cmd, '--image-opts';
6694         $src_path = convert_iscsi_path($src_path);
6695     } elsif ($src_format) {
6696         push @$cmd, '-f', $src_format;
6697     }
6698
6699     if ($dst_is_iscsi) {
6700         push @$cmd, '--target-image-opts';
6701         $dst_path = convert_iscsi_path($dst_path);
6702     } else {
6703         push @$cmd, '-O', $dst_format;
6704     }
6705
6706     push @$cmd, $src_path;
6707
6708     if (!$dst_is_iscsi && $is_zero_initialized) {
6709         push @$cmd, "zeroinit:$dst_path";
6710     } else {
6711         push @$cmd, $dst_path;
6712     }
6713
6714     my $parser = sub {
6715         my $line = shift;
6716         if($line =~ m/\((\S+)\/100\%\)/){
6717             my $percent = $1;
6718             my $transferred = int($size * $percent / 100);
6719             my $remaining = $size - $transferred;
6720
6721             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6722         }
6723
6724     };
6725
6726     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6727     my $err = $@;
6728     die "copy failed: $err" if $err;
6729 }
6730
6731 sub qemu_img_format {
6732     my ($scfg, $volname) = @_;
6733
6734     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6735         return $1;
6736     } else {
6737         return "raw";
6738     }
6739 }
6740
6741 sub qemu_drive_mirror {
6742     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6743
6744     $jobs = {} if !$jobs;
6745
6746     my $qemu_target;
6747     my $format;
6748     $jobs->{"drive-$drive"} = {};
6749
6750     if ($dst_volid =~ /^nbd:/) {
6751         $qemu_target = $dst_volid;
6752         $format = "nbd";
6753     } else {
6754         my $storecfg = PVE::Storage::config();
6755         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6756
6757         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6758
6759         $format = qemu_img_format($dst_scfg, $dst_volname);
6760
6761         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6762
6763         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6764     }
6765
6766     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6767     $opts->{format} = $format if $format;
6768
6769     if (defined($src_bitmap)) {
6770         $opts->{sync} = 'incremental';
6771         $opts->{bitmap} = $src_bitmap;
6772         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6773     }
6774
6775     if (defined($bwlimit)) {
6776         $opts->{speed} = $bwlimit * 1024;
6777         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6778     } else {
6779         print "drive mirror is starting for drive-$drive\n";
6780     }
6781
6782     # if a job already runs for this device we get an error, catch it for cleanup
6783     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6784     if (my $err = $@) {
6785         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6786         warn "$@\n" if $@;
6787         die "mirroring error: $err\n";
6788     }
6789
6790     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6791 }
6792
6793 # $completion can be either
6794 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6795 # 'cancel': wait until all jobs are ready, block-job-cancel them
6796 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6797 sub qemu_drive_mirror_monitor {
6798     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6799
6800     $completion //= 'complete';
6801
6802     eval {
6803         my $err_complete = 0;
6804
6805         while (1) {
6806             die "storage migration timed out\n" if $err_complete > 300;
6807
6808             my $stats = mon_cmd($vmid, "query-block-jobs");
6809
6810             my $running_mirror_jobs = {};
6811             foreach my $stat (@$stats) {
6812                 next if $stat->{type} ne 'mirror';
6813                 $running_mirror_jobs->{$stat->{device}} = $stat;
6814             }
6815
6816             my $readycounter = 0;
6817
6818             foreach my $job (keys %$jobs) {
6819
6820                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6821                     print "$job : finished\n";
6822                     delete $jobs->{$job};
6823                     next;
6824                 }
6825
6826                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6827
6828                 my $busy = $running_mirror_jobs->{$job}->{busy};
6829                 my $ready = $running_mirror_jobs->{$job}->{ready};
6830                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6831                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6832                     my $remaining = $total - $transferred;
6833                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6834
6835                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6836                 }
6837
6838                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6839             }
6840
6841             last if scalar(keys %$jobs) == 0;
6842
6843             if ($readycounter == scalar(keys %$jobs)) {
6844                 print "all mirroring jobs are ready \n";
6845                 last if $completion eq 'skip'; #do the complete later
6846
6847                 if ($vmiddst && $vmiddst != $vmid) {
6848                     my $agent_running = $qga && qga_check_running($vmid);
6849                     if ($agent_running) {
6850                         print "freeze filesystem\n";
6851                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6852                     } else {
6853                         print "suspend vm\n";
6854                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6855                     }
6856
6857                     # if we clone a disk for a new target vm, we don't switch the disk
6858                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6859
6860                     if ($agent_running) {
6861                         print "unfreeze filesystem\n";
6862                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6863                     } else {
6864                         print "resume vm\n";
6865                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6866                     }
6867
6868                     last;
6869                 } else {
6870
6871                     foreach my $job (keys %$jobs) {
6872                         # try to switch the disk if source and destination are on the same guest
6873                         print "$job: Completing block job...\n";
6874
6875                         my $op;
6876                         if ($completion eq 'complete') {
6877                             $op = 'block-job-complete';
6878                         } elsif ($completion eq 'cancel') {
6879                             $op = 'block-job-cancel';
6880                         } else {
6881                             die "invalid completion value: $completion\n";
6882                         }
6883                         eval { mon_cmd($vmid, $op, device => $job) };
6884                         if ($@ =~ m/cannot be completed/) {
6885                             print "$job: Block job cannot be completed, try again.\n";
6886                             $err_complete++;
6887                         }else {
6888                             print "$job: Completed successfully.\n";
6889                             $jobs->{$job}->{complete} = 1;
6890                         }
6891                     }
6892                 }
6893             }
6894             sleep 1;
6895         }
6896     };
6897     my $err = $@;
6898
6899     if ($err) {
6900         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6901         die "mirroring error: $err";
6902     }
6903
6904 }
6905
6906 sub qemu_blockjobs_cancel {
6907     my ($vmid, $jobs) = @_;
6908
6909     foreach my $job (keys %$jobs) {
6910         print "$job: Cancelling block job\n";
6911         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6912         $jobs->{$job}->{cancel} = 1;
6913     }
6914
6915     while (1) {
6916         my $stats = mon_cmd($vmid, "query-block-jobs");
6917
6918         my $running_jobs = {};
6919         foreach my $stat (@$stats) {
6920             $running_jobs->{$stat->{device}} = $stat;
6921         }
6922
6923         foreach my $job (keys %$jobs) {
6924
6925             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6926                 print "$job: Done.\n";
6927                 delete $jobs->{$job};
6928             }
6929         }
6930
6931         last if scalar(keys %$jobs) == 0;
6932
6933         sleep 1;
6934     }
6935 }
6936
6937 sub clone_disk {
6938     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6939         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6940
6941     my $newvolid;
6942
6943     if (!$full) {
6944         print "create linked clone of drive $drivename ($drive->{file})\n";
6945         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6946         push @$newvollist, $newvolid;
6947     } else {
6948
6949         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6950         $storeid = $storage if $storage;
6951
6952         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6953
6954         print "create full clone of drive $drivename ($drive->{file})\n";
6955         my $name = undef;
6956         my $size = undef;
6957         if (drive_is_cloudinit($drive)) {
6958             $name = "vm-$newvmid-cloudinit";
6959             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6960             if ($scfg->{path}) {
6961                 $name .= ".$dst_format";
6962             }
6963             $snapname = undef;
6964             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6965         } elsif ($drivename eq 'efidisk0') {
6966             $size = get_efivars_size($conf);
6967         } else {
6968             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
6969         }
6970         $newvolid = PVE::Storage::vdisk_alloc(
6971             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
6972         );
6973         push @$newvollist, $newvolid;
6974
6975         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6976
6977         if (drive_is_cloudinit($drive)) {
6978             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
6979             # if this is the case, we have to complete any block-jobs still there from
6980             # previous drive-mirrors
6981             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
6982                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
6983             }
6984             goto no_data_clone;
6985         }
6986
6987         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6988         if (!$running || $snapname) {
6989             # TODO: handle bwlimits
6990             if ($drivename eq 'efidisk0') {
6991                 # the relevant data on the efidisk may be smaller than the source
6992                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
6993                 # that is given by the OVMF_VARS.fd
6994                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
6995                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
6996
6997                 # better for Ceph if block size is not too small, see bug #3324
6998                 my $bs = 1024*1024;
6999
7000                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7001                     "if=$src_path", "of=$dst_path"]);
7002             } else {
7003                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7004             }
7005         } else {
7006
7007             my $kvmver = get_running_qemu_version ($vmid);
7008             if (!min_version($kvmver, 2, 7)) {
7009                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7010                     if $drive->{iothread};
7011             }
7012
7013             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7014                 $completion, $qga, $bwlimit);
7015         }
7016     }
7017
7018 no_data_clone:
7019     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7020
7021     my $disk = $drive;
7022     $disk->{format} = undef;
7023     $disk->{file} = $newvolid;
7024     $disk->{size} = $size if defined($size);
7025
7026     return $disk;
7027 }
7028
7029 sub get_running_qemu_version {
7030     my ($vmid) = @_;
7031     my $res = mon_cmd($vmid, "query-version");
7032     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7033 }
7034
7035 sub qemu_use_old_bios_files {
7036     my ($machine_type) = @_;
7037
7038     return if !$machine_type;
7039
7040     my $use_old_bios_files = undef;
7041
7042     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7043         $machine_type = $1;
7044         $use_old_bios_files = 1;
7045     } else {
7046         my $version = extract_version($machine_type, kvm_user_version());
7047         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7048         # load new efi bios files on migration. So this hack is required to allow
7049         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7050         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7051         $use_old_bios_files = !min_version($version, 2, 4);
7052     }
7053
7054     return ($use_old_bios_files, $machine_type);
7055 }
7056
7057 sub get_efivars_size {
7058     my ($conf) = @_;
7059     my $arch = get_vm_arch($conf);
7060     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7061     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7062     return -s $ovmf_vars;
7063 }
7064
7065 sub update_efidisk_size {
7066     my ($conf) = @_;
7067
7068     return if !defined($conf->{efidisk0});
7069
7070     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7071     $disk->{size} = get_efivars_size($conf);
7072     $conf->{efidisk0} = print_drive($disk);
7073
7074     return;
7075 }
7076
7077 sub create_efidisk($$$$$) {
7078     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7079
7080     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7081     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7082
7083     my $vars_size_b = -s $ovmf_vars;
7084     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7085     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7086     PVE::Storage::activate_volumes($storecfg, [$volid]);
7087
7088     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7089     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7090
7091     return ($volid, $size/1024);
7092 }
7093
7094 sub vm_iothreads_list {
7095     my ($vmid) = @_;
7096
7097     my $res = mon_cmd($vmid, 'query-iothreads');
7098
7099     my $iothreads = {};
7100     foreach my $iothread (@$res) {
7101         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7102     }
7103
7104     return $iothreads;
7105 }
7106
7107 sub scsihw_infos {
7108     my ($conf, $drive) = @_;
7109
7110     my $maxdev = 0;
7111
7112     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7113         $maxdev = 7;
7114     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7115         $maxdev = 1;
7116     } else {
7117         $maxdev = 256;
7118     }
7119
7120     my $controller = int($drive->{index} / $maxdev);
7121     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7122         ? "virtioscsi"
7123         : "scsihw";
7124
7125     return ($maxdev, $controller, $controller_prefix);
7126 }
7127
7128 sub windows_version {
7129     my ($ostype) = @_;
7130
7131     return 0 if !$ostype;
7132
7133     my $winversion = 0;
7134
7135     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7136         $winversion = 5;
7137     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7138         $winversion = 6;
7139     } elsif ($ostype =~ m/^win(\d+)$/) {
7140         $winversion = $1;
7141     }
7142
7143     return $winversion;
7144 }
7145
7146 sub resolve_dst_disk_format {
7147         my ($storecfg, $storeid, $src_volname, $format) = @_;
7148         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7149
7150         if (!$format) {
7151             # if no target format is specified, use the source disk format as hint
7152             if ($src_volname) {
7153                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7154                 $format = qemu_img_format($scfg, $src_volname);
7155             } else {
7156                 return $defFormat;
7157             }
7158         }
7159
7160         # test if requested format is supported - else use default
7161         my $supported = grep { $_ eq $format } @$validFormats;
7162         $format = $defFormat if !$supported;
7163         return $format;
7164 }
7165
7166 # NOTE: if this logic changes, please update docs & possibly gui logic
7167 sub find_vmstate_storage {
7168     my ($conf, $storecfg) = @_;
7169
7170     # first, return storage from conf if set
7171     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7172
7173     my ($target, $shared, $local);
7174
7175     foreach_storage_used_by_vm($conf, sub {
7176         my ($sid) = @_;
7177         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7178         my $dst = $scfg->{shared} ? \$shared : \$local;
7179         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7180     });
7181
7182     # second, use shared storage where VM has at least one disk
7183     # third, use local storage where VM has at least one disk
7184     # fall back to local storage
7185     $target = $shared // $local // 'local';
7186
7187     return $target;
7188 }
7189
7190 sub generate_uuid {
7191     my ($uuid, $uuid_str);
7192     UUID::generate($uuid);
7193     UUID::unparse($uuid, $uuid_str);
7194     return $uuid_str;
7195 }
7196
7197 sub generate_smbios1_uuid {
7198     return "uuid=".generate_uuid();
7199 }
7200
7201 sub nbd_stop {
7202     my ($vmid) = @_;
7203
7204     mon_cmd($vmid, 'nbd-server-stop');
7205 }
7206
7207 sub create_reboot_request {
7208     my ($vmid) = @_;
7209     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7210         or die "failed to create reboot trigger file: $!\n";
7211     close($fh);
7212 }
7213
7214 sub clear_reboot_request {
7215     my ($vmid) = @_;
7216     my $path = "/run/qemu-server/$vmid.reboot";
7217     my $res = 0;
7218
7219     $res = unlink($path);
7220     die "could not remove reboot request for $vmid: $!"
7221         if !$res && $! != POSIX::ENOENT;
7222
7223     return $res;
7224 }
7225
7226 sub bootorder_from_legacy {
7227     my ($conf, $bootcfg) = @_;
7228
7229     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7230     my $bootindex_hash = {};
7231     my $i = 1;
7232     foreach my $o (split(//, $boot)) {
7233         $bootindex_hash->{$o} = $i*100;
7234         $i++;
7235     }
7236
7237     my $bootorder = {};
7238
7239     PVE::QemuConfig->foreach_volume($conf, sub {
7240         my ($ds, $drive) = @_;
7241
7242         if (drive_is_cdrom ($drive, 1)) {
7243             if ($bootindex_hash->{d}) {
7244                 $bootorder->{$ds} = $bootindex_hash->{d};
7245                 $bootindex_hash->{d} += 1;
7246             }
7247         } elsif ($bootindex_hash->{c}) {
7248             $bootorder->{$ds} = $bootindex_hash->{c}
7249                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7250             $bootindex_hash->{c} += 1;
7251         }
7252     });
7253
7254     if ($bootindex_hash->{n}) {
7255         for (my $i = 0; $i < $MAX_NETS; $i++) {
7256             my $netname = "net$i";
7257             next if !$conf->{$netname};
7258             $bootorder->{$netname} = $bootindex_hash->{n};
7259             $bootindex_hash->{n} += 1;
7260         }
7261     }
7262
7263     return $bootorder;
7264 }
7265
7266 # Generate default device list for 'boot: order=' property. Matches legacy
7267 # default boot order, but with explicit device names. This is important, since
7268 # the fallback for when neither 'order' nor the old format is specified relies
7269 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7270 sub get_default_bootdevices {
7271     my ($conf) = @_;
7272
7273     my @ret = ();
7274
7275     # harddisk
7276     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7277     push @ret, $first if $first;
7278
7279     # cdrom
7280     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7281     push @ret, $first if $first;
7282
7283     # network
7284     for (my $i = 0; $i < $MAX_NETS; $i++) {
7285         my $netname = "net$i";
7286         next if !$conf->{$netname};
7287         push @ret, $netname;
7288         last;
7289     }
7290
7291     return \@ret;
7292 }
7293
7294 sub device_bootorder {
7295     my ($conf) = @_;
7296
7297     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7298
7299     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7300
7301     my $bootorder = {};
7302     if (!defined($boot) || $boot->{legacy}) {
7303         $bootorder = bootorder_from_legacy($conf, $boot);
7304     } elsif ($boot->{order}) {
7305         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7306         for my $dev (PVE::Tools::split_list($boot->{order})) {
7307             $bootorder->{$dev} = $i++;
7308         }
7309     }
7310
7311     return $bootorder;
7312 }
7313
7314 # bash completion helper
7315
7316 sub complete_backup_archives {
7317     my ($cmdname, $pname, $cvalue) = @_;
7318
7319     my $cfg = PVE::Storage::config();
7320
7321     my $storeid;
7322
7323     if ($cvalue =~ m/^([^:]+):/) {
7324         $storeid = $1;
7325     }
7326
7327     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7328
7329     my $res = [];
7330     foreach my $id (keys %$data) {
7331         foreach my $item (@{$data->{$id}}) {
7332             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7333             push @$res, $item->{volid} if defined($item->{volid});
7334         }
7335     }
7336
7337     return $res;
7338 }
7339
7340 my $complete_vmid_full = sub {
7341     my ($running) = @_;
7342
7343     my $idlist = vmstatus();
7344
7345     my $res = [];
7346
7347     foreach my $id (keys %$idlist) {
7348         my $d = $idlist->{$id};
7349         if (defined($running)) {
7350             next if $d->{template};
7351             next if $running && $d->{status} ne 'running';
7352             next if !$running && $d->{status} eq 'running';
7353         }
7354         push @$res, $id;
7355
7356     }
7357     return $res;
7358 };
7359
7360 sub complete_vmid {
7361     return &$complete_vmid_full();
7362 }
7363
7364 sub complete_vmid_stopped {
7365     return &$complete_vmid_full(0);
7366 }
7367
7368 sub complete_vmid_running {
7369     return &$complete_vmid_full(1);
7370 }
7371
7372 sub complete_storage {
7373
7374     my $cfg = PVE::Storage::config();
7375     my $ids = $cfg->{ids};
7376
7377     my $res = [];
7378     foreach my $sid (keys %$ids) {
7379         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7380         next if !$ids->{$sid}->{content}->{images};
7381         push @$res, $sid;
7382     }
7383
7384     return $res;
7385 }
7386
7387 sub complete_migration_storage {
7388     my ($cmd, $param, $current_value, $all_args) = @_;
7389
7390     my $targetnode = @$all_args[1];
7391
7392     my $cfg = PVE::Storage::config();
7393     my $ids = $cfg->{ids};
7394
7395     my $res = [];
7396     foreach my $sid (keys %$ids) {
7397         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7398         next if !$ids->{$sid}->{content}->{images};
7399         push @$res, $sid;
7400     }
7401
7402     return $res;
7403 }
7404
7405 sub vm_is_paused {
7406     my ($vmid) = @_;
7407     my $qmpstatus = eval {
7408         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7409         mon_cmd($vmid, "query-status");
7410     };
7411     warn "$@\n" if $@;
7412     return $qmpstatus && $qmpstatus->{status} eq "paused";
7413 }
7414
7415 1;