PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file);
  30 use PVE::CGroup;
  31 use PVE::DataCenterConfig;
  32 use PVE::Exception qw(raise raise_param_exc);
  33 use PVE::GuestHelpers qw(safe_string_ne safe_num_ne safe_boolean_ne);
  34 use PVE::INotify;
  35 use PVE::JSONSchema qw(get_standard_option parse_property_string);
  36 use PVE::ProcFSTools;
  37 use PVE::PBSClient;
  38 use PVE::RPCEnvironment;
  39 use PVE::Storage;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use PVE::Tools qw(run_command file_read_firstline file_get_contents dir_glob_foreach get_host_arch $IPV6RE);
  43
  44 use PVE::QMPClient;
  45 use PVE::QemuConfig;
  46 use PVE::QemuServer::Helpers qw(min_version config_aware_timeout);
  47 use PVE::QemuServer::Cloudinit;
  48 use PVE::QemuServer::CGroup;
  49 use PVE::QemuServer::CPUConfig qw(print_cpu_device get_cpu_options);
  50 use PVE::QemuServer::Drive qw(is_valid_drivename drive_is_cloudinit drive_is_cdrom parse_drive print_drive);
  51 use PVE::QemuServer::Machine;
  52 use PVE::QemuServer::Memory;
  53 use PVE::QemuServer::Monitor qw(mon_cmd);
  54 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port parse_hostpci);
  55 use PVE::QemuServer::USB qw(parse_usb_device);
  56
  57 my $have_sdn;
  58 eval {
  59     require PVE::Network::SDN::Zones;
  60     $have_sdn = 1;
  61 };
  62
  63 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  64 my $OVMF = {
  65     x86_64 => [
  66         "$EDK2_FW_BASE/OVMF_CODE.fd",
  67         "$EDK2_FW_BASE/OVMF_VARS.fd"
  68     ],
  69     aarch64 => [
  70         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  71         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  72     ],
  73 };
  74
  75 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  76
  77 # Note about locking: we use flock on the config file protect
  78 # against concurent actions.
  79 # Aditionaly, we have a 'lock' setting in the config file. This
  80 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  81 # allowed when such lock is set. But you can ignore this kind of
  82 # lock with the --skiplock flag.
  83
  84 cfs_register_file('/qemu-server/',
  85                   \&parse_vm_config,
  86                   \&write_vm_config);
  87
  88 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  89     description => "Some command save/restore state from this location.",
  90     type => 'string',
  91     maxLength => 128,
  92     optional => 1,
  93 });
  94
  95 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  96         description => "Specifies the Qemu machine type.",
  97         type => 'string',
  98         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\+pve\d+)?(\.pxe)?|virt(?:-\d+(\.\d+)+)?(\+pve\d+)?)',
  99         maxLength => 40,
 100         optional => 1,
 101 });
 102
 103
 104 sub map_storage {
 105     my ($map, $source) = @_;
 106
 107     return $source if !defined($map);
 108
 109     return $map->{entries}->{$source}
 110         if $map->{entries} && defined($map->{entries}->{$source});
 111
 112     return $map->{default} if $map->{default};
 113
 114     # identity (fallback)
 115     return $source;
 116 }
 117
 118 PVE::JSONSchema::register_standard_option('pve-targetstorage', {
 119     description => "Mapping from source to target storages. Providing only a single storage ID maps all source storages to that storage. Providing the special value '1' will map each source storage to itself.",
 120     type => 'string',
 121     format => 'storagepair-list',
 122     optional => 1,
 123 });
 124
 125 #no warnings 'redefine';
 126
 127 my $nodename_cache;
 128 sub nodename {
 129     $nodename_cache //= PVE::INotify::nodename();
 130     return $nodename_cache;
 131 }
 132
 133 my $watchdog_fmt = {
 134     model => {
 135         default_key => 1,
 136         type => 'string',
 137         enum => [qw(i6300esb ib700)],
 138         description => "Watchdog type to emulate.",
 139         default => 'i6300esb',
 140         optional => 1,
 141     },
 142     action => {
 143         type => 'string',
 144         enum => [qw(reset shutdown poweroff pause debug none)],
 145         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 146         optional => 1,
 147     },
 148 };
 149 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 150
 151 my $agent_fmt = {
 152     enabled => {
 153         description => "Enable/disable Qemu GuestAgent.",
 154         type => 'boolean',
 155         default => 0,
 156         default_key => 1,
 157     },
 158     fstrim_cloned_disks => {
 159         description => "Run fstrim after moving a disk or migrating the VM.",
 160         type => 'boolean',
 161         optional => 1,
 162         default => 0
 163     },
 164     type => {
 165         description => "Select the agent type",
 166         type => 'string',
 167         default => 'virtio',
 168         optional => 1,
 169         enum => [qw(virtio isa)],
 170     },
 171 };
 172
 173 my $vga_fmt = {
 174     type => {
 175         description => "Select the VGA type.",
 176         type => 'string',
 177         default => 'std',
 178         optional => 1,
 179         default_key => 1,
 180         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 181     },
 182     memory => {
 183         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 184         type => 'integer',
 185         optional => 1,
 186         minimum => 4,
 187         maximum => 512,
 188     },
 189 };
 190
 191 my $ivshmem_fmt = {
 192     size => {
 193         type => 'integer',
 194         minimum => 1,
 195         description => "The size of the file in MB.",
 196     },
 197     name => {
 198         type => 'string',
 199         pattern => '[a-zA-Z0-9\-]+',
 200         optional => 1,
 201         format_description => 'string',
 202         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 203     },
 204 };
 205
 206 my $audio_fmt = {
 207     device => {
 208         type => 'string',
 209         enum => [qw(ich9-intel-hda intel-hda AC97)],
 210         description =>  "Configure an audio device."
 211     },
 212     driver =>  {
 213         type => 'string',
 214         enum => ['spice', 'none'],
 215         default => 'spice',
 216         optional => 1,
 217         description => "Driver backend for the audio device."
 218     },
 219 };
 220
 221 my $spice_enhancements_fmt = {
 222     foldersharing => {
 223         type => 'boolean',
 224         optional => 1,
 225         default => '0',
 226         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 227     },
 228     videostreaming =>  {
 229         type => 'string',
 230         enum => ['off', 'all', 'filter'],
 231         default => 'off',
 232         optional => 1,
 233         description => "Enable video streaming. Uses compression for detected video streams."
 234     },
 235 };
 236
 237 my $rng_fmt = {
 238     source => {
 239         type => 'string',
 240         enum => ['/dev/urandom', '/dev/random', '/dev/hwrng'],
 241         default_key => 1,
 242         description => "The file on the host to gather entropy from. In most"
 243                      . " cases /dev/urandom should be preferred over /dev/random"
 244                      . " to avoid entropy-starvation issues on the host. Using"
 245                      . " urandom does *not* decrease security in any meaningful"
 246                      . " way, as it's still seeded from real entropy, and the"
 247                      . " bytes provided will most likely be mixed with real"
 248                      . " entropy on the guest as well. /dev/hwrng can be used"
 249                      . " to pass through a hardware RNG from the host.",
 250     },
 251     max_bytes => {
 252         type => 'integer',
 253         description => "Maximum bytes of entropy injected into the guest every"
 254                      . " 'period' milliseconds. Prefer a lower value when using"
 255                      . " /dev/random as source. Use 0 to disable limiting"
 256                      . " (potentially dangerous!).",
 257         optional => 1,
 258
 259         # default is 1 KiB/s, provides enough entropy to the guest to avoid
 260         # boot-starvation issues (e.g. systemd etc...) while allowing no chance
 261         # of overwhelming the host, provided we're reading from /dev/urandom
 262         default => 1024,
 263     },
 264     period => {
 265         type => 'integer',
 266         description => "Every 'period' milliseconds the entropy-injection quota"
 267                      . " is reset, allowing the guest to retrieve another"
 268                      . " 'max_bytes' of entropy.",
 269         optional => 1,
 270         default => 1000,
 271     },
 272 };
 273
 274 my $confdesc = {
 275     onboot => {
 276         optional => 1,
 277         type => 'boolean',
 278         description => "Specifies whether a VM will be started during system bootup.",
 279         default => 0,
 280     },
 281     autostart => {
 282         optional => 1,
 283         type => 'boolean',
 284         description => "Automatic restart after crash (currently ignored).",
 285         default => 0,
 286     },
 287     hotplug => {
 288         optional => 1,
 289         type => 'string', format => 'pve-hotplug-features',
 290         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 291         default => 'network,disk,usb',
 292     },
 293     reboot => {
 294         optional => 1,
 295         type => 'boolean',
 296         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 297         default => 1,
 298     },
 299     lock => {
 300         optional => 1,
 301         type => 'string',
 302         description => "Lock/unlock the VM.",
 303         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 304     },
 305     cpulimit => {
 306         optional => 1,
 307         type => 'number',
 308         description => "Limit of CPU usage.",
 309         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 310         minimum => 0,
 311         maximum => 128,
 312         default => 0,
 313     },
 314     cpuunits => {
 315         optional => 1,
 316         type => 'integer',
 317         description => "CPU weight for a VM.",
 318         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 319         minimum => 2,
 320         maximum => 262144,
 321         default => 1024,
 322     },
 323     memory => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 327         minimum => 16,
 328         default => 512,
 329     },
 330     balloon => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 334         minimum => 0,
 335     },
 336     shares => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 340         minimum => 0,
 341         maximum => 50000,
 342         default => 1000,
 343     },
 344     keyboard => {
 345         optional => 1,
 346         type => 'string',
 347         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 348                        "It should not be necessary to set it.",
 349         enum => PVE::Tools::kvmkeymaplist(),
 350         default => undef,
 351     },
 352     name => {
 353         optional => 1,
 354         type => 'string', format => 'dns-name',
 355         description => "Set a name for the VM. Only used on the configuration web interface.",
 356     },
 357     scsihw => {
 358         optional => 1,
 359         type => 'string',
 360         description => "SCSI controller model",
 361         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 362         default => 'lsi',
 363     },
 364     description => {
 365         optional => 1,
 366         type => 'string',
 367         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 368     },
 369     ostype => {
 370         optional => 1,
 371         type => 'string',
 372         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 373         description => "Specify guest operating system.",
 374         verbose_description => <<EODESC,
 375 Specify guest operating system. This is used to enable special
 376 optimization/features for specific operating systems:
 377
 378 [horizontal]
 379 other;; unspecified OS
 380 wxp;; Microsoft Windows XP
 381 w2k;; Microsoft Windows 2000
 382 w2k3;; Microsoft Windows 2003
 383 w2k8;; Microsoft Windows 2008
 384 wvista;; Microsoft Windows Vista
 385 win7;; Microsoft Windows 7
 386 win8;; Microsoft Windows 8/2012/2012r2
 387 win10;; Microsoft Windows 10/2016/2019
 388 l24;; Linux 2.4 Kernel
 389 l26;; Linux 2.6 - 5.X Kernel
 390 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 391 EODESC
 392     },
 393     boot => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-boot',
 396         description => "Specify guest boot order. Use with 'order=', usage with"
 397                      . " no key or 'legacy=' is deprecated.",
 398     },
 399     bootdisk => {
 400         optional => 1,
 401         type => 'string', format => 'pve-qm-bootdisk',
 402         description => "Enable booting from specified disk. Deprecated: Use 'boot: order=foo;bar' instead.",
 403         pattern => '(ide|sata|scsi|virtio)\d+',
 404     },
 405     smp => {
 406         optional => 1,
 407         type => 'integer',
 408         description => "The number of CPUs. Please use option -sockets instead.",
 409         minimum => 1,
 410         default => 1,
 411     },
 412     sockets => {
 413         optional => 1,
 414         type => 'integer',
 415         description => "The number of CPU sockets.",
 416         minimum => 1,
 417         default => 1,
 418     },
 419     cores => {
 420         optional => 1,
 421         type => 'integer',
 422         description => "The number of cores per socket.",
 423         minimum => 1,
 424         default => 1,
 425     },
 426     numa => {
 427         optional => 1,
 428         type => 'boolean',
 429         description => "Enable/disable NUMA.",
 430         default => 0,
 431     },
 432     hugepages => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Enable/disable hugepages memory.",
 436         enum => [qw(any 2 1024)],
 437     },
 438     keephugepages => {
 439         optional => 1,
 440         type => 'boolean',
 441         default => 0,
 442         description => "Use together with hugepages. If enabled, hugepages will not not be deleted"
 443             ." after VM shutdown and can be used for subsequent starts.",
 444     },
 445     vcpus => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Number of hotplugged vcpus.",
 449         minimum => 1,
 450         default => 0,
 451     },
 452     acpi => {
 453         optional => 1,
 454         type => 'boolean',
 455         description => "Enable/disable ACPI.",
 456         default => 1,
 457     },
 458     agent => {
 459         optional => 1,
 460         description => "Enable/disable Qemu GuestAgent and its properties.",
 461         type => 'string',
 462         format => $agent_fmt,
 463     },
 464     kvm => {
 465         optional => 1,
 466         type => 'boolean',
 467         description => "Enable/disable KVM hardware virtualization.",
 468         default => 1,
 469     },
 470     tdf => {
 471         optional => 1,
 472         type => 'boolean',
 473         description => "Enable/disable time drift fix.",
 474         default => 0,
 475     },
 476     localtime => {
 477         optional => 1,
 478         type => 'boolean',
 479         description => "Set the real time clock to local time. This is enabled by default if ostype"
 480             ." indicates a Microsoft OS.",
 481     },
 482     freeze => {
 483         optional => 1,
 484         type => 'boolean',
 485         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 486     },
 487     vga => {
 488         optional => 1,
 489         type => 'string', format => $vga_fmt,
 490         description => "Configure the VGA hardware.",
 491         verbose_description => "Configure the VGA Hardware. If you want to use high resolution"
 492             ." modes (>= 1280x1024x16) you may need to increase the vga memory option. Since QEMU"
 493             ." 2.9 the default VGA display type is 'std' for all OS types besides some Windows"
 494             ." versions (XP and older) which use 'cirrus'. The 'qxl' option enables the SPICE"
 495             ." display server. For win* OS you can select how many independent displays you want,"
 496             ." Linux guests can add displays them self.\nYou can also run without any graphic card,"
 497             ." using a serial device as terminal.",
 498     },
 499     watchdog => {
 500         optional => 1,
 501         type => 'string', format => 'pve-qm-watchdog',
 502         description => "Create a virtual hardware watchdog device.",
 503         verbose_description => "Create a virtual hardware watchdog device. Once enabled (by a guest"
 504             ." action), the watchdog must be periodically polled by an agent inside the guest or"
 505             ." else the watchdog will reset the guest (or execute the respective action specified)",
 506     },
 507     startdate => {
 508         optional => 1,
 509         type => 'string',
 510         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 511         description => "Set the initial date of the real time clock. Valid format for date are:"
 512             ."'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 513         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 514         default => 'now',
 515     },
 516     startup =>  get_standard_option('pve-startup-order'),
 517     template => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Enable/disable Template.",
 521         default => 0,
 522     },
 523     args => {
 524         optional => 1,
 525         type => 'string',
 526         description => "Arbitrary arguments passed to kvm.",
 527         verbose_description => <<EODESCR,
 528 Arbitrary arguments passed to kvm, for example:
 529
 530 args: -no-reboot -no-hpet
 531
 532 NOTE: this option is for experts only.
 533 EODESCR
 534     },
 535     tablet => {
 536         optional => 1,
 537         type => 'boolean',
 538         default => 1,
 539         description => "Enable/disable the USB tablet device.",
 540         verbose_description => "Enable/disable the USB tablet device. This device is usually needed"
 541             ." to allow absolute mouse positioning with VNC. Else the mouse runs out of sync with"
 542             ." normal VNC clients. If you're running lots of console-only guests on one host, you"
 543             ." may consider disabling this to save some context switches. This is turned off by"
 544             ." default if you use spice (`qm set <vmid> --vga qxl`).",
 545     },
 546     migrate_speed => {
 547         optional => 1,
 548         type => 'integer',
 549         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 550         minimum => 0,
 551         default => 0,
 552     },
 553     migrate_downtime => {
 554         optional => 1,
 555         type => 'number',
 556         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 557         minimum => 0,
 558         default => 0.1,
 559     },
 560     cdrom => {
 561         optional => 1,
 562         type => 'string', format => 'pve-qm-ide',
 563         typetext => '<volume>',
 564         description => "This is an alias for option -ide2",
 565     },
 566     cpu => {
 567         optional => 1,
 568         description => "Emulated CPU type.",
 569         type => 'string',
 570         format => 'pve-vm-cpu-conf',
 571     },
 572     parent => get_standard_option('pve-snapshot-name', {
 573         optional => 1,
 574         description => "Parent snapshot name. This is used internally, and should not be modified.",
 575     }),
 576     snaptime => {
 577         optional => 1,
 578         description => "Timestamp for snapshots.",
 579         type => 'integer',
 580         minimum => 0,
 581     },
 582     vmstate => {
 583         optional => 1,
 584         type => 'string', format => 'pve-volume-id',
 585         description => "Reference to a volume which stores the VM state. This is used internally"
 586             ." for snapshots.",
 587     },
 588     vmstatestorage => get_standard_option('pve-storage-id', {
 589         description => "Default storage for VM state volumes/files.",
 590         optional => 1,
 591     }),
 592     runningmachine => get_standard_option('pve-qemu-machine', {
 593         description => "Specifies the QEMU machine type of the running vm. This is used internally"
 594             ." for snapshots.",
 595     }),
 596     runningcpu => {
 597         description => "Specifies the QEMU '-cpu' parameter of the running vm. This is used"
 598             ." internally for snapshots.",
 599         optional => 1,
 600         type => 'string',
 601         pattern => $PVE::QemuServer::CPUConfig::qemu_cmdline_cpu_re,
 602         format_description => 'QEMU -cpu parameter'
 603     },
 604     machine => get_standard_option('pve-qemu-machine'),
 605     arch => {
 606         description => "Virtual processor architecture. Defaults to the host.",
 607         optional => 1,
 608         type => 'string',
 609         enum => [qw(x86_64 aarch64)],
 610     },
 611     smbios1 => {
 612         description => "Specify SMBIOS type 1 fields.",
 613         type => 'string', format => 'pve-qm-smbios1',
 614         maxLength => 512,
 615         optional => 1,
 616     },
 617     protection => {
 618         optional => 1,
 619         type => 'boolean',
 620         description => "Sets the protection flag of the VM. This will disable the remove VM and"
 621             ." remove disk operations.",
 622         default => 0,
 623     },
 624     bios => {
 625         optional => 1,
 626         type => 'string',
 627         enum => [ qw(seabios ovmf) ],
 628         description => "Select BIOS implementation.",
 629         default => 'seabios',
 630     },
 631     vmgenid => {
 632         type => 'string',
 633         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 634         format_description => 'UUID',
 635         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0'"
 636             ." to disable explicitly.",
 637         verbose_description => "The VM generation ID (vmgenid) device exposes a 128-bit integer"
 638             ." value identifier to the guest OS. This allows to notify the guest operating system"
 639             ." when the virtual machine is executed with a different configuration (e.g. snapshot"
 640             ." execution or creation from a template). The guest operating system notices the"
 641             ." change, and is then able to react as appropriate by marking its copies of"
 642             ." distributed databases as dirty, re-initializing its random number generator, etc.\n"
 643             ."Note that auto-creation only works when done through API/CLI create or update methods"
 644             .", but not when manually editing the config file.",
 645         default => "1 (autogenerated)",
 646         optional => 1,
 647     },
 648     hookscript => {
 649         type => 'string',
 650         format => 'pve-volume-id',
 651         optional => 1,
 652         description => "Script that will be executed during various steps in the vms lifetime.",
 653     },
 654     ivshmem => {
 655         type => 'string',
 656         format => $ivshmem_fmt,
 657         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to"
 658             ." the host.",
 659         optional => 1,
 660     },
 661     audio0 => {
 662         type => 'string',
 663         format => $audio_fmt,
 664         description => "Configure a audio device, useful in combination with QXL/Spice.",
 665         optional => 1
 666     },
 667     spice_enhancements => {
 668         type => 'string',
 669         format => $spice_enhancements_fmt,
 670         description => "Configure additional enhancements for SPICE.",
 671         optional => 1
 672     },
 673     tags => {
 674         type => 'string', format => 'pve-tag-list',
 675         description => 'Tags of the VM. This is only meta information.',
 676         optional => 1,
 677     },
 678     rng0 => {
 679         type => 'string',
 680         format => $rng_fmt,
 681         description => "Configure a VirtIO-based Random Number Generator.",
 682         optional => 1,
 683     },
 684 };
 685
 686 my $cicustom_fmt = {
 687     meta => {
 688         type => 'string',
 689         optional => 1,
 690         description => 'Specify a custom file containing all meta data passed to the VM via"
 691             ." cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 692         format => 'pve-volume-id',
 693         format_description => 'volume',
 694     },
 695     network => {
 696         type => 'string',
 697         optional => 1,
 698         description => 'Specify a custom file containing all network data passed to the VM via'
 699             .' cloud-init.',
 700         format => 'pve-volume-id',
 701         format_description => 'volume',
 702     },
 703     user => {
 704         type => 'string',
 705         optional => 1,
 706         description => 'Specify a custom file containing all user data passed to the VM via'
 707             .' cloud-init.',
 708         format => 'pve-volume-id',
 709         format_description => 'volume',
 710     },
 711 };
 712 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 713
 714 my $confdesc_cloudinit = {
 715     citype => {
 716         optional => 1,
 717         type => 'string',
 718         description => 'Specifies the cloud-init configuration format. The default depends on the'
 719             .' configured operating system type (`ostype`. We use the `nocloud` format for Linux,'
 720             .' and `configdrive2` for windows.',
 721         enum => ['configdrive2', 'nocloud', 'opennebula'],
 722     },
 723     ciuser => {
 724         optional => 1,
 725         type => 'string',
 726         description => "cloud-init: User name to change ssh keys and password for instead of the"
 727             ." image's configured default user.",
 728     },
 729     cipassword => {
 730         optional => 1,
 731         type => 'string',
 732         description => 'cloud-init: Password to assign the user. Using this is generally not'
 733             .' recommended. Use ssh keys instead. Also note that older cloud-init versions do not'
 734             .' support hashed passwords.',
 735     },
 736     cicustom => {
 737         optional => 1,
 738         type => 'string',
 739         description => 'cloud-init: Specify custom files to replace the automatically generated'
 740             .' ones at start.',
 741         format => 'pve-qm-cicustom',
 742     },
 743     searchdomain => {
 744         optional => 1,
 745         type => 'string',
 746         description => "cloud-init: Sets DNS search domains for a container. Create will'
 747             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 748             .' are set.",
 749     },
 750     nameserver => {
 751         optional => 1,
 752         type => 'string', format => 'address-list',
 753         description => "cloud-init: Sets DNS server IP address for a container. Create will'
 754             .' automatically use the setting from the host if neither searchdomain nor nameserver'
 755             .' are set.",
 756     },
 757     sshkeys => {
 758         optional => 1,
 759         type => 'string',
 760         format => 'urlencoded',
 761         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 762     },
 763 };
 764
 765 # what about other qemu settings ?
 766 #cpu => 'string',
 767 #machine => 'string',
 768 #fda => 'file',
 769 #fdb => 'file',
 770 #mtdblock => 'file',
 771 #sd => 'file',
 772 #pflash => 'file',
 773 #snapshot => 'bool',
 774 #bootp => 'file',
 775 ##tftp => 'dir',
 776 ##smb => 'dir',
 777 #kernel => 'file',
 778 #append => 'string',
 779 #initrd => 'file',
 780 ##soundhw => 'string',
 781
 782 while (my ($k, $v) = each %$confdesc) {
 783     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 784 }
 785
 786 my $MAX_USB_DEVICES = 5;
 787 my $MAX_NETS = 32;
 788 my $MAX_SERIAL_PORTS = 4;
 789 my $MAX_PARALLEL_PORTS = 3;
 790 my $MAX_NUMA = 8;
 791
 792 my $numa_fmt = {
 793     cpus => {
 794         type => "string",
 795         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 796         description => "CPUs accessing this NUMA node.",
 797         format_description => "id[-id];...",
 798     },
 799     memory => {
 800         type => "number",
 801         description => "Amount of memory this NUMA node provides.",
 802         optional => 1,
 803     },
 804     hostnodes => {
 805         type => "string",
 806         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 807         description => "Host NUMA nodes to use.",
 808         format_description => "id[-id];...",
 809         optional => 1,
 810     },
 811     policy => {
 812         type => 'string',
 813         enum => [qw(preferred bind interleave)],
 814         description => "NUMA allocation policy.",
 815         optional => 1,
 816     },
 817 };
 818 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 819 my $numadesc = {
 820     optional => 1,
 821     type => 'string', format => $numa_fmt,
 822     description => "NUMA topology.",
 823 };
 824 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 825
 826 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 827     $confdesc->{"numa$i"} = $numadesc;
 828 }
 829
 830 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 831                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 832                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 833 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 834
 835 my $net_fmt_bridge_descr = <<__EOD__;
 836 Bridge to attach the network device to. The Proxmox VE standard bridge
 837 is called 'vmbr0'.
 838
 839 If you do not specify a bridge, we create a kvm user (NATed) network
 840 device, which provides DHCP and DNS services. The following addresses
 841 are used:
 842
 843  10.0.2.2   Gateway
 844  10.0.2.3   DNS Server
 845  10.0.2.4   SMB Server
 846
 847 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 848 __EOD__
 849
 850 my $net_fmt = {
 851     macaddr  => get_standard_option('mac-addr', {
 852         description => "MAC address. That address must be unique withing your network. This is"
 853             ." automatically generated if not specified.",
 854     }),
 855     model => {
 856         type => 'string',
 857         description => "Network Card Model. The 'virtio' model provides the best performance with"
 858             ." very low CPU overhead. If your guest does not support this driver, it is usually"
 859             ." best to use 'e1000'.",
 860         enum => $nic_model_list,
 861         default_key => 1,
 862     },
 863     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 864     bridge => {
 865         type => 'string',
 866         description => $net_fmt_bridge_descr,
 867         format_description => 'bridge',
 868         pattern => '[-_.\w\d]+',
 869         optional => 1,
 870     },
 871     queues => {
 872         type => 'integer',
 873         minimum => 0, maximum => 16,
 874         description => 'Number of packet queues to be used on the device.',
 875         optional => 1,
 876     },
 877     rate => {
 878         type => 'number',
 879         minimum => 0,
 880         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 881         optional => 1,
 882     },
 883     tag => {
 884         type => 'integer',
 885         minimum => 1, maximum => 4094,
 886         description => 'VLAN tag to apply to packets on this interface.',
 887         optional => 1,
 888     },
 889     trunks => {
 890         type => 'string',
 891         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 892         description => 'VLAN trunks to pass through this interface.',
 893         format_description => 'vlanid[;vlanid...]',
 894         optional => 1,
 895     },
 896     firewall => {
 897         type => 'boolean',
 898         description => 'Whether this interface should be protected by the firewall.',
 899         optional => 1,
 900     },
 901     link_down => {
 902         type => 'boolean',
 903         description => 'Whether this interface should be disconnected (like pulling the plug).',
 904         optional => 1,
 905     },
 906     mtu => {
 907         type => 'integer',
 908         minimum => 1, maximum => 65520,
 909         description => "Force MTU, for VirtIO only. Set to '1' to use the bridge MTU",
 910         optional => 1,
 911     },
 912 };
 913
 914 my $netdesc = {
 915     optional => 1,
 916     type => 'string', format => $net_fmt,
 917     description => "Specify network devices.",
 918 };
 919
 920 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 921
 922 my $ipconfig_fmt = {
 923     ip => {
 924         type => 'string',
 925         format => 'pve-ipv4-config',
 926         format_description => 'IPv4Format/CIDR',
 927         description => 'IPv4 address in CIDR format.',
 928         optional => 1,
 929         default => 'dhcp',
 930     },
 931     gw => {
 932         type => 'string',
 933         format => 'ipv4',
 934         format_description => 'GatewayIPv4',
 935         description => 'Default gateway for IPv4 traffic.',
 936         optional => 1,
 937         requires => 'ip',
 938     },
 939     ip6 => {
 940         type => 'string',
 941         format => 'pve-ipv6-config',
 942         format_description => 'IPv6Format/CIDR',
 943         description => 'IPv6 address in CIDR format.',
 944         optional => 1,
 945         default => 'dhcp',
 946     },
 947     gw6 => {
 948         type => 'string',
 949         format => 'ipv6',
 950         format_description => 'GatewayIPv6',
 951         description => 'Default gateway for IPv6 traffic.',
 952         optional => 1,
 953         requires => 'ip6',
 954     },
 955 };
 956 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 957 my $ipconfigdesc = {
 958     optional => 1,
 959     type => 'string', format => 'pve-qm-ipconfig',
 960     description => <<'EODESCR',
 961 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 962
 963 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 964
 965 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit
 966 gateway should be provided.
 967 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 968
 969 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using
 970 dhcp on IPv4.
 971 EODESCR
 972 };
 973 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 974
 975 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 976     $confdesc->{"net$i"} = $netdesc;
 977     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 978 }
 979
 980 foreach my $key (keys %$confdesc_cloudinit) {
 981     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 982 }
 983
 984 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 985 sub verify_volume_id_or_qm_path {
 986     my ($volid, $noerr) = @_;
 987
 988     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 989         return $volid;
 990     }
 991
 992     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 993     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 994     if ($@) {
 995         return if $noerr;
 996         die $@;
 997     }
 998     return $volid;
 999 }
1000
1001 my $usb_fmt = {
1002     host => {
1003         default_key => 1,
1004         type => 'string', format => 'pve-qm-usb-device',
1005         format_description => 'HOSTUSBDEVICE|spice',
1006         description => <<EODESCR,
1007 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1008
1009  'bus-port(.port)*' (decimal numbers) or
1010  'vendor_id:product_id' (hexadeciaml numbers) or
1011  'spice'
1012
1013 You can use the 'lsusb -t' command to list existing usb devices.
1014
1015 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1016 machines - use with special care.
1017
1018 The value 'spice' can be used to add a usb redirection devices for spice.
1019 EODESCR
1020     },
1021     usb3 => {
1022         optional => 1,
1023         type => 'boolean',
1024         description => "Specifies whether if given host option is a USB3 device or port.",
1025         default => 0,
1026     },
1027 };
1028
1029 my $usbdesc = {
1030     optional => 1,
1031     type => 'string', format => $usb_fmt,
1032     description => "Configure an USB device (n is 0 to 4).",
1033 };
1034 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1035
1036 my $serialdesc = {
1037         optional => 1,
1038         type => 'string',
1039         pattern => '(/dev/.+|socket)',
1040         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1041         verbose_description =>  <<EODESCR,
1042 Create a serial device inside the VM (n is 0 to 3), and pass through a
1043 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1044 host side (use 'qm terminal' to open a terminal connection).
1045
1046 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines -
1047 use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052
1053 my $paralleldesc= {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1057         description =>  "Map host parallel devices (n is 0 to 2).",
1058         verbose_description =>  <<EODESCR,
1059 Map host parallel devices (n is 0 to 2).
1060
1061 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such
1062 machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $PVE::QemuServer::PCI::hostpcidesc;
1078 }
1079
1080 for my $key (keys %{$PVE::QemuServer::Drive::drivedesc_hash}) {
1081     $confdesc->{$key} = $PVE::QemuServer::Drive::drivedesc_hash->{$key};
1082 }
1083
1084 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1085     $confdesc->{"usb$i"} = $usbdesc;
1086 }
1087
1088 my $boot_fmt = {
1089     legacy => {
1090         optional => 1,
1091         default_key => 1,
1092         type => 'string',
1093         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n)."
1094                      . " Deprecated, use 'order=' instead.",
1095         pattern => '[acdn]{1,4}',
1096         format_description => "[acdn]{1,4}",
1097
1098         # note: this is also the fallback if boot: is not given at all
1099         default => 'cdn',
1100     },
1101     order => {
1102         optional => 1,
1103         type => 'string',
1104         format => 'pve-qm-bootdev-list',
1105         format_description => "device[;device...]",
1106         description => <<EODESC,
1107 The guest will attempt to boot from devices in the order they appear here.
1108
1109 Disks, optical drives and passed-through storage USB devices will be directly
1110 booted from, NICs will load PXE, and PCIe devices will either behave like disks
1111 (e.g. NVMe) or load an option ROM (e.g. RAID controller, hardware NIC).
1112
1113 Note that only devices in this list will be marked as bootable and thus loaded
1114 by the guest firmware (BIOS/UEFI). If you require multiple disks for booting
1115 (e.g. software-raid), you need to specify all of them here.
1116
1117 Overrides the deprecated 'legacy=[acdn]*' value when given.
1118 EODESC
1119     },
1120 };
1121 PVE::JSONSchema::register_format('pve-qm-boot', $boot_fmt);
1122
1123 PVE::JSONSchema::register_format('pve-qm-bootdev', \&verify_bootdev);
1124 sub verify_bootdev {
1125     my ($dev, $noerr) = @_;
1126
1127     return $dev if PVE::QemuServer::Drive::is_valid_drivename($dev) && $dev !~ m/^efidisk/;
1128
1129     my $check = sub {
1130         my ($base) = @_;
1131         return 0 if $dev !~ m/^$base\d+$/;
1132         return 0 if !$confdesc->{$dev};
1133         return 1;
1134     };
1135
1136     return $dev if $check->("net");
1137     return $dev if $check->("usb");
1138     return $dev if $check->("hostpci");
1139
1140     return if $noerr;
1141     die "invalid boot device '$dev'\n";
1142 }
1143
1144 sub print_bootorder {
1145     my ($devs) = @_;
1146     return "" if !@$devs;
1147     my $data = { order => join(';', @$devs) };
1148     return PVE::JSONSchema::print_property_string($data, $boot_fmt);
1149 }
1150
1151 my $kvm_api_version = 0;
1152
1153 sub kvm_version {
1154     return $kvm_api_version if $kvm_api_version;
1155
1156     open my $fh, '<', '/dev/kvm' or return;
1157
1158     # 0xae00 => KVM_GET_API_VERSION
1159     $kvm_api_version = ioctl($fh, 0xae00, 0);
1160     close($fh);
1161
1162     return $kvm_api_version;
1163 }
1164
1165 my $kvm_user_version = {};
1166 my $kvm_mtime = {};
1167
1168 sub kvm_user_version {
1169     my ($binary) = @_;
1170
1171     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1172     my $st = stat($binary);
1173
1174     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1175     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1176         $cachedmtime == $st->mtime;
1177
1178     $kvm_user_version->{$binary} = 'unknown';
1179     $kvm_mtime->{$binary} = $st->mtime;
1180
1181     my $code = sub {
1182         my $line = shift;
1183         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1184             $kvm_user_version->{$binary} = $2;
1185         }
1186     };
1187
1188     eval { run_command([$binary, '--version'], outfunc => $code); };
1189     warn $@ if $@;
1190
1191     return $kvm_user_version->{$binary};
1192
1193 }
1194 my sub extract_version {
1195     my ($machine_type, $version) = @_;
1196     $version = kvm_user_version() if !defined($version);
1197     PVE::QemuServer::Machine::extract_version($machine_type, $version)
1198 }
1199
1200 sub kernel_has_vhost_net {
1201     return -c '/dev/vhost-net';
1202 }
1203
1204 sub option_exists {
1205     my $key = shift;
1206     return defined($confdesc->{$key});
1207 }
1208
1209 my $cdrom_path;
1210 sub get_cdrom_path {
1211
1212     return  $cdrom_path if $cdrom_path;
1213
1214     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1215     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1216     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1217 }
1218
1219 sub get_iso_path {
1220     my ($storecfg, $vmid, $cdrom) = @_;
1221
1222     if ($cdrom eq 'cdrom') {
1223         return get_cdrom_path();
1224     } elsif ($cdrom eq 'none') {
1225         return '';
1226     } elsif ($cdrom =~ m|^/|) {
1227         return $cdrom;
1228     } else {
1229         return PVE::Storage::path($storecfg, $cdrom);
1230     }
1231 }
1232
1233 # try to convert old style file names to volume IDs
1234 sub filename_to_volume_id {
1235     my ($vmid, $file, $media) = @_;
1236
1237      if (!($file eq 'none' || $file eq 'cdrom' ||
1238           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1239
1240         return if $file =~ m|/|;
1241
1242         if ($media && $media eq 'cdrom') {
1243             $file = "local:iso/$file";
1244         } else {
1245             $file = "local:$vmid/$file";
1246         }
1247     }
1248
1249     return $file;
1250 }
1251
1252 sub verify_media_type {
1253     my ($opt, $vtype, $media) = @_;
1254
1255     return if !$media;
1256
1257     my $etype;
1258     if ($media eq 'disk') {
1259         $etype = 'images';
1260     } elsif ($media eq 'cdrom') {
1261         $etype = 'iso';
1262     } else {
1263         die "internal error";
1264     }
1265
1266     return if ($vtype eq $etype);
1267
1268     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1269 }
1270
1271 sub cleanup_drive_path {
1272     my ($opt, $storecfg, $drive) = @_;
1273
1274     # try to convert filesystem paths to volume IDs
1275
1276     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1277         ($drive->{file} !~ m|^/dev/.+|) &&
1278         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1279         ($drive->{file} !~ m/^\d+$/)) {
1280         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1281         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"})
1282             if !$vtype;
1283         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1284         verify_media_type($opt, $vtype, $drive->{media});
1285         $drive->{file} = $volid;
1286     }
1287
1288     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1289 }
1290
1291 sub parse_hotplug_features {
1292     my ($data) = @_;
1293
1294     my $res = {};
1295
1296     return $res if $data eq '0';
1297
1298     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1299
1300     foreach my $feature (PVE::Tools::split_list($data)) {
1301         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1302             $res->{$1} = 1;
1303         } else {
1304             die "invalid hotplug feature '$feature'\n";
1305         }
1306     }
1307     return $res;
1308 }
1309
1310 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1311 sub pve_verify_hotplug_features {
1312     my ($value, $noerr) = @_;
1313
1314     return $value if parse_hotplug_features($value);
1315
1316     return if $noerr;
1317
1318     die "unable to parse hotplug option\n";
1319 }
1320
1321 sub scsi_inquiry {
1322     my($fh, $noerr) = @_;
1323
1324     my $SG_IO = 0x2285;
1325     my $SG_GET_VERSION_NUM = 0x2282;
1326
1327     my $versionbuf = "\x00" x 8;
1328     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1329     if (!$ret) {
1330         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1331         return;
1332     }
1333     my $version = unpack("I", $versionbuf);
1334     if ($version < 30000) {
1335         die "scsi generic interface too old\n"  if !$noerr;
1336         return;
1337     }
1338
1339     my $buf = "\x00" x 36;
1340     my $sensebuf = "\x00" x 8;
1341     my $cmd = pack("C x3 C x1", 0x12, 36);
1342
1343     # see /usr/include/scsi/sg.h
1344     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1345
1346     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1347                       length($sensebuf), 0, length($buf), $buf,
1348                       $cmd, $sensebuf, 6000);
1349
1350     $ret = ioctl($fh, $SG_IO, $packet);
1351     if (!$ret) {
1352         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1353         return;
1354     }
1355
1356     my @res = unpack($sg_io_hdr_t, $packet);
1357     if ($res[17] || $res[18]) {
1358         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1359         return;
1360     }
1361
1362     my $res = {};
1363     (my $byte0, my $byte1, $res->{vendor},
1364      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1365
1366     $res->{removable} = $byte1 & 128 ? 1 : 0;
1367     $res->{type} = $byte0 & 31;
1368
1369     return $res;
1370 }
1371
1372 sub path_is_scsi {
1373     my ($path) = @_;
1374
1375     my $fh = IO::File->new("+<$path") || return;
1376     my $res = scsi_inquiry($fh, 1);
1377     close($fh);
1378
1379     return $res;
1380 }
1381
1382 sub print_tabletdevice_full {
1383     my ($conf, $arch) = @_;
1384
1385     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1386
1387     # we use uhci for old VMs because tablet driver was buggy in older qemu
1388     my $usbbus;
1389     if (PVE::QemuServer::Machine::machine_type_is_q35($conf) || $arch eq 'aarch64') {
1390         $usbbus = 'ehci';
1391     } else {
1392         $usbbus = 'uhci';
1393     }
1394
1395     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1396 }
1397
1398 sub print_keyboarddevice_full {
1399     my ($conf, $arch, $machine) = @_;
1400
1401     return if $arch ne 'aarch64';
1402
1403     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1404 }
1405
1406 sub print_drivedevice_full {
1407     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1408
1409     my $device = '';
1410     my $maxdev = 0;
1411
1412     my $drive_id = "$drive->{interface}$drive->{index}";
1413     if ($drive->{interface} eq 'virtio') {
1414         my $pciaddr = print_pci_addr("$drive_id", $bridges, $arch, $machine_type);
1415         $device = "virtio-blk-pci,drive=drive-$drive_id,id=${drive_id}${pciaddr}";
1416         $device .= ",iothread=iothread-$drive_id" if $drive->{iothread};
1417     } elsif ($drive->{interface} eq 'scsi') {
1418
1419         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1420         my $unit = $drive->{index} % $maxdev;
1421         my $devicetype = 'hd';
1422         my $path = '';
1423         if (drive_is_cdrom($drive)) {
1424             $devicetype = 'cd';
1425         } else {
1426             if ($drive->{file} =~ m|^/|) {
1427                 $path = $drive->{file};
1428                 if (my $info = path_is_scsi($path)) {
1429                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1430                         $devicetype = 'block';
1431                     } elsif ($info->{type} == 1) { # tape
1432                         $devicetype = 'generic';
1433                     }
1434                 }
1435             } else {
1436                  $path = PVE::Storage::path($storecfg, $drive->{file});
1437             }
1438
1439             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1440             my $version = extract_version($machine_type, kvm_user_version());
1441             if ($path =~ m/^iscsi\:\/\// &&
1442                !min_version($version, 4, 1)) {
1443                 $devicetype = 'generic';
1444             }
1445         }
1446
1447         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1448             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit";
1449         } else {
1450             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0"
1451                 .",lun=$drive->{index}";
1452         }
1453         $device .= ",drive=drive-$drive_id,id=$drive_id";
1454
1455         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1456             $device .= ",rotation_rate=1";
1457         }
1458         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1459
1460     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1461         my $maxdev = ($drive->{interface} eq 'sata') ? $PVE::QemuServer::Drive::MAX_SATA_DISKS : 2;
1462         my $controller = int($drive->{index} / $maxdev);
1463         my $unit = $drive->{index} % $maxdev;
1464         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1465
1466         $device = "ide-$devicetype";
1467         if ($drive->{interface} eq 'ide') {
1468             $device .= ",bus=ide.$controller,unit=$unit";
1469         } else {
1470             $device .= ",bus=ahci$controller.$unit";
1471         }
1472         $device .= ",drive=drive-$drive_id,id=$drive_id";
1473
1474         if ($devicetype eq 'hd') {
1475             if (my $model = $drive->{model}) {
1476                 $model = URI::Escape::uri_unescape($model);
1477                 $device .= ",model=$model";
1478             }
1479             if ($drive->{ssd}) {
1480                 $device .= ",rotation_rate=1";
1481             }
1482         }
1483         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1484     } elsif ($drive->{interface} eq 'usb') {
1485         die "implement me";
1486         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1487     } else {
1488         die "unsupported interface type";
1489     }
1490
1491     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1492
1493     if (my $serial = $drive->{serial}) {
1494         $serial = URI::Escape::uri_unescape($serial);
1495         $device .= ",serial=$serial";
1496     }
1497
1498
1499     return $device;
1500 }
1501
1502 sub get_initiator_name {
1503     my $initiator;
1504
1505     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return;
1506     while (defined(my $line = <$fh>)) {
1507         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1508         $initiator = $1;
1509         last;
1510     }
1511     $fh->close();
1512
1513     return $initiator;
1514 }
1515
1516 sub print_drive_commandline_full {
1517     my ($storecfg, $vmid, $drive) = @_;
1518
1519     my $path;
1520     my $volid = $drive->{file};
1521     my $format;
1522
1523     if (drive_is_cdrom($drive)) {
1524         $path = get_iso_path($storecfg, $vmid, $volid);
1525     } else {
1526         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1527         if ($storeid) {
1528             $path = PVE::Storage::path($storecfg, $volid);
1529             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1530             $format = qemu_img_format($scfg, $volname);
1531         } else {
1532             $path = $volid;
1533             $format = "raw";
1534         }
1535    }
1536
1537     my $opts = '';
1538     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1539     foreach my $o (@qemu_drive_options) {
1540         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1541     }
1542
1543     # snapshot only accepts on|off
1544     if (defined($drive->{snapshot})) {
1545         my $v = $drive->{snapshot} ? 'on' : 'off';
1546         $opts .= ",snapshot=$v";
1547     }
1548
1549     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1550         my ($dir, $qmpname) = @$type;
1551         if (my $v = $drive->{"mbps$dir"}) {
1552             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1553         }
1554         if (my $v = $drive->{"mbps${dir}_max"}) {
1555             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1556         }
1557         if (my $v = $drive->{"bps${dir}_max_length"}) {
1558             $opts .= ",throttling.bps$qmpname-max-length=$v";
1559         }
1560         if (my $v = $drive->{"iops${dir}"}) {
1561             $opts .= ",throttling.iops$qmpname=$v";
1562         }
1563         if (my $v = $drive->{"iops${dir}_max"}) {
1564             $opts .= ",throttling.iops$qmpname-max=$v";
1565         }
1566         if (my $v = $drive->{"iops${dir}_max_length"}) {
1567             $opts .= ",throttling.iops$qmpname-max-length=$v";
1568         }
1569     }
1570
1571     $opts .= ",format=$format" if $format && !$drive->{format};
1572
1573     my $cache_direct = 0;
1574
1575     if (my $cache = $drive->{cache}) {
1576         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1577     } elsif (!drive_is_cdrom($drive)) {
1578         $opts .= ",cache=none";
1579         $cache_direct = 1;
1580     }
1581
1582     # aio native works only with O_DIRECT
1583     if (!$drive->{aio}) {
1584         if($cache_direct) {
1585             $opts .= ",aio=native";
1586         } else {
1587             $opts .= ",aio=threads";
1588         }
1589     }
1590
1591     if (!drive_is_cdrom($drive)) {
1592         my $detectzeroes;
1593         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1594             $detectzeroes = 'off';
1595         } elsif ($drive->{discard}) {
1596             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1597         } else {
1598             # This used to be our default with discard not being specified:
1599             $detectzeroes = 'on';
1600         }
1601         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1602     }
1603
1604     my $pathinfo = $path ? "file=$path," : '';
1605
1606     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1607 }
1608
1609 sub print_netdevice_full {
1610     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
1611
1612     my $device = $net->{model};
1613     if ($net->{model} eq 'virtio') {
1614          $device = 'virtio-net-pci';
1615      };
1616
1617     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
1618     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1619     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1620         # Consider we have N queues, the number of vectors needed is 2 * N + 2, i.e., one per in
1621         # and out of each queue plus one config interrupt and control vector queue
1622         my $vectors = $net->{queues} * 2 + 2;
1623         $tmpstr .= ",vectors=$vectors,mq=on";
1624     }
1625     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1626
1627     if (my $mtu = $net->{mtu}) {
1628         if ($net->{model} eq 'virtio' && $net->{bridge}) {
1629             my $bridge_mtu = PVE::Network::read_bridge_mtu($net->{bridge});
1630             if ($mtu == 1) {
1631                  $mtu = $bridge_mtu;
1632             } elsif ($mtu < 576) {
1633                 die "netdev $netid: MTU '$mtu' is smaller than the IP minimum MTU '576'\n";
1634             } elsif ($mtu > $bridge_mtu) {
1635                 die "netdev $netid: MTU '$mtu' is bigger than the bridge MTU '$bridge_mtu'\n";
1636             }
1637             $tmpstr .= ",host_mtu=$mtu";
1638         } else {
1639             warn "WARN: netdev $netid: ignoring MTU '$mtu', not using VirtIO or no bridge configured.\n";
1640         }
1641     }
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     if (is_native($arch)) {
1680         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
1681     }
1682
1683     my $vmname = $conf->{name} || "vm$vmid";
1684
1685     my $netdev = "";
1686     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1687
1688     if ($net->{bridge}) {
1689         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script"
1690             .",downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1691     } else {
1692         $netdev = "type=user,id=$netid,hostname=$vmname";
1693     }
1694
1695     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1696
1697     return $netdev;
1698 }
1699
1700 my $vga_map = {
1701     'cirrus' => 'cirrus-vga',
1702     'std' => 'VGA',
1703     'vmware' => 'vmware-svga',
1704     'virtio' => 'virtio-vga',
1705 };
1706
1707 sub print_vga_device {
1708     my ($conf, $vga, $arch, $machine_version, $machine, $id, $qxlnum, $bridges) = @_;
1709
1710     my $type = $vga_map->{$vga->{type}};
1711     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
1712         $type = 'virtio-gpu';
1713     }
1714     my $vgamem_mb = $vga->{memory};
1715
1716     my $max_outputs = '';
1717     if ($qxlnum) {
1718         $type = $id ? 'qxl' : 'qxl-vga';
1719
1720         if (!$conf->{ostype} || $conf->{ostype} =~ m/^(?:l\d\d)|(?:other)$/) {
1721             # set max outputs so linux can have up to 4 qxl displays with one device
1722             if (min_version($machine_version, 4, 1)) {
1723                 $max_outputs = ",max_outputs=4";
1724             }
1725         }
1726     }
1727
1728     die "no devicetype for $vga->{type}\n" if !$type;
1729
1730     my $memory = "";
1731     if ($vgamem_mb) {
1732         if ($vga->{type} eq 'virtio') {
1733             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
1734             $memory = ",max_hostmem=$bytes";
1735         } elsif ($qxlnum) {
1736             # from https://www.spice-space.org/multiple-monitors.html
1737             $memory = ",vgamem_mb=$vga->{memory}";
1738             my $ram = $vgamem_mb * 4;
1739             my $vram = $vgamem_mb * 2;
1740             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
1741         } else {
1742             $memory = ",vgamem_mb=$vga->{memory}";
1743         }
1744     } elsif ($qxlnum && $id) {
1745         $memory = ",ram_size=67108864,vram_size=33554432";
1746     }
1747
1748     my $edidoff = "";
1749     if ($type eq 'VGA' && windows_version($conf->{ostype})) {
1750         $edidoff=",edid=off" if (!defined($conf->{bios}) || $conf->{bios} ne 'ovmf');
1751     }
1752
1753     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
1754     my $vgaid = "vga" . ($id // '');
1755     my $pciaddr;
1756
1757     if ($q35 && $vgaid eq 'vga') {
1758         # the first display uses pcie.0 bus on q35 machines
1759         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
1760     } else {
1761         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
1762     }
1763
1764     return "$type,id=${vgaid}${memory}${max_outputs}${pciaddr}${edidoff}";
1765 }
1766
1767 sub parse_number_sets {
1768     my ($set) = @_;
1769     my $res = [];
1770     foreach my $part (split(/;/, $set)) {
1771         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1772             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1773             push @$res, [ $1, $2 ];
1774         } else {
1775             die "invalid range: $part\n";
1776         }
1777     }
1778     return $res;
1779 }
1780
1781 sub parse_numa {
1782     my ($data) = @_;
1783
1784     my $res = parse_property_string($numa_fmt, $data);
1785     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1786     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1787     return $res;
1788 }
1789
1790 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1791 sub parse_net {
1792     my ($data) = @_;
1793
1794     my $res = eval { parse_property_string($net_fmt, $data) };
1795     if ($@) {
1796         warn $@;
1797         return;
1798     }
1799     if (!defined($res->{macaddr})) {
1800         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1801         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1802     }
1803     return $res;
1804 }
1805
1806 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1807 sub parse_ipconfig {
1808     my ($data) = @_;
1809
1810     my $res = eval { parse_property_string($ipconfig_fmt, $data) };
1811     if ($@) {
1812         warn $@;
1813         return;
1814     }
1815
1816     if ($res->{gw} && !$res->{ip}) {
1817         warn 'gateway specified without specifying an IP address';
1818         return;
1819     }
1820     if ($res->{gw6} && !$res->{ip6}) {
1821         warn 'IPv6 gateway specified without specifying an IPv6 address';
1822         return;
1823     }
1824     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1825         warn 'gateway specified together with DHCP';
1826         return;
1827     }
1828     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
1829         # gw6 + auto/dhcp
1830         warn "IPv6 gateway specified together with $res->{ip6} address";
1831         return;
1832     }
1833
1834     if (!$res->{ip} && !$res->{ip6}) {
1835         return { ip => 'dhcp', ip6 => 'dhcp' };
1836     }
1837
1838     return $res;
1839 }
1840
1841 sub print_net {
1842     my $net = shift;
1843
1844     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1845 }
1846
1847 sub add_random_macs {
1848     my ($settings) = @_;
1849
1850     foreach my $opt (keys %$settings) {
1851         next if $opt !~ m/^net(\d+)$/;
1852         my $net = parse_net($settings->{$opt});
1853         next if !$net;
1854         $settings->{$opt} = print_net($net);
1855     }
1856 }
1857
1858 sub vm_is_volid_owner {
1859     my ($storecfg, $vmid, $volid) = @_;
1860
1861     if ($volid !~  m|^/|) {
1862         my ($path, $owner);
1863         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1864         if ($owner && ($owner == $vmid)) {
1865             return 1;
1866         }
1867     }
1868
1869     return;
1870 }
1871
1872 sub vmconfig_register_unused_drive {
1873     my ($storecfg, $vmid, $conf, $drive) = @_;
1874
1875     if (drive_is_cloudinit($drive)) {
1876         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
1877         warn $@ if $@;
1878     } elsif (!drive_is_cdrom($drive)) {
1879         my $volid = $drive->{file};
1880         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1881             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1882         }
1883     }
1884 }
1885
1886 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
1887 my $smbios1_fmt = {
1888     uuid => {
1889         type => 'string',
1890         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1891         format_description => 'UUID',
1892         description => "Set SMBIOS1 UUID.",
1893         optional => 1,
1894     },
1895     version => {
1896         type => 'string',
1897         pattern => '[A-Za-z0-9+\/]+={0,2}',
1898         format_description => 'Base64 encoded string',
1899         description => "Set SMBIOS1 version.",
1900         optional => 1,
1901     },
1902     serial => {
1903         type => 'string',
1904         pattern => '[A-Za-z0-9+\/]+={0,2}',
1905         format_description => 'Base64 encoded string',
1906         description => "Set SMBIOS1 serial number.",
1907         optional => 1,
1908     },
1909     manufacturer => {
1910         type => 'string',
1911         pattern => '[A-Za-z0-9+\/]+={0,2}',
1912         format_description => 'Base64 encoded string',
1913         description => "Set SMBIOS1 manufacturer.",
1914         optional => 1,
1915     },
1916     product => {
1917         type => 'string',
1918         pattern => '[A-Za-z0-9+\/]+={0,2}',
1919         format_description => 'Base64 encoded string',
1920         description => "Set SMBIOS1 product ID.",
1921         optional => 1,
1922     },
1923     sku => {
1924         type => 'string',
1925         pattern => '[A-Za-z0-9+\/]+={0,2}',
1926         format_description => 'Base64 encoded string',
1927         description => "Set SMBIOS1 SKU string.",
1928         optional => 1,
1929     },
1930     family => {
1931         type => 'string',
1932         pattern => '[A-Za-z0-9+\/]+={0,2}',
1933         format_description => 'Base64 encoded string',
1934         description => "Set SMBIOS1 family string.",
1935         optional => 1,
1936     },
1937     base64 => {
1938         type => 'boolean',
1939         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
1940         optional => 1,
1941     },
1942 };
1943
1944 sub parse_smbios1 {
1945     my ($data) = @_;
1946
1947     my $res = eval { parse_property_string($smbios1_fmt, $data) };
1948     warn $@ if $@;
1949     return $res;
1950 }
1951
1952 sub print_smbios1 {
1953     my ($smbios1) = @_;
1954     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1955 }
1956
1957 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1958
1959 sub parse_watchdog {
1960     my ($value) = @_;
1961
1962     return if !$value;
1963
1964     my $res = eval { parse_property_string($watchdog_fmt, $value) };
1965     warn $@ if $@;
1966     return $res;
1967 }
1968
1969 sub parse_guest_agent {
1970     my ($conf) = @_;
1971
1972     return {} if !defined($conf->{agent});
1973
1974     my $res = eval { parse_property_string($agent_fmt, $conf->{agent}) };
1975     warn $@ if $@;
1976
1977     # if the agent is disabled ignore the other potentially set properties
1978     return {} if !$res->{enabled};
1979     return $res;
1980 }
1981
1982 sub get_qga_key {
1983     my ($conf, $key) = @_;
1984     return undef if !defined($conf->{agent});
1985
1986     my $agent = parse_guest_agent($conf);
1987     return $agent->{$key};
1988 }
1989
1990 sub parse_vga {
1991     my ($value) = @_;
1992
1993     return {} if !$value;
1994     my $res = eval { parse_property_string($vga_fmt, $value) };
1995     warn $@ if $@;
1996     return $res;
1997 }
1998
1999 sub parse_rng {
2000     my ($value) = @_;
2001
2002     return if !$value;
2003
2004     my $res = eval { parse_property_string($rng_fmt, $value) };
2005     warn $@ if $@;
2006     return $res;
2007 }
2008
2009 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2010 sub verify_usb_device {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if parse_usb_device($value);
2014
2015     return if $noerr;
2016
2017     die "unable to parse usb device\n";
2018 }
2019
2020 # add JSON properties for create and set function
2021 sub json_config_properties {
2022     my $prop = shift;
2023
2024     foreach my $opt (keys %$confdesc) {
2025         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' ||
2026             $opt eq 'runningmachine' || $opt eq 'runningcpu';
2027         $prop->{$opt} = $confdesc->{$opt};
2028     }
2029
2030     return $prop;
2031 }
2032
2033 # return copy of $confdesc_cloudinit to generate documentation
2034 sub cloudinit_config_properties {
2035
2036     return dclone($confdesc_cloudinit);
2037 }
2038
2039 sub check_type {
2040     my ($key, $value) = @_;
2041
2042     die "unknown setting '$key'\n" if !$confdesc->{$key};
2043
2044     my $type = $confdesc->{$key}->{type};
2045
2046     if (!defined($value)) {
2047         die "got undefined value\n";
2048     }
2049
2050     if ($value =~ m/[\n\r]/) {
2051         die "property contains a line feed\n";
2052     }
2053
2054     if ($type eq 'boolean') {
2055         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2056         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2057         die "type check ('boolean') failed - got '$value'\n";
2058     } elsif ($type eq 'integer') {
2059         return int($1) if $value =~ m/^(\d+)$/;
2060         die "type check ('integer') failed - got '$value'\n";
2061     } elsif ($type eq 'number') {
2062         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2063         die "type check ('number') failed - got '$value'\n";
2064     } elsif ($type eq 'string') {
2065         if (my $fmt = $confdesc->{$key}->{format}) {
2066             PVE::JSONSchema::check_format($fmt, $value);
2067             return $value;
2068         }
2069         $value =~ s/^\"(.*)\"$/$1/;
2070         return $value;
2071     } else {
2072         die "internal error"
2073     }
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $skiplock, $replacement_conf, $purge_unreferenced) = @_;
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     if ($conf->{template}) {
2084         # check if any base image is still used by a linked clone
2085         PVE::QemuConfig->foreach_volume($conf, sub {
2086                 my ($ds, $drive) = @_;
2087                 return if drive_is_cdrom($drive);
2088
2089                 my $volid = $drive->{file};
2090                 return if !$volid || $volid =~ m|^/|;
2091
2092                 die "base volume '$volid' is still in use by linked cloned\n"
2093                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2094
2095         });
2096     }
2097
2098     # only remove disks owned by this VM (referenced in the config)
2099     PVE::QemuConfig->foreach_volume_full($conf, { include_unused => 1 }, sub {
2100         my ($ds, $drive) = @_;
2101         return if drive_is_cdrom($drive, 1);
2102
2103         my $volid = $drive->{file};
2104         return if !$volid || $volid =~ m|^/|;
2105
2106         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2107         return if !$path || !$owner || ($owner != $vmid);
2108
2109         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2110         warn "Could not remove disk '$volid', check manually: $@" if $@;
2111     });
2112
2113     if ($purge_unreferenced) { # also remove unreferenced disk
2114         my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2115         PVE::Storage::foreach_volid($vmdisks, sub {
2116             my ($volid, $sid, $volname, $d) = @_;
2117             eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2118             warn $@ if $@;
2119         });
2120     }
2121
2122     if (defined $replacement_conf) {
2123         PVE::QemuConfig->write_config($vmid, $replacement_conf);
2124     } else {
2125         PVE::QemuConfig->destroy_config($vmid);
2126     }
2127 }
2128
2129 sub parse_vm_config {
2130     my ($filename, $raw) = @_;
2131
2132     return if !defined($raw);
2133
2134     my $res = {
2135         digest => Digest::SHA::sha1_hex($raw),
2136         snapshots => {},
2137         pending => {},
2138     };
2139
2140     $filename =~ m|/qemu-server/(\d+)\.conf$|
2141         || die "got strange filename '$filename'";
2142
2143     my $vmid = $1;
2144
2145     my $conf = $res;
2146     my $descr;
2147     my $section = '';
2148
2149     my @lines = split(/\n/, $raw);
2150     foreach my $line (@lines) {
2151         next if $line =~ m/^\s*$/;
2152
2153         if ($line =~ m/^\[PENDING\]\s*$/i) {
2154             $section = 'pending';
2155             if (defined($descr)) {
2156                 $descr =~ s/\s+$//;
2157                 $conf->{description} = $descr;
2158             }
2159             $descr = undef;
2160             $conf = $res->{$section} = {};
2161             next;
2162
2163         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2164             $section = $1;
2165             if (defined($descr)) {
2166                 $descr =~ s/\s+$//;
2167                 $conf->{description} = $descr;
2168             }
2169             $descr = undef;
2170             $conf = $res->{snapshots}->{$section} = {};
2171             next;
2172         }
2173
2174         if ($line =~ m/^\#(.*)\s*$/) {
2175             $descr = '' if !defined($descr);
2176             $descr .= PVE::Tools::decode_text($1) . "\n";
2177             next;
2178         }
2179
2180         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2181             $descr = '' if !defined($descr);
2182             $descr .= PVE::Tools::decode_text($2);
2183         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2184             $conf->{snapstate} = $1;
2185         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2186             my $key = $1;
2187             my $value = $2;
2188             $conf->{$key} = $value;
2189         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2190             my $value = $1;
2191             if ($section eq 'pending') {
2192                 $conf->{delete} = $value; # we parse this later
2193             } else {
2194                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2195             }
2196         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2197             my $key = $1;
2198             my $value = $2;
2199             eval { $value = check_type($key, $value); };
2200             if ($@) {
2201                 warn "vm $vmid - unable to parse value of '$key' - $@";
2202             } else {
2203                 $key = 'ide2' if $key eq 'cdrom';
2204                 my $fmt = $confdesc->{$key}->{format};
2205                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2206                     my $v = parse_drive($key, $value);
2207                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2208                         $v->{file} = $volid;
2209                         $value = print_drive($v);
2210                     } else {
2211                         warn "vm $vmid - unable to parse value of '$key'\n";
2212                         next;
2213                     }
2214                 }
2215
2216                 $conf->{$key} = $value;
2217             }
2218         } else {
2219             warn "vm $vmid - unable to parse config: $line\n";
2220         }
2221     }
2222
2223     if (defined($descr)) {
2224         $descr =~ s/\s+$//;
2225         $conf->{description} = $descr;
2226     }
2227     delete $res->{snapstate}; # just to be sure
2228
2229     return $res;
2230 }
2231
2232 sub write_vm_config {
2233     my ($filename, $conf) = @_;
2234
2235     delete $conf->{snapstate}; # just to be sure
2236
2237     if ($conf->{cdrom}) {
2238         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2239         $conf->{ide2} = $conf->{cdrom};
2240         delete $conf->{cdrom};
2241     }
2242
2243     # we do not use 'smp' any longer
2244     if ($conf->{sockets}) {
2245         delete $conf->{smp};
2246     } elsif ($conf->{smp}) {
2247         $conf->{sockets} = $conf->{smp};
2248         delete $conf->{cores};
2249         delete $conf->{smp};
2250     }
2251
2252     my $used_volids = {};
2253
2254     my $cleanup_config = sub {
2255         my ($cref, $pending, $snapname) = @_;
2256
2257         foreach my $key (keys %$cref) {
2258             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2259                 $key eq 'snapstate' || $key eq 'pending';
2260             my $value = $cref->{$key};
2261             if ($key eq 'delete') {
2262                 die "propertry 'delete' is only allowed in [PENDING]\n"
2263                     if !$pending;
2264                 # fixme: check syntax?
2265                 next;
2266             }
2267             eval { $value = check_type($key, $value); };
2268             die "unable to parse value of '$key' - $@" if $@;
2269
2270             $cref->{$key} = $value;
2271
2272             if (!$snapname && is_valid_drivename($key)) {
2273                 my $drive = parse_drive($key, $value);
2274                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2275             }
2276         }
2277     };
2278
2279     &$cleanup_config($conf);
2280
2281     &$cleanup_config($conf->{pending}, 1);
2282
2283     foreach my $snapname (keys %{$conf->{snapshots}}) {
2284         die "internal error: snapshot name '$snapname' is forbidden" if lc($snapname) eq 'pending';
2285         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2286     }
2287
2288     # remove 'unusedX' settings if we re-add a volume
2289     foreach my $key (keys %$conf) {
2290         my $value = $conf->{$key};
2291         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2292             delete $conf->{$key};
2293         }
2294     }
2295
2296     my $generate_raw_config = sub {
2297         my ($conf, $pending) = @_;
2298
2299         my $raw = '';
2300
2301         # add description as comment to top of file
2302         if (defined(my $descr = $conf->{description})) {
2303             if ($descr) {
2304                 foreach my $cl (split(/\n/, $descr)) {
2305                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2306                 }
2307             } else {
2308                 $raw .= "#\n" if $pending;
2309             }
2310         }
2311
2312         foreach my $key (sort keys %$conf) {
2313             next if $key =~ /^(digest|description|pending|snapshots)$/;
2314             $raw .= "$key: $conf->{$key}\n";
2315         }
2316         return $raw;
2317     };
2318
2319     my $raw = &$generate_raw_config($conf);
2320
2321     if (scalar(keys %{$conf->{pending}})){
2322         $raw .= "\n[PENDING]\n";
2323         $raw .= &$generate_raw_config($conf->{pending}, 1);
2324     }
2325
2326     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2327         $raw .= "\n[$snapname]\n";
2328         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2329     }
2330
2331     return $raw;
2332 }
2333
2334 sub load_defaults {
2335
2336     my $res = {};
2337
2338     # we use static defaults from our JSON schema configuration
2339     foreach my $key (keys %$confdesc) {
2340         if (defined(my $default = $confdesc->{$key}->{default})) {
2341             $res->{$key} = $default;
2342         }
2343     }
2344
2345     return $res;
2346 }
2347
2348 sub config_list {
2349     my $vmlist = PVE::Cluster::get_vmlist();
2350     my $res = {};
2351     return $res if !$vmlist || !$vmlist->{ids};
2352     my $ids = $vmlist->{ids};
2353     my $nodename = nodename();
2354
2355     foreach my $vmid (keys %$ids) {
2356         my $d = $ids->{$vmid};
2357         next if !$d->{node} || $d->{node} ne $nodename;
2358         next if !$d->{type} || $d->{type} ne 'qemu';
2359         $res->{$vmid}->{exists} = 1;
2360     }
2361     return $res;
2362 }
2363
2364 # test if VM uses local resources (to prevent migration)
2365 sub check_local_resources {
2366     my ($conf, $noerr) = @_;
2367
2368     my @loc_res = ();
2369
2370     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2371     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2372
2373     push @loc_res, "ivshmem" if $conf->{ivshmem};
2374
2375     foreach my $k (keys %$conf) {
2376         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2377         # sockets are safe: they will recreated be on the target side post-migrate
2378         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2379         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2380     }
2381
2382     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2383
2384     return \@loc_res;
2385 }
2386
2387 # check if used storages are available on all nodes (use by migrate)
2388 sub check_storage_availability {
2389     my ($storecfg, $conf, $node) = @_;
2390
2391     PVE::QemuConfig->foreach_volume($conf, sub {
2392         my ($ds, $drive) = @_;
2393
2394         my $volid = $drive->{file};
2395         return if !$volid;
2396
2397         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2398         return if !$sid;
2399
2400         # check if storage is available on both nodes
2401         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2402         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2403    });
2404 }
2405
2406 # list nodes where all VM images are available (used by has_feature API)
2407 sub shared_nodes {
2408     my ($conf, $storecfg) = @_;
2409
2410     my $nodelist = PVE::Cluster::get_nodelist();
2411     my $nodehash = { map { $_ => 1 } @$nodelist };
2412     my $nodename = nodename();
2413
2414     PVE::QemuConfig->foreach_volume($conf, sub {
2415         my ($ds, $drive) = @_;
2416
2417         my $volid = $drive->{file};
2418         return if !$volid;
2419
2420         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2421         if ($storeid) {
2422             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2423             if ($scfg->{disable}) {
2424                 $nodehash = {};
2425             } elsif (my $avail = $scfg->{nodes}) {
2426                 foreach my $node (keys %$nodehash) {
2427                     delete $nodehash->{$node} if !$avail->{$node};
2428                 }
2429             } elsif (!$scfg->{shared}) {
2430                 foreach my $node (keys %$nodehash) {
2431                     delete $nodehash->{$node} if $node ne $nodename
2432                 }
2433             }
2434         }
2435     });
2436
2437     return $nodehash
2438 }
2439
2440 sub check_local_storage_availability {
2441     my ($conf, $storecfg) = @_;
2442
2443     my $nodelist = PVE::Cluster::get_nodelist();
2444     my $nodehash = { map { $_ => {} } @$nodelist };
2445
2446     PVE::QemuConfig->foreach_volume($conf, sub {
2447         my ($ds, $drive) = @_;
2448
2449         my $volid = $drive->{file};
2450         return if !$volid;
2451
2452         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2453         if ($storeid) {
2454             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2455
2456             if ($scfg->{disable}) {
2457                 foreach my $node (keys %$nodehash) {
2458                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2459                 }
2460             } elsif (my $avail = $scfg->{nodes}) {
2461                 foreach my $node (keys %$nodehash) {
2462                     if (!$avail->{$node}) {
2463                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2464                     }
2465                 }
2466             }
2467         }
2468     });
2469
2470     foreach my $node (values %$nodehash) {
2471         if (my $unavail = $node->{unavailable_storages}) {
2472             $node->{unavailable_storages} = [ sort keys %$unavail ];
2473         }
2474     }
2475
2476     return $nodehash
2477 }
2478
2479 # Compat only, use assert_config_exists_on_node and vm_running_locally where possible
2480 sub check_running {
2481     my ($vmid, $nocheck, $node) = @_;
2482
2483     PVE::QemuConfig::assert_config_exists_on_node($vmid, $node) if !$nocheck;
2484     return PVE::QemuServer::Helpers::vm_running_locally($vmid);
2485 }
2486
2487 sub vzlist {
2488
2489     my $vzlist = config_list();
2490
2491     my $fd = IO::Dir->new($PVE::QemuServer::Helpers::var_run_tmpdir) || return $vzlist;
2492
2493     while (defined(my $de = $fd->read)) {
2494         next if $de !~ m/^(\d+)\.pid$/;
2495         my $vmid = $1;
2496         next if !defined($vzlist->{$vmid});
2497         if (my $pid = check_running($vmid)) {
2498             $vzlist->{$vmid}->{pid} = $pid;
2499         }
2500     }
2501
2502     return $vzlist;
2503 }
2504
2505 our $vmstatus_return_properties = {
2506     vmid => get_standard_option('pve-vmid'),
2507     status => {
2508         description => "Qemu process status.",
2509         type => 'string',
2510         enum => ['stopped', 'running'],
2511     },
2512     maxmem => {
2513         description => "Maximum memory in bytes.",
2514         type => 'integer',
2515         optional => 1,
2516         renderer => 'bytes',
2517     },
2518     maxdisk => {
2519         description => "Root disk size in bytes.",
2520         type => 'integer',
2521         optional => 1,
2522         renderer => 'bytes',
2523     },
2524     name => {
2525         description => "VM name.",
2526         type => 'string',
2527         optional => 1,
2528     },
2529     qmpstatus => {
2530         description => "Qemu QMP agent status.",
2531         type => 'string',
2532         optional => 1,
2533     },
2534     pid => {
2535         description => "PID of running qemu process.",
2536         type => 'integer',
2537         optional => 1,
2538     },
2539     uptime => {
2540         description => "Uptime.",
2541         type => 'integer',
2542         optional => 1,
2543         renderer => 'duration',
2544     },
2545     cpus => {
2546         description => "Maximum usable CPUs.",
2547         type => 'number',
2548         optional => 1,
2549     },
2550     lock => {
2551         description => "The current config lock, if any.",
2552         type => 'string',
2553         optional => 1,
2554     },
2555     tags => {
2556         description  => "The current configured tags, if any",
2557         type => 'string',
2558         optional => 1,
2559     },
2560     'running-machine' => {
2561         description => "The currently running machine type (if running).",
2562         type => 'string',
2563         optional => 1,
2564     },
2565     'running-qemu' => {
2566         description => "The currently running QEMU version (if running).",
2567         type => 'string',
2568         optional => 1,
2569     },
2570 };
2571
2572 my $last_proc_pid_stat;
2573
2574 # get VM status information
2575 # This must be fast and should not block ($full == false)
2576 # We only query KVM using QMP if $full == true (this can be slow)
2577 sub vmstatus {
2578     my ($opt_vmid, $full) = @_;
2579
2580     my $res = {};
2581
2582     my $storecfg = PVE::Storage::config();
2583
2584     my $list = vzlist();
2585     my $defaults = load_defaults();
2586
2587     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2588
2589     my $cpucount = $cpuinfo->{cpus} || 1;
2590
2591     foreach my $vmid (keys %$list) {
2592         next if $opt_vmid && ($vmid ne $opt_vmid);
2593
2594         my $conf = PVE::QemuConfig->load_config($vmid);
2595
2596         my $d = { vmid => $vmid };
2597         $d->{pid} = $list->{$vmid}->{pid};
2598
2599         # fixme: better status?
2600         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2601
2602         my $size = PVE::QemuServer::Drive::bootdisk_size($storecfg, $conf);
2603         if (defined($size)) {
2604             $d->{disk} = 0; # no info available
2605             $d->{maxdisk} = $size;
2606         } else {
2607             $d->{disk} = 0;
2608             $d->{maxdisk} = 0;
2609         }
2610
2611         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2612             * ($conf->{cores} || $defaults->{cores});
2613         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2614         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2615
2616         $d->{name} = $conf->{name} || "VM $vmid";
2617         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2618             : $defaults->{memory}*(1024*1024);
2619
2620         if ($conf->{balloon}) {
2621             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2622             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2623                 : $defaults->{shares};
2624         }
2625
2626         $d->{uptime} = 0;
2627         $d->{cpu} = 0;
2628         $d->{mem} = 0;
2629
2630         $d->{netout} = 0;
2631         $d->{netin} = 0;
2632
2633         $d->{diskread} = 0;
2634         $d->{diskwrite} = 0;
2635
2636         $d->{template} = PVE::QemuConfig->is_template($conf);
2637
2638         $d->{serial} = 1 if conf_has_serial($conf);
2639         $d->{lock} = $conf->{lock} if $conf->{lock};
2640         $d->{tags} = $conf->{tags} if defined($conf->{tags});
2641
2642         $res->{$vmid} = $d;
2643     }
2644
2645     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2646     foreach my $dev (keys %$netdev) {
2647         next if $dev !~ m/^tap([1-9]\d*)i/;
2648         my $vmid = $1;
2649         my $d = $res->{$vmid};
2650         next if !$d;
2651
2652         $d->{netout} += $netdev->{$dev}->{receive};
2653         $d->{netin} += $netdev->{$dev}->{transmit};
2654
2655         if ($full) {
2656             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2657             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2658         }
2659
2660     }
2661
2662     my $ctime = gettimeofday;
2663
2664     foreach my $vmid (keys %$list) {
2665
2666         my $d = $res->{$vmid};
2667         my $pid = $d->{pid};
2668         next if !$pid;
2669
2670         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2671         next if !$pstat; # not running
2672
2673         my $used = $pstat->{utime} + $pstat->{stime};
2674
2675         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2676
2677         if ($pstat->{vsize}) {
2678             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2679         }
2680
2681         my $old = $last_proc_pid_stat->{$pid};
2682         if (!$old) {
2683             $last_proc_pid_stat->{$pid} = {
2684                 time => $ctime,
2685                 used => $used,
2686                 cpu => 0,
2687             };
2688             next;
2689         }
2690
2691         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2692
2693         if ($dtime > 1000) {
2694             my $dutime = $used -  $old->{used};
2695
2696             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2697             $last_proc_pid_stat->{$pid} = {
2698                 time => $ctime,
2699                 used => $used,
2700                 cpu => $d->{cpu},
2701             };
2702         } else {
2703             $d->{cpu} = $old->{cpu};
2704         }
2705     }
2706
2707     return $res if !$full;
2708
2709     my $qmpclient = PVE::QMPClient->new();
2710
2711     my $ballooncb = sub {
2712         my ($vmid, $resp) = @_;
2713
2714         my $info = $resp->{'return'};
2715         return if !$info->{max_mem};
2716
2717         my $d = $res->{$vmid};
2718
2719         # use memory assigned to VM
2720         $d->{maxmem} = $info->{max_mem};
2721         $d->{balloon} = $info->{actual};
2722
2723         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2724             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2725             $d->{freemem} = $info->{free_mem};
2726         }
2727
2728         $d->{ballooninfo} = $info;
2729     };
2730
2731     my $blockstatscb = sub {
2732         my ($vmid, $resp) = @_;
2733         my $data = $resp->{'return'} || [];
2734         my $totalrdbytes = 0;
2735         my $totalwrbytes = 0;
2736
2737         for my $blockstat (@$data) {
2738             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2739             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2740
2741             $blockstat->{device} =~ s/drive-//;
2742             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2743         }
2744         $res->{$vmid}->{diskread} = $totalrdbytes;
2745         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2746     };
2747
2748     my $machinecb = sub {
2749         my ($vmid, $resp) = @_;
2750         my $data = $resp->{'return'} || [];
2751
2752         $res->{$vmid}->{'running-machine'} =
2753             PVE::QemuServer::Machine::current_from_query_machines($data);
2754     };
2755
2756     my $versioncb = sub {
2757         my ($vmid, $resp) = @_;
2758         my $data = $resp->{'return'} // {};
2759         my $version = 'unknown';
2760
2761         if (my $v = $data->{qemu}) {
2762             $version = $v->{major} . "." . $v->{minor} . "." . $v->{micro};
2763         }
2764
2765         $res->{$vmid}->{'running-qemu'} = $version;
2766     };
2767
2768     my $statuscb = sub {
2769         my ($vmid, $resp) = @_;
2770
2771         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2772         $qmpclient->queue_cmd($vmid, $machinecb, 'query-machines');
2773         $qmpclient->queue_cmd($vmid, $versioncb, 'query-version');
2774         # this fails if ballon driver is not loaded, so this must be
2775         # the last commnand (following command are aborted if this fails).
2776         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2777
2778         my $status = 'unknown';
2779         if (!defined($status = $resp->{'return'}->{status})) {
2780             warn "unable to get VM status\n";
2781             return;
2782         }
2783
2784         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2785     };
2786
2787     foreach my $vmid (keys %$list) {
2788         next if $opt_vmid && ($vmid ne $opt_vmid);
2789         next if !$res->{$vmid}->{pid}; # not running
2790         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2791     }
2792
2793     $qmpclient->queue_execute(undef, 2);
2794
2795     foreach my $vmid (keys %$list) {
2796         next if $opt_vmid && ($vmid ne $opt_vmid);
2797         next if !$res->{$vmid}->{pid}; #not running
2798
2799         # we can't use the $qmpclient since it might have already aborted on
2800         # 'query-balloon', but this might also fail for older versions...
2801         my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
2802         $res->{$vmid}->{'proxmox-support'} = $qemu_support // {};
2803     }
2804
2805     foreach my $vmid (keys %$list) {
2806         next if $opt_vmid && ($vmid ne $opt_vmid);
2807         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2808     }
2809
2810     return $res;
2811 }
2812
2813 sub conf_has_serial {
2814     my ($conf) = @_;
2815
2816     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2817         if ($conf->{"serial$i"}) {
2818             return 1;
2819         }
2820     }
2821
2822     return 0;
2823 }
2824
2825 sub conf_has_audio {
2826     my ($conf, $id) = @_;
2827
2828     $id //= 0;
2829     my $audio = $conf->{"audio$id"};
2830     return if !defined($audio);
2831
2832     my $audioproperties = parse_property_string($audio_fmt, $audio);
2833     my $audiodriver = $audioproperties->{driver} // 'spice';
2834
2835     return {
2836         dev => $audioproperties->{device},
2837         dev_id => "audiodev$id",
2838         backend => $audiodriver,
2839         backend_id => "$audiodriver-backend${id}",
2840     };
2841 }
2842
2843 sub audio_devs {
2844     my ($audio, $audiopciaddr, $machine_version) = @_;
2845
2846     my $devs = [];
2847
2848     my $id = $audio->{dev_id};
2849     my $audiodev = "";
2850     if (min_version($machine_version, 4, 2)) {
2851         $audiodev = ",audiodev=$audio->{backend_id}";
2852     }
2853
2854     if ($audio->{dev} eq 'AC97') {
2855         push @$devs, '-device', "AC97,id=${id}${audiopciaddr}$audiodev";
2856     } elsif ($audio->{dev} =~ /intel\-hda$/) {
2857         push @$devs, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
2858         push @$devs, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0$audiodev";
2859         push @$devs, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1$audiodev";
2860     } else {
2861         die "unkown audio device '$audio->{dev}', implement me!";
2862     }
2863
2864     push @$devs, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
2865
2866     return $devs;
2867 }
2868
2869 sub vga_conf_has_spice {
2870     my ($vga) = @_;
2871
2872     my $vgaconf = parse_vga($vga);
2873     my $vgatype = $vgaconf->{type};
2874     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
2875
2876     return $1 || 1;
2877 }
2878
2879 sub is_native($) {
2880     my ($arch) = @_;
2881     return get_host_arch() eq $arch;
2882 }
2883
2884 sub get_vm_arch {
2885     my ($conf) = @_;
2886     return $conf->{arch} // get_host_arch();
2887 }
2888
2889 my $default_machines = {
2890     x86_64 => 'pc',
2891     aarch64 => 'virt',
2892 };
2893
2894 sub get_vm_machine {
2895     my ($conf, $forcemachine, $arch, $add_pve_version, $kvmversion) = @_;
2896
2897     my $machine = $forcemachine || $conf->{machine};
2898
2899     if (!$machine || $machine =~ m/^(?:pc|q35|virt)$/) {
2900         $arch //= 'x86_64';
2901         $machine ||= $default_machines->{$arch};
2902         if ($add_pve_version) {
2903             $kvmversion //= kvm_user_version();
2904             my $pvever = PVE::QemuServer::Machine::get_pve_version($kvmversion);
2905             $machine .= "+pve$pvever";
2906         }
2907     }
2908
2909     if ($add_pve_version && $machine !~ m/\+pve\d+$/) {
2910         # for version-pinned machines that do not include a pve-version (e.g.
2911         # pc-q35-4.1), we assume 0 to keep them stable in case we bump
2912         $machine .= '+pve0';
2913     }
2914
2915     return $machine;
2916 }
2917
2918 sub get_ovmf_files($) {
2919     my ($arch) = @_;
2920
2921     my $ovmf = $OVMF->{$arch}
2922         or die "no OVMF images known for architecture '$arch'\n";
2923
2924     return @$ovmf;
2925 }
2926
2927 my $Arch2Qemu = {
2928     aarch64 => '/usr/bin/qemu-system-aarch64',
2929     x86_64 => '/usr/bin/qemu-system-x86_64',
2930 };
2931 sub get_command_for_arch($) {
2932     my ($arch) = @_;
2933     return '/usr/bin/kvm' if is_native($arch);
2934
2935     my $cmd = $Arch2Qemu->{$arch}
2936         or die "don't know how to emulate architecture '$arch'\n";
2937     return $cmd;
2938 }
2939
2940 # To use query_supported_cpu_flags and query_understood_cpu_flags to get flags
2941 # to use in a QEMU command line (-cpu element), first array_intersect the result
2942 # of query_supported_ with query_understood_. This is necessary because:
2943 #
2944 # a) query_understood_ returns flags the host cannot use and
2945 # b) query_supported_ (rather the QMP call) doesn't actually return CPU
2946 #    flags, but CPU settings - with most of them being flags. Those settings
2947 #    (and some flags, curiously) cannot be specified as a "-cpu" argument.
2948 #
2949 # query_supported_ needs to start up to 2 temporary VMs and is therefore rather
2950 # expensive. If you need the value returned from this, you can get it much
2951 # cheaper from pmxcfs using PVE::Cluster::get_node_kv('cpuflags-$accel') with
2952 # $accel being 'kvm' or 'tcg'.
2953 #
2954 # pvestatd calls this function on startup and whenever the QEMU/KVM version
2955 # changes, automatically populating pmxcfs.
2956 #
2957 # Returns: { kvm => [ flagX, flagY, ... ], tcg => [ flag1, flag2, ... ] }
2958 # since kvm and tcg machines support different flags
2959 #
2960 sub query_supported_cpu_flags {
2961     my ($arch) = @_;
2962
2963     $arch //= get_host_arch();
2964     my $default_machine = $default_machines->{$arch};
2965
2966     my $flags = {};
2967
2968     # FIXME: Once this is merged, the code below should work for ARM as well:
2969     # https://lists.nongnu.org/archive/html/qemu-devel/2019-06/msg04947.html
2970     die "QEMU/KVM cannot detect CPU flags on ARM (aarch64)\n" if
2971         $arch eq "aarch64";
2972
2973     my $kvm_supported = defined(kvm_version());
2974     my $qemu_cmd = get_command_for_arch($arch);
2975     my $fakevmid = -1;
2976     my $pidfile = PVE::QemuServer::Helpers::pidfile_name($fakevmid);
2977
2978     # Start a temporary (frozen) VM with vmid -1 to allow sending a QMP command
2979     my $query_supported_run_qemu = sub {
2980         my ($kvm) = @_;
2981
2982         my $flags = {};
2983         my $cmd = [
2984             $qemu_cmd,
2985             '-machine', $default_machine,
2986             '-display', 'none',
2987             '-chardev', "socket,id=qmp,path=/var/run/qemu-server/$fakevmid.qmp,server,nowait",
2988             '-mon', 'chardev=qmp,mode=control',
2989             '-pidfile', $pidfile,
2990             '-S', '-daemonize'
2991         ];
2992
2993         if (!$kvm) {
2994             push @$cmd, '-accel', 'tcg';
2995         }
2996
2997         my $rc = run_command($cmd, noerr => 1, quiet => 0);
2998         die "QEMU flag querying VM exited with code " . $rc if $rc;
2999
3000         eval {
3001             my $cmd_result = mon_cmd(
3002                 $fakevmid,
3003                 'query-cpu-model-expansion',
3004                 type => 'full',
3005                 model => { name => 'host' }
3006             );
3007
3008             my $props = $cmd_result->{model}->{props};
3009             foreach my $prop (keys %$props) {
3010                 next if $props->{$prop} ne '1';
3011                 # QEMU returns some flags multiple times, with '_', '.' or '-'
3012                 # (e.g. lahf_lm and lahf-lm; sse4.2, sse4-2 and sse4_2; ...).
3013                 # We only keep those with underscores, to match /proc/cpuinfo
3014                 $prop =~ s/\.|-/_/g;
3015                 $flags->{$prop} = 1;
3016             }
3017         };
3018         my $err = $@;
3019
3020         # force stop with 10 sec timeout and 'nocheck'
3021         # always stop, even if QMP failed
3022         vm_stop(undef, $fakevmid, 1, 1, 10, 0, 1);
3023
3024         die $err if $err;
3025
3026         return [ sort keys %$flags ];
3027     };
3028
3029     # We need to query QEMU twice, since KVM and TCG have different supported flags
3030     PVE::QemuConfig->lock_config($fakevmid, sub {
3031         $flags->{tcg} = eval { $query_supported_run_qemu->(0) };
3032         warn "warning: failed querying supported tcg flags: $@\n" if $@;
3033
3034         if ($kvm_supported) {
3035             $flags->{kvm} = eval { $query_supported_run_qemu->(1) };
3036             warn "warning: failed querying supported kvm flags: $@\n" if $@;
3037         }
3038     });
3039
3040     return $flags;
3041 }
3042
3043 # Understood CPU flags are written to a file at 'pve-qemu' compile time
3044 my $understood_cpu_flag_dir = "/usr/share/kvm";
3045 sub query_understood_cpu_flags {
3046     my $arch = get_host_arch();
3047     my $filepath = "$understood_cpu_flag_dir/recognized-CPUID-flags-$arch";
3048
3049     die "Cannot query understood QEMU CPU flags for architecture: $arch (file not found)\n"
3050         if ! -e $filepath;
3051
3052     my $raw = file_get_contents($filepath);
3053     $raw =~ s/^\s+|\s+$//g;
3054     my @flags = split(/\s+/, $raw);
3055
3056     return \@flags;
3057 }
3058
3059 sub config_to_command {
3060     my ($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu) = @_;
3061
3062     my $cmd = [];
3063     my $globalFlags = [];
3064     my $machineFlags = [];
3065     my $rtcFlags = [];
3066     my $devices = [];
3067     my $pciaddr = '';
3068     my $bridges = {};
3069     my $ostype = $conf->{ostype};
3070     my $winversion = windows_version($ostype);
3071     my $kvm = $conf->{kvm};
3072     my $nodename = nodename();
3073
3074     my $arch = get_vm_arch($conf);
3075     my $kvm_binary = get_command_for_arch($arch);
3076     my $kvmver = kvm_user_version($kvm_binary);
3077
3078     if (!$kvmver || $kvmver !~ m/^(\d+)\.(\d+)/ || $1 < 3) {
3079         $kvmver //= "undefined";
3080         die "Detected old QEMU binary ('$kvmver', at least 3.0 is required)\n";
3081     }
3082
3083     my $add_pve_version = min_version($kvmver, 4, 1);
3084
3085     my $machine_type = get_vm_machine($conf, $forcemachine, $arch, $add_pve_version);
3086     my $machine_version = extract_version($machine_type, $kvmver);
3087     $kvm //= 1 if is_native($arch);
3088
3089     $machine_version =~ m/(\d+)\.(\d+)/;
3090     my ($machine_major, $machine_minor) = ($1, $2);
3091
3092     if ($kvmver =~ m/^\d+\.\d+\.(\d+)/ && $1 >= 90) {
3093         warn "warning: Installed QEMU version ($kvmver) is a release candidate, ignoring version checks\n";
3094     } elsif (!min_version($kvmver, $machine_major, $machine_minor)) {
3095         die "Installed QEMU version '$kvmver' is too old to run machine type '$machine_type',"
3096             ." please upgrade node '$nodename'\n"
3097     } elsif (!PVE::QemuServer::Machine::can_run_pve_machine_version($machine_version, $kvmver)) {
3098         my $max_pve_version = PVE::QemuServer::Machine::get_pve_version($machine_version);
3099         die "Installed qemu-server (max feature level for $machine_major.$machine_minor is"
3100             ." pve$max_pve_version) is too old to run machine type '$machine_type', please upgrade"
3101             ." node '$nodename'\n";
3102     }
3103
3104     # if a specific +pve version is required for a feature, use $version_guard
3105     # instead of min_version to allow machines to be run with the minimum
3106     # required version
3107     my $required_pve_version = 0;
3108     my $version_guard = sub {
3109         my ($major, $minor, $pve) = @_;
3110         return 0 if !min_version($machine_version, $major, $minor, $pve);
3111         my $max_pve = PVE::QemuServer::Machine::get_pve_version("$major.$minor");
3112         return 1 if min_version($machine_version, $major, $minor, $max_pve+1);
3113         $required_pve_version = $pve if $pve && $pve > $required_pve_version;
3114         return 1;
3115     };
3116
3117     if ($kvm && !defined kvm_version()) {
3118         die "KVM virtualisation configured, but not available. Either disable in VM configuration"
3119             ." or enable in BIOS.\n";
3120     }
3121
3122     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3123     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3124     my $use_old_bios_files = undef;
3125     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3126
3127     my $cpuunits = defined($conf->{cpuunits}) ?
3128             $conf->{cpuunits} : $defaults->{cpuunits};
3129
3130     push @$cmd, $kvm_binary;
3131
3132     push @$cmd, '-id', $vmid;
3133
3134     my $vmname = $conf->{name} || "vm$vmid";
3135
3136     push @$cmd, '-name', $vmname;
3137
3138     push @$cmd, '-no-shutdown';
3139
3140     my $use_virtio = 0;
3141
3142     my $qmpsocket = PVE::QemuServer::Helpers::qmp_socket($vmid);
3143     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3144     push @$cmd, '-mon', "chardev=qmp,mode=control";
3145
3146     if (min_version($machine_version, 2, 12)) {
3147         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3148         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3149     }
3150
3151     push @$cmd, '-pidfile' , PVE::QemuServer::Helpers::pidfile_name($vmid);
3152
3153     push @$cmd, '-daemonize';
3154
3155     if ($conf->{smbios1}) {
3156         my $smbios_conf = parse_smbios1($conf->{smbios1});
3157         if ($smbios_conf->{base64}) {
3158             # Do not pass base64 flag to qemu
3159             delete $smbios_conf->{base64};
3160             my $smbios_string = "";
3161             foreach my $key (keys %$smbios_conf) {
3162                 my $value;
3163                 if ($key eq "uuid") {
3164                     $value = $smbios_conf->{uuid}
3165                 } else {
3166                     $value = decode_base64($smbios_conf->{$key});
3167                 }
3168                 # qemu accepts any binary data, only commas need escaping by double comma
3169                 $value =~ s/,/,,/g;
3170                 $smbios_string .= "," . $key . "=" . $value if $value;
3171             }
3172             push @$cmd, '-smbios', "type=1" . $smbios_string;
3173         } else {
3174             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3175         }
3176     }
3177
3178     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3179         my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3180         die "uefi base image '$ovmf_code' not found\n" if ! -f $ovmf_code;
3181
3182         my ($path, $format);
3183         if (my $efidisk = $conf->{efidisk0}) {
3184             my $d = parse_drive('efidisk0', $efidisk);
3185             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3186             $format = $d->{format};
3187             if ($storeid) {
3188                 $path = PVE::Storage::path($storecfg, $d->{file});
3189                 if (!defined($format)) {
3190                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3191                     $format = qemu_img_format($scfg, $volname);
3192                 }
3193             } else {
3194                 $path = $d->{file};
3195                 die "efidisk format must be specified\n"
3196                     if !defined($format);
3197             }
3198         } else {
3199             warn "no efidisk configured! Using temporary efivars disk.\n";
3200             $path = "/tmp/$vmid-ovmf.fd";
3201             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3202             $format = 'raw';
3203         }
3204
3205         my $size_str = "";
3206
3207         if ($format eq 'raw' && $version_guard->(4, 1, 2)) {
3208             $size_str = ",size=" . (-s $ovmf_vars);
3209         }
3210
3211         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3212         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0$size_str,file=$path";
3213     }
3214
3215     # load q35 config
3216     if ($q35) {
3217         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3218         if (min_version($machine_version, 4, 0)) {
3219             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3220         } else {
3221             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3222         }
3223     }
3224
3225     if ($conf->{vmgenid}) {
3226         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3227     }
3228
3229     # add usb controllers
3230     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers(
3231         $conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3232     push @$devices, @usbcontrollers if @usbcontrollers;
3233     my $vga = parse_vga($conf->{vga});
3234
3235     my $qxlnum = vga_conf_has_spice($conf->{vga});
3236     $vga->{type} = 'qxl' if $qxlnum;
3237
3238     if (!$vga->{type}) {
3239         if ($arch eq 'aarch64') {
3240             $vga->{type} = 'virtio';
3241         } elsif (min_version($machine_version, 2, 9)) {
3242             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3243         } else {
3244             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3245         }
3246     }
3247
3248     # enable absolute mouse coordinates (needed by vnc)
3249     my $tablet;
3250     if (defined($conf->{tablet})) {
3251         $tablet = $conf->{tablet};
3252     } else {
3253         $tablet = $defaults->{tablet};
3254         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3255         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3256     }
3257
3258     if ($tablet) {
3259         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3260         my $kbd = print_keyboarddevice_full($conf, $arch);
3261         push @$devices, '-device', $kbd if defined($kbd);
3262     }
3263
3264     my $bootorder = device_bootorder($conf);
3265
3266     # host pci device passthrough
3267     my ($kvm_off, $gpu_passthrough, $legacy_igd) = PVE::QemuServer::PCI::print_hostpci_devices(
3268         $vmid, $conf, $devices, $vga, $winversion, $q35, $bridges, $arch, $machine_type, $bootorder);
3269
3270     # usb devices
3271     my $usb_dev_features = {};
3272     $usb_dev_features->{spice_usb3} = 1 if min_version($machine_version, 4, 0);
3273
3274     my @usbdevices = PVE::QemuServer::USB::get_usb_devices(
3275         $conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features, $bootorder);
3276     push @$devices, @usbdevices if @usbdevices;
3277
3278     # serial devices
3279     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3280         if (my $path = $conf->{"serial$i"}) {
3281             if ($path eq 'socket') {
3282                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3283                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3284                 # On aarch64, serial0 is the UART device. Qemu only allows
3285                 # connecting UART devices via the '-serial' command line, as
3286                 # the device has a fixed slot on the hardware...
3287                 if ($arch eq 'aarch64' && $i == 0) {
3288                     push @$devices, '-serial', "chardev:serial$i";
3289                 } else {
3290                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3291                 }
3292             } else {
3293                 die "no such serial device\n" if ! -c $path;
3294                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3295                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3296             }
3297         }
3298     }
3299
3300     # parallel devices
3301     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3302         if (my $path = $conf->{"parallel$i"}) {
3303             die "no such parallel device\n" if ! -c $path;
3304             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3305             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3306             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3307         }
3308     }
3309
3310     if (min_version($machine_version, 4, 0) && (my $audio = conf_has_audio($conf))) {
3311         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3312         my $audio_devs = audio_devs($audio, $audiopciaddr, $machine_version);
3313         push @$devices, @$audio_devs;
3314     }
3315
3316     my $sockets = 1;
3317     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3318     $sockets = $conf->{sockets} if  $conf->{sockets};
3319
3320     my $cores = $conf->{cores} || 1;
3321
3322     my $maxcpus = $sockets * $cores;
3323
3324     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3325
3326     my $allowed_vcpus = $cpuinfo->{cpus};
3327
3328     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3329         if ($allowed_vcpus < $maxcpus);
3330
3331     if($hotplug_features->{cpu} && min_version($machine_version, 2, 7)) {
3332
3333         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3334         for (my $i = 2; $i <= $vcpus; $i++)  {
3335             my $cpustr = print_cpu_device($conf,$i);
3336             push @$cmd, '-device', $cpustr;
3337         }
3338
3339     } else {
3340
3341         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3342     }
3343     push @$cmd, '-nodefaults';
3344
3345     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3346
3347     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3348
3349     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3350
3351     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3352         push @$devices, '-device', print_vga_device(
3353             $conf, $vga, $arch, $machine_version, $machine_type, undef, $qxlnum, $bridges);
3354         my $socket = PVE::QemuServer::Helpers::vnc_socket($vmid);
3355         push @$cmd,  '-vnc', "unix:$socket,password";
3356     } else {
3357         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3358         push @$cmd, '-nographic';
3359     }
3360
3361     # time drift fix
3362     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3363     my $useLocaltime = $conf->{localtime};
3364
3365     if ($winversion >= 5) { # windows
3366         $useLocaltime = 1 if !defined($conf->{localtime});
3367
3368         # use time drift fix when acpi is enabled
3369         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3370             $tdf = 1 if !defined($conf->{tdf});
3371         }
3372     }
3373
3374     if ($winversion >= 6) {
3375         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3376         push @$cmd, '-no-hpet';
3377     }
3378
3379     push @$rtcFlags, 'driftfix=slew' if $tdf;
3380
3381     if ($conf->{startdate} && $conf->{startdate} ne 'now') {
3382         push @$rtcFlags, "base=$conf->{startdate}";
3383     } elsif ($useLocaltime) {
3384         push @$rtcFlags, 'base=localtime';
3385     }
3386
3387     if ($forcecpu) {
3388         push @$cmd, '-cpu', $forcecpu;
3389     } else {
3390         push @$cmd, get_cpu_options($conf, $arch, $kvm, $kvm_off, $machine_version, $winversion, $gpu_passthrough);
3391     }
3392
3393     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3394
3395     push @$cmd, '-S' if $conf->{freeze};
3396
3397     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3398
3399     my $guest_agent = parse_guest_agent($conf);
3400
3401     if ($guest_agent->{enabled}) {
3402         my $qgasocket = PVE::QemuServer::Helpers::qmp_socket($vmid, 1);
3403         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3404
3405         if (!$guest_agent->{type} || $guest_agent->{type} eq 'virtio') {
3406             my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3407             push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3408             push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3409         } elsif ($guest_agent->{type} eq 'isa') {
3410             push @$devices, '-device', "isa-serial,chardev=qga0";
3411         }
3412     }
3413
3414     my $rng = $conf->{rng0} ? parse_rng($conf->{rng0}) : undef;
3415     if ($rng && $version_guard->(4, 1, 2)) {
3416         check_rng_source($rng->{source});
3417
3418         my $max_bytes = $rng->{max_bytes} // $rng_fmt->{max_bytes}->{default};
3419         my $period = $rng->{period} // $rng_fmt->{period}->{default};
3420         my $limiter_str = "";
3421         if ($max_bytes) {
3422             $limiter_str = ",max-bytes=$max_bytes,period=$period";
3423         }
3424
3425         my $rng_addr = print_pci_addr("rng0", $bridges, $arch, $machine_type);
3426         push @$devices, '-object', "rng-random,filename=$rng->{source},id=rng0";
3427         push @$devices, '-device', "virtio-rng-pci,rng=rng0$limiter_str$rng_addr";
3428     }
3429
3430     my $spice_port;
3431
3432     if ($qxlnum) {
3433         if ($qxlnum > 1) {
3434             if ($winversion){
3435                 for (my $i = 1; $i < $qxlnum; $i++){
3436                     push @$devices, '-device', print_vga_device(
3437                         $conf, $vga, $arch, $machine_version, $machine_type, $i, $qxlnum, $bridges);
3438                 }
3439             } else {
3440                 # assume other OS works like Linux
3441                 my ($ram, $vram) = ("134217728", "67108864");
3442                 if ($vga->{memory}) {
3443                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3444                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3445                 }
3446                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3447                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3448             }
3449         }
3450
3451         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3452
3453         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3454         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3455         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3456
3457         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3458         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3459         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3460
3461         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3462         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3463
3464         my $spice_enhancement_str = $conf->{spice_enhancements} // '';
3465         my $spice_enhancement = parse_property_string($spice_enhancements_fmt, $spice_enhancement_str);
3466         if ($spice_enhancement->{foldersharing}) {
3467             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3468             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3469         }
3470
3471         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3472         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}"
3473             if $spice_enhancement->{videostreaming};
3474
3475         push @$devices, '-spice', "$spice_opts";
3476     }
3477
3478     # enable balloon by default, unless explicitly disabled
3479     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3480         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3481         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3482     }
3483
3484     if ($conf->{watchdog}) {
3485         my $wdopts = parse_watchdog($conf->{watchdog});
3486         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3487         my $watchdog = $wdopts->{model} || 'i6300esb';
3488         push @$devices, '-device', "$watchdog$pciaddr";
3489         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3490     }
3491
3492     my $vollist = [];
3493     my $scsicontroller = {};
3494     my $ahcicontroller = {};
3495     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3496
3497     # Add iscsi initiator name if available
3498     if (my $initiator = get_initiator_name()) {
3499         push @$devices, '-iscsi', "initiator-name=$initiator";
3500     }
3501
3502     PVE::QemuConfig->foreach_volume($conf, sub {
3503         my ($ds, $drive) = @_;
3504
3505         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3506             push @$vollist, $drive->{file};
3507         }
3508
3509         # ignore efidisk here, already added in bios/fw handling code above
3510         return if $drive->{interface} eq 'efidisk';
3511
3512         $use_virtio = 1 if $ds =~ m/^virtio/;
3513
3514         $drive->{bootindex} = $bootorder->{$ds} if $bootorder->{$ds};
3515
3516         if ($drive->{interface} eq 'virtio'){
3517            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3518         }
3519
3520         if ($drive->{interface} eq 'scsi') {
3521
3522             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3523
3524             die "scsi$drive->{index}: machine version 4.1~pve2 or higher is required to use more than 14 SCSI disks\n"
3525                 if $drive->{index} > 13 && !&$version_guard(4, 1, 2);
3526
3527             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
3528             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3529
3530             my $iothread = '';
3531             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3532                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3533                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3534             } elsif ($drive->{iothread}) {
3535                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3536             }
3537
3538             my $queues = '';
3539             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3540                 $queues = ",num_queues=$drive->{queues}";
3541             }
3542
3543             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues"
3544                 if !$scsicontroller->{$controller};
3545             $scsicontroller->{$controller}=1;
3546         }
3547
3548         if ($drive->{interface} eq 'sata') {
3549             my $controller = int($drive->{index} / $PVE::QemuServer::Drive::MAX_SATA_DISKS);
3550             $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
3551             push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr"
3552                 if !$ahcicontroller->{$controller};
3553             $ahcicontroller->{$controller}=1;
3554         }
3555
3556         my $drive_cmd = print_drive_commandline_full($storecfg, $vmid, $drive);
3557         $drive_cmd .= ',readonly' if PVE::QemuConfig->is_template($conf);
3558
3559         push @$devices, '-drive',$drive_cmd;
3560         push @$devices, '-device', print_drivedevice_full(
3561             $storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
3562     });
3563
3564     for (my $i = 0; $i < $MAX_NETS; $i++) {
3565         my $netname = "net$i";
3566
3567         next if !$conf->{$netname};
3568         my $d = parse_net($conf->{$netname});
3569         next if !$d;
3570
3571         $use_virtio = 1 if $d->{model} eq 'virtio';
3572
3573         $d->{bootindex} = $bootorder->{$netname} if $bootorder->{$netname};
3574
3575         my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, $netname);
3576         push @$devices, '-netdev', $netdevfull;
3577
3578         my $netdevicefull = print_netdevice_full(
3579             $vmid, $conf, $d, $netname, $bridges, $use_old_bios_files, $arch, $machine_type);
3580
3581         push @$devices, '-device', $netdevicefull;
3582     }
3583
3584     if ($conf->{ivshmem}) {
3585         my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
3586
3587         my $bus;
3588         if ($q35) {
3589             $bus = print_pcie_addr("ivshmem");
3590         } else {
3591             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
3592         }
3593
3594         my $ivshmem_name = $ivshmem->{name} // $vmid;
3595         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
3596
3597         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
3598         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path"
3599             .",size=$ivshmem->{size}M";
3600     }
3601
3602     # pci.4 is nested in pci.1
3603     $bridges->{1} = 1 if $bridges->{4};
3604
3605     if (!$q35) {
3606         # add pci bridges
3607         if (min_version($machine_version, 2, 3)) {
3608            $bridges->{1} = 1;
3609            $bridges->{2} = 1;
3610         }
3611
3612         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3613
3614     }
3615
3616     for my $k (sort {$b cmp $a} keys %$bridges) {
3617         next if $q35 && $k < 4; # q35.cfg already includes bridges up to 3
3618
3619         my $k_name = $k;
3620         if ($k == 2 && $legacy_igd) {
3621             $k_name = "$k-igd";
3622         }
3623         $pciaddr = print_pci_addr("pci.$k_name", undef, $arch, $machine_type);
3624
3625         my $devstr = "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr";
3626         if ($q35) {
3627             # add after -readconfig pve-q35.cfg
3628             splice @$devices, 2, 0, '-device', $devstr;
3629         } else {
3630             unshift @$devices, '-device', $devstr if $k > 0;
3631         }
3632     }
3633
3634     if (!$kvm) {
3635         push @$machineFlags, 'accel=tcg';
3636     }
3637
3638     my $machine_type_min = $machine_type;
3639     if ($add_pve_version) {
3640         $machine_type_min =~ s/\+pve\d+$//;
3641         $machine_type_min .= "+pve$required_pve_version";
3642     }
3643     push @$machineFlags, "type=${machine_type_min}";
3644
3645     push @$cmd, @$devices;
3646     push @$cmd, '-rtc', join(',', @$rtcFlags) if scalar(@$rtcFlags);
3647     push @$cmd, '-machine', join(',', @$machineFlags) if scalar(@$machineFlags);
3648     push @$cmd, '-global', join(',', @$globalFlags) if scalar(@$globalFlags);
3649
3650     if (my $vmstate = $conf->{vmstate}) {
3651         my $statepath = PVE::Storage::path($storecfg, $vmstate);
3652         push @$vollist, $vmstate;
3653         push @$cmd, '-loadstate', $statepath;
3654         print "activating and using '$vmstate' as vmstate\n";
3655     }
3656
3657     # add custom args
3658     if ($conf->{args}) {
3659         my $aa = PVE::Tools::split_args($conf->{args});
3660         push @$cmd, @$aa;
3661     }
3662
3663     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3664 }
3665
3666 sub check_rng_source {
3667     my ($source) = @_;
3668
3669     # mostly relevant for /dev/hwrng, but doesn't hurt to check others too
3670     die "cannot create VirtIO RNG device: source file '$source' doesn't exist\n"
3671         if ! -e $source;
3672
3673     my $rng_current = '/sys/devices/virtual/misc/hw_random/rng_current';
3674     if ($source eq '/dev/hwrng' && file_read_firstline($rng_current) eq 'none') {
3675         # Needs to abort, otherwise QEMU crashes on first rng access. Note that rng_current cannot
3676         # be changed to 'none' manually, so once the VM is past this point, it's no longer an issue.
3677         die "Cannot start VM with passed-through RNG device: '/dev/hwrng' exists, but"
3678             ." '$rng_current' is set to 'none'. Ensure that a compatible hardware-RNG is attached"
3679             ." to the host.\n";
3680     }
3681 }
3682
3683 sub spice_port {
3684     my ($vmid) = @_;
3685
3686     my $res = mon_cmd($vmid, 'query-spice');
3687
3688     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3689 }
3690
3691 sub vm_devices_list {
3692     my ($vmid) = @_;
3693
3694     my $res = mon_cmd($vmid, 'query-pci');
3695     my $devices_to_check = [];
3696     my $devices = {};
3697     foreach my $pcibus (@$res) {
3698         push @$devices_to_check, @{$pcibus->{devices}},
3699     }
3700
3701     while (@$devices_to_check) {
3702         my $to_check = [];
3703         for my $d (@$devices_to_check) {
3704             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
3705             next if !$d->{'pci_bridge'};
3706
3707             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
3708             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
3709         }
3710         $devices_to_check = $to_check;
3711     }
3712
3713     my $resblock = mon_cmd($vmid, 'query-block');
3714     foreach my $block (@$resblock) {
3715         if($block->{device} =~ m/^drive-(\S+)/){
3716                 $devices->{$1} = 1;
3717         }
3718     }
3719
3720     my $resmice = mon_cmd($vmid, 'query-mice');
3721     foreach my $mice (@$resmice) {
3722         if ($mice->{name} eq 'QEMU HID Tablet') {
3723             $devices->{tablet} = 1;
3724             last;
3725         }
3726     }
3727
3728     # for usb devices there is no query-usb
3729     # but we can iterate over the entries in
3730     # qom-list path=/machine/peripheral
3731     my $resperipheral = mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3732     foreach my $per (@$resperipheral) {
3733         if ($per->{name} =~ m/^usb\d+$/) {
3734             $devices->{$per->{name}} = 1;
3735         }
3736     }
3737
3738     return $devices;
3739 }
3740
3741 sub vm_deviceplug {
3742     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
3743
3744     my $q35 = PVE::QemuServer::Machine::machine_type_is_q35($conf);
3745
3746     my $devices_list = vm_devices_list($vmid);
3747     return 1 if defined($devices_list->{$deviceid});
3748
3749     # add PCI bridge if we need it for the device
3750     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type);
3751
3752     if ($deviceid eq 'tablet') {
3753
3754         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
3755
3756     } elsif ($deviceid eq 'keyboard') {
3757
3758         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
3759
3760     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3761
3762         die "usb hotplug currently not reliable\n";
3763         # since we can't reliably hot unplug all added usb devices and usb
3764         # passthrough breaks live migration we disable usb hotplugging for now
3765         #qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3766
3767     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3768
3769         qemu_iothread_add($vmid, $deviceid, $device);
3770
3771         qemu_driveadd($storecfg, $vmid, $device);
3772         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3773
3774         qemu_deviceadd($vmid, $devicefull);
3775         eval { qemu_deviceaddverify($vmid, $deviceid); };
3776         if (my $err = $@) {
3777             eval { qemu_drivedel($vmid, $deviceid); };
3778             warn $@ if $@;
3779             die $err;
3780         }
3781
3782     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3783
3784
3785         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3786         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3787         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3788
3789         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3790
3791         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3792             qemu_iothread_add($vmid, $deviceid, $device);
3793             $devicefull .= ",iothread=iothread-$deviceid";
3794         }
3795
3796         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3797             $devicefull .= ",num_queues=$device->{queues}";
3798         }
3799
3800         qemu_deviceadd($vmid, $devicefull);
3801         qemu_deviceaddverify($vmid, $deviceid);
3802
3803     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3804
3805         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
3806         qemu_driveadd($storecfg, $vmid, $device);
3807
3808         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, undef, $arch, $machine_type);
3809         eval { qemu_deviceadd($vmid, $devicefull); };
3810         if (my $err = $@) {
3811             eval { qemu_drivedel($vmid, $deviceid); };
3812             warn $@ if $@;
3813             die $err;
3814         }
3815
3816     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3817
3818         return if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
3819
3820         my $machine_type = PVE::QemuServer::Machine::qemu_machine_pxe($vmid, $conf);
3821         my $use_old_bios_files = undef;
3822         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3823
3824         my $netdevicefull = print_netdevice_full(
3825             $vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
3826         qemu_deviceadd($vmid, $netdevicefull);
3827         eval {
3828             qemu_deviceaddverify($vmid, $deviceid);
3829             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
3830         };
3831         if (my $err = $@) {
3832             eval { qemu_netdevdel($vmid, $deviceid); };
3833             warn $@ if $@;
3834             die $err;
3835         }
3836
3837     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3838
3839         my $bridgeid = $2;
3840         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
3841         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3842
3843         qemu_deviceadd($vmid, $devicefull);
3844         qemu_deviceaddverify($vmid, $deviceid);
3845
3846     } else {
3847         die "can't hotplug device '$deviceid'\n";
3848     }
3849
3850     return 1;
3851 }
3852
3853 # fixme: this should raise exceptions on error!
3854 sub vm_deviceunplug {
3855     my ($vmid, $conf, $deviceid) = @_;
3856
3857     my $devices_list = vm_devices_list($vmid);
3858     return 1 if !defined($devices_list->{$deviceid});
3859
3860     my $bootdisks = PVE::QemuServer::Drive::get_bootdisks($conf);
3861     die "can't unplug bootdisk '$deviceid'\n" if grep {$_ eq $deviceid} @$bootdisks;
3862
3863     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
3864
3865         qemu_devicedel($vmid, $deviceid);
3866
3867     } elsif ($deviceid =~ m/^usb\d+$/) {
3868
3869         die "usb hotplug currently not reliable\n";
3870         # when unplugging usb devices this way, there may be remaining usb
3871         # controllers/hubs so we disable it for now
3872         #qemu_devicedel($vmid, $deviceid);
3873         #qemu_devicedelverify($vmid, $deviceid);
3874
3875     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3876
3877         qemu_devicedel($vmid, $deviceid);
3878         qemu_devicedelverify($vmid, $deviceid);
3879         qemu_drivedel($vmid, $deviceid);
3880         qemu_iothread_del($conf, $vmid, $deviceid);
3881
3882     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3883
3884         qemu_devicedel($vmid, $deviceid);
3885         qemu_devicedelverify($vmid, $deviceid);
3886         qemu_iothread_del($conf, $vmid, $deviceid);
3887
3888     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3889
3890         qemu_devicedel($vmid, $deviceid);
3891         qemu_drivedel($vmid, $deviceid);
3892         qemu_deletescsihw($conf, $vmid, $deviceid);
3893
3894     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3895
3896         qemu_devicedel($vmid, $deviceid);
3897         qemu_devicedelverify($vmid, $deviceid);
3898         qemu_netdevdel($vmid, $deviceid);
3899
3900     } else {
3901         die "can't unplug device '$deviceid'\n";
3902     }
3903
3904     return 1;
3905 }
3906
3907 sub qemu_deviceadd {
3908     my ($vmid, $devicefull) = @_;
3909
3910     $devicefull = "driver=".$devicefull;
3911     my %options =  split(/[=,]/, $devicefull);
3912
3913     mon_cmd($vmid, "device_add" , %options);
3914 }
3915
3916 sub qemu_devicedel {
3917     my ($vmid, $deviceid) = @_;
3918
3919     my $ret = mon_cmd($vmid, "device_del", id => $deviceid);
3920 }
3921
3922 sub qemu_iothread_add {
3923     my($vmid, $deviceid, $device) = @_;
3924
3925     if ($device->{iothread}) {
3926         my $iothreads = vm_iothreads_list($vmid);
3927         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3928     }
3929 }
3930
3931 sub qemu_iothread_del {
3932     my($conf, $vmid, $deviceid) = @_;
3933
3934     my $confid = $deviceid;
3935     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
3936         $confid = 'scsi' . $1;
3937     }
3938     my $device = parse_drive($confid, $conf->{$confid});
3939     if ($device->{iothread}) {
3940         my $iothreads = vm_iothreads_list($vmid);
3941         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3942     }
3943 }
3944
3945 sub qemu_objectadd {
3946     my($vmid, $objectid, $qomtype) = @_;
3947
3948     mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3949
3950     return 1;
3951 }
3952
3953 sub qemu_objectdel {
3954     my($vmid, $objectid) = @_;
3955
3956     mon_cmd($vmid, "object-del", id => $objectid);
3957
3958     return 1;
3959 }
3960
3961 sub qemu_driveadd {
3962     my ($storecfg, $vmid, $device) = @_;
3963
3964     my $drive = print_drive_commandline_full($storecfg, $vmid, $device);
3965     $drive =~ s/\\/\\\\/g;
3966     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_add auto \"$drive\"");
3967
3968     # If the command succeeds qemu prints: "OK"
3969     return 1 if $ret =~ m/OK/s;
3970
3971     die "adding drive failed: $ret\n";
3972 }
3973
3974 sub qemu_drivedel {
3975     my($vmid, $deviceid) = @_;
3976
3977     my $ret = PVE::QemuServer::Monitor::hmp_cmd($vmid, "drive_del drive-$deviceid");
3978     $ret =~ s/^\s+//;
3979
3980     return 1 if $ret eq "";
3981
3982     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3983     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3984
3985     die "deleting drive $deviceid failed : $ret\n";
3986 }
3987
3988 sub qemu_deviceaddverify {
3989     my ($vmid, $deviceid) = @_;
3990
3991     for (my $i = 0; $i <= 5; $i++) {
3992          my $devices_list = vm_devices_list($vmid);
3993          return 1 if defined($devices_list->{$deviceid});
3994          sleep 1;
3995     }
3996
3997     die "error on hotplug device '$deviceid'\n";
3998 }
3999
4000
4001 sub qemu_devicedelverify {
4002     my ($vmid, $deviceid) = @_;
4003
4004     # need to verify that the device is correctly removed as device_del
4005     # is async and empty return is not reliable
4006
4007     for (my $i = 0; $i <= 5; $i++) {
4008          my $devices_list = vm_devices_list($vmid);
4009          return 1 if !defined($devices_list->{$deviceid});
4010          sleep 1;
4011     }
4012
4013     die "error on hot-unplugging device '$deviceid'\n";
4014 }
4015
4016 sub qemu_findorcreatescsihw {
4017     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4018
4019     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4020
4021     my $scsihwid="$controller_prefix$controller";
4022     my $devices_list = vm_devices_list($vmid);
4023
4024     if(!defined($devices_list->{$scsihwid})) {
4025         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4026     }
4027
4028     return 1;
4029 }
4030
4031 sub qemu_deletescsihw {
4032     my ($conf, $vmid, $opt) = @_;
4033
4034     my $device = parse_drive($opt, $conf->{$opt});
4035
4036     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4037         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4038         return 1;
4039     }
4040
4041     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4042
4043     my $devices_list = vm_devices_list($vmid);
4044     foreach my $opt (keys %{$devices_list}) {
4045         if (is_valid_drivename($opt)) {
4046             my $drive = parse_drive($opt, $conf->{$opt});
4047             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4048                 return 1;
4049             }
4050         }
4051     }
4052
4053     my $scsihwid="scsihw$controller";
4054
4055     vm_deviceunplug($vmid, $conf, $scsihwid);
4056
4057     return 1;
4058 }
4059
4060 sub qemu_add_pci_bridge {
4061     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4062
4063     my $bridges = {};
4064
4065     my $bridgeid;
4066
4067     print_pci_addr($device, $bridges, $arch, $machine_type);
4068
4069     while (my ($k, $v) = each %$bridges) {
4070         $bridgeid = $k;
4071     }
4072     return 1 if !defined($bridgeid) || $bridgeid < 1;
4073
4074     my $bridge = "pci.$bridgeid";
4075     my $devices_list = vm_devices_list($vmid);
4076
4077     if (!defined($devices_list->{$bridge})) {
4078         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4079     }
4080
4081     return 1;
4082 }
4083
4084 sub qemu_set_link_status {
4085     my ($vmid, $device, $up) = @_;
4086
4087     mon_cmd($vmid, "set_link", name => $device,
4088                up => $up ? JSON::true : JSON::false);
4089 }
4090
4091 sub qemu_netdevadd {
4092     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4093
4094     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4095     my %options =  split(/[=,]/, $netdev);
4096
4097     if (defined(my $vhost = $options{vhost})) {
4098         $options{vhost} = JSON::boolean(PVE::JSONSchema::parse_boolean($vhost));
4099     }
4100
4101     if (defined(my $queues = $options{queues})) {
4102         $options{queues} = $queues + 0;
4103     }
4104
4105     mon_cmd($vmid, "netdev_add",  %options);
4106     return 1;
4107 }
4108
4109 sub qemu_netdevdel {
4110     my ($vmid, $deviceid) = @_;
4111
4112     mon_cmd($vmid, "netdev_del", id => $deviceid);
4113 }
4114
4115 sub qemu_usb_hotplug {
4116     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4117
4118     return if !$device;
4119
4120     # remove the old one first
4121     vm_deviceunplug($vmid, $conf, $deviceid);
4122
4123     # check if xhci controller is necessary and available
4124     if ($device->{usb3}) {
4125
4126         my $devicelist = vm_devices_list($vmid);
4127
4128         if (!$devicelist->{xhci}) {
4129             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4130             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4131         }
4132     }
4133     my $d = parse_usb_device($device->{host});
4134     $d->{usb3} = $device->{usb3};
4135
4136     # add the new one
4137     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4138 }
4139
4140 sub qemu_cpu_hotplug {
4141     my ($vmid, $conf, $vcpus) = @_;
4142
4143     my $machine_type = PVE::QemuServer::Machine::get_current_qemu_machine($vmid);
4144
4145     my $sockets = 1;
4146     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4147     $sockets = $conf->{sockets} if  $conf->{sockets};
4148     my $cores = $conf->{cores} || 1;
4149     my $maxcpus = $sockets * $cores;
4150
4151     $vcpus = $maxcpus if !$vcpus;
4152
4153     die "you can't add more vcpus than maxcpus\n"
4154         if $vcpus > $maxcpus;
4155
4156     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4157
4158     if ($vcpus < $currentvcpus) {
4159
4160         if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4161
4162             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4163                 qemu_devicedel($vmid, "cpu$i");
4164                 my $retry = 0;
4165                 my $currentrunningvcpus = undef;
4166                 while (1) {
4167                     $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4168                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4169                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4170                     $retry++;
4171                     sleep 1;
4172                 }
4173                 #update conf after each succesfull cpu unplug
4174                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4175                 PVE::QemuConfig->write_config($vmid, $conf);
4176             }
4177         } else {
4178             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4179         }
4180
4181         return;
4182     }
4183
4184     my $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4185     die "vcpus in running vm does not match its configuration\n"
4186         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4187
4188     if (PVE::QemuServer::Machine::machine_version($machine_type, 2, 7)) {
4189
4190         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4191             my $cpustr = print_cpu_device($conf, $i);
4192             qemu_deviceadd($vmid, $cpustr);
4193
4194             my $retry = 0;
4195             my $currentrunningvcpus = undef;
4196             while (1) {
4197                 $currentrunningvcpus = mon_cmd($vmid, "query-cpus-fast");
4198                 last if scalar(@{$currentrunningvcpus}) == $i;
4199                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4200                 sleep 1;
4201                 $retry++;
4202             }
4203             #update conf after each succesfull cpu hotplug
4204             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4205             PVE::QemuConfig->write_config($vmid, $conf);
4206         }
4207     } else {
4208
4209         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4210             mon_cmd($vmid, "cpu-add", id => int($i));
4211         }
4212     }
4213 }
4214
4215 sub qemu_block_set_io_throttle {
4216     my ($vmid, $deviceid,
4217         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4218         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4219         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4220         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4221
4222     return if !check_running($vmid) ;
4223
4224     mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4225         bps => int($bps),
4226         bps_rd => int($bps_rd),
4227         bps_wr => int($bps_wr),
4228         iops => int($iops),
4229         iops_rd => int($iops_rd),
4230         iops_wr => int($iops_wr),
4231         bps_max => int($bps_max),
4232         bps_rd_max => int($bps_rd_max),
4233         bps_wr_max => int($bps_wr_max),
4234         iops_max => int($iops_max),
4235         iops_rd_max => int($iops_rd_max),
4236         iops_wr_max => int($iops_wr_max),
4237         bps_max_length => int($bps_max_length),
4238         bps_rd_max_length => int($bps_rd_max_length),
4239         bps_wr_max_length => int($bps_wr_max_length),
4240         iops_max_length => int($iops_max_length),
4241         iops_rd_max_length => int($iops_rd_max_length),
4242         iops_wr_max_length => int($iops_wr_max_length),
4243     );
4244
4245 }
4246
4247 sub qemu_block_resize {
4248     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4249
4250     my $running = check_running($vmid);
4251
4252     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4253
4254     return if !$running;
4255
4256     my $padding = (1024 - $size % 1024) % 1024;
4257     $size = $size + $padding;
4258
4259     mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4260
4261 }
4262
4263 sub qemu_volume_snapshot {
4264     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4265
4266     my $running = check_running($vmid);
4267
4268     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4269         mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4270     } else {
4271         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4272     }
4273 }
4274
4275 sub qemu_volume_snapshot_delete {
4276     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4277
4278     my $running = check_running($vmid);
4279
4280     if($running) {
4281
4282         $running = undef;
4283         my $conf = PVE::QemuConfig->load_config($vmid);
4284         PVE::QemuConfig->foreach_volume($conf, sub {
4285             my ($ds, $drive) = @_;
4286             $running = 1 if $drive->{file} eq $volid;
4287         });
4288     }
4289
4290     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4291         mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4292     } else {
4293         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4294     }
4295 }
4296
4297 sub set_migration_caps {
4298     my ($vmid) = @_;
4299
4300     my $qemu_support = eval { mon_cmd($vmid, "query-proxmox-support") };
4301
4302     my $cap_ref = [];
4303
4304     my $enabled_cap = {
4305         "auto-converge" => 1,
4306         "xbzrle" => 1,
4307         "x-rdma-pin-all" => 0,
4308         "zero-blocks" => 0,
4309         "compress" => 0,
4310         "dirty-bitmaps" => $qemu_support->{'pbs-dirty-bitmap-migration'} ? 1 : 0,
4311     };
4312
4313     my $supported_capabilities = mon_cmd($vmid, "query-migrate-capabilities");
4314
4315     for my $supported_capability (@$supported_capabilities) {
4316         push @$cap_ref, {
4317             capability => $supported_capability->{capability},
4318             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4319         };
4320     }
4321
4322     mon_cmd($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4323 }
4324
4325 sub foreach_volid {
4326     my ($conf, $func, @param) = @_;
4327
4328     my $volhash = {};
4329
4330     my $test_volid = sub {
4331         my ($key, $drive, $snapname) = @_;
4332
4333         my $volid = $drive->{file};
4334         return if !$volid;
4335
4336         $volhash->{$volid}->{cdrom} //= 1;
4337         $volhash->{$volid}->{cdrom} = 0 if !drive_is_cdrom($drive);
4338
4339         my $replicate = $drive->{replicate} // 1;
4340         $volhash->{$volid}->{replicate} //= 0;
4341         $volhash->{$volid}->{replicate} = 1 if $replicate;
4342
4343         $volhash->{$volid}->{shared} //= 0;
4344         $volhash->{$volid}->{shared} = 1 if $drive->{shared};
4345
4346         $volhash->{$volid}->{referenced_in_config} //= 0;
4347         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
4348
4349         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
4350             if defined($snapname);
4351
4352         my $size = $drive->{size};
4353         $volhash->{$volid}->{size} //= $size if $size;
4354
4355         $volhash->{$volid}->{is_vmstate} //= 0;
4356         $volhash->{$volid}->{is_vmstate} = 1 if $key eq 'vmstate';
4357
4358         $volhash->{$volid}->{is_unused} //= 0;
4359         $volhash->{$volid}->{is_unused} = 1 if $key =~ /^unused\d+$/;
4360     };
4361
4362     my $include_opts = {
4363         extra_keys => ['vmstate'],
4364         include_unused => 1,
4365     };
4366
4367     PVE::QemuConfig->foreach_volume_full($conf, $include_opts, $test_volid);
4368     foreach my $snapname (keys %{$conf->{snapshots}}) {
4369         my $snap = $conf->{snapshots}->{$snapname};
4370         PVE::QemuConfig->foreach_volume_full($snap, $include_opts, $test_volid, $snapname);
4371     }
4372
4373     foreach my $volid (keys %$volhash) {
4374         &$func($volid, $volhash->{$volid}, @param);
4375     }
4376 }
4377
4378 my $fast_plug_option = {
4379     'lock' => 1,
4380     'name' => 1,
4381     'onboot' => 1,
4382     'shares' => 1,
4383     'startup' => 1,
4384     'description' => 1,
4385     'protection' => 1,
4386     'vmstatestorage' => 1,
4387     'hookscript' => 1,
4388     'tags' => 1,
4389 };
4390
4391 # hotplug changes in [PENDING]
4392 # $selection hash can be used to only apply specified options, for
4393 # example: { cores => 1 } (only apply changed 'cores')
4394 # $errors ref is used to return error messages
4395 sub vmconfig_hotplug_pending {
4396     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4397
4398     my $defaults = load_defaults();
4399     my $arch = get_vm_arch($conf);
4400     my $machine_type = get_vm_machine($conf, undef, $arch);
4401
4402     # commit values which do not have any impact on running VM first
4403     # Note: those option cannot raise errors, we we do not care about
4404     # $selection and always apply them.
4405
4406     my $add_error = sub {
4407         my ($opt, $msg) = @_;
4408         $errors->{$opt} = "hotplug problem - $msg";
4409     };
4410
4411     my $changes = 0;
4412     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4413         if ($fast_plug_option->{$opt}) {
4414             $conf->{$opt} = $conf->{pending}->{$opt};
4415             delete $conf->{pending}->{$opt};
4416             $changes = 1;
4417         }
4418     }
4419
4420     if ($changes) {
4421         PVE::QemuConfig->write_config($vmid, $conf);
4422     }
4423
4424     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4425
4426     my $cgroup = PVE::QemuServer::CGroup->new($vmid);
4427     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4428     foreach my $opt (sort keys %$pending_delete_hash) {
4429         next if $selection && !$selection->{$opt};
4430         my $force = $pending_delete_hash->{$opt}->{force};
4431         eval {
4432             if ($opt eq 'hotplug') {
4433                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4434             } elsif ($opt eq 'tablet') {
4435                 die "skip\n" if !$hotplug_features->{usb};
4436                 if ($defaults->{tablet}) {
4437                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4438                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4439                         if $arch eq 'aarch64';
4440                 } else {
4441                     vm_deviceunplug($vmid, $conf, 'tablet');
4442                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4443                 }
4444             } elsif ($opt =~ m/^usb\d+/) {
4445                 die "skip\n";
4446                 # since we cannot reliably hot unplug usb devices we are disabling it
4447                 #die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4448                 #vm_deviceunplug($vmid, $conf, $opt);
4449             } elsif ($opt eq 'vcpus') {
4450                 die "skip\n" if !$hotplug_features->{cpu};
4451                 qemu_cpu_hotplug($vmid, $conf, undef);
4452             } elsif ($opt eq 'balloon') {
4453                 # enable balloon device is not hotpluggable
4454                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4455                 # here we reset the ballooning value to memory
4456                 my $balloon = $conf->{memory} || $defaults->{memory};
4457                 mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4458             } elsif ($fast_plug_option->{$opt}) {
4459                 # do nothing
4460             } elsif ($opt =~ m/^net(\d+)$/) {
4461                 die "skip\n" if !$hotplug_features->{network};
4462                 vm_deviceunplug($vmid, $conf, $opt);
4463             } elsif (is_valid_drivename($opt)) {
4464                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4465                 vm_deviceunplug($vmid, $conf, $opt);
4466                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4467             } elsif ($opt =~ m/^memory$/) {
4468                 die "skip\n" if !$hotplug_features->{memory};
4469                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4470             } elsif ($opt eq 'cpuunits') {
4471                 $cgroup->change_cpu_shares(undef, $defaults->{cpuunits});
4472             } elsif ($opt eq 'cpulimit') {
4473                 $cgroup->change_cpu_quota(-1, 100000);
4474             } else {
4475                 die "skip\n";
4476             }
4477         };
4478         if (my $err = $@) {
4479             &$add_error($opt, $err) if $err ne "skip\n";
4480         } else {
4481             delete $conf->{$opt};
4482             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4483         }
4484     }
4485
4486     my ($apply_pending_cloudinit, $apply_pending_cloudinit_done);
4487     $apply_pending_cloudinit = sub {
4488         return if $apply_pending_cloudinit_done; # once is enough
4489         $apply_pending_cloudinit_done = 1; # once is enough
4490
4491         my ($key, $value) = @_;
4492
4493         my @cloudinit_opts = keys %$confdesc_cloudinit;
4494         foreach my $opt (keys %{$conf->{pending}}) {
4495             next if !grep { $_ eq $opt } @cloudinit_opts;
4496             $conf->{$opt} = delete $conf->{pending}->{$opt};
4497         }
4498
4499         my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4500         foreach my $opt (sort keys %$pending_delete_hash) {
4501             next if !grep { $_ eq $opt } @cloudinit_opts;
4502             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4503             delete $conf->{$opt};
4504         }
4505
4506         my $new_conf = { %$conf };
4507         $new_conf->{$key} = $value;
4508         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4509     };
4510
4511     foreach my $opt (keys %{$conf->{pending}}) {
4512         next if $selection && !$selection->{$opt};
4513         my $value = $conf->{pending}->{$opt};
4514         eval {
4515             if ($opt eq 'hotplug') {
4516                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4517             } elsif ($opt eq 'tablet') {
4518                 die "skip\n" if !$hotplug_features->{usb};
4519                 if ($value == 1) {
4520                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4521                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4522                         if $arch eq 'aarch64';
4523                 } elsif ($value == 0) {
4524                     vm_deviceunplug($vmid, $conf, 'tablet');
4525                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4526                 }
4527             } elsif ($opt =~ m/^usb\d+$/) {
4528                 die "skip\n";
4529                 # since we cannot reliably hot unplug usb devices we disable it for now
4530                 #die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4531                 #my $d = eval { parse_property_string($usbdesc->{format}, $value) };
4532                 #die "skip\n" if !$d;
4533                 #qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4534             } elsif ($opt eq 'vcpus') {
4535                 die "skip\n" if !$hotplug_features->{cpu};
4536                 qemu_cpu_hotplug($vmid, $conf, $value);
4537             } elsif ($opt eq 'balloon') {
4538                 # enable/disable balloning device is not hotpluggable
4539                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4540                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4541                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4542
4543                 # allow manual ballooning if shares is set to zero
4544                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4545                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4546                     mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4547                 }
4548             } elsif ($opt =~ m/^net(\d+)$/) {
4549                 # some changes can be done without hotplug
4550                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4551                                     $vmid, $opt, $value, $arch, $machine_type);
4552             } elsif (is_valid_drivename($opt)) {
4553                 die "skip\n" if $opt eq 'efidisk0';
4554                 # some changes can be done without hotplug
4555                 my $drive = parse_drive($opt, $value);
4556                 if (drive_is_cloudinit($drive)) {
4557                     &$apply_pending_cloudinit($opt, $value);
4558                 }
4559                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4560                                      $vmid, $opt, $value, $arch, $machine_type);
4561             } elsif ($opt =~ m/^memory$/) { #dimms
4562                 die "skip\n" if !$hotplug_features->{memory};
4563                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4564             } elsif ($opt eq 'cpuunits') {
4565                 $cgroup->change_cpu_shares($conf->{pending}->{$opt}, $defaults->{cpuunits});
4566             } elsif ($opt eq 'cpulimit') {
4567                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4568                 $cgroup->change_cpu_quota($cpulimit, 100000);
4569             } else {
4570                 die "skip\n";  # skip non-hot-pluggable options
4571             }
4572         };
4573         if (my $err = $@) {
4574             &$add_error($opt, $err) if $err ne "skip\n";
4575         } else {
4576             $conf->{$opt} = $value;
4577             delete $conf->{pending}->{$opt};
4578         }
4579     }
4580
4581     PVE::QemuConfig->write_config($vmid, $conf);
4582 }
4583
4584 sub try_deallocate_drive {
4585     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4586
4587     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4588         my $volid = $drive->{file};
4589         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4590             my $sid = PVE::Storage::parse_volume_id($volid);
4591             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4592
4593             # check if the disk is really unused
4594             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4595                 if PVE::QemuServer::Drive::is_volume_in_use($storecfg, $conf, $key, $volid);
4596             PVE::Storage::vdisk_free($storecfg, $volid);
4597             return 1;
4598         } else {
4599             # If vm is not owner of this disk remove from config
4600             return 1;
4601         }
4602     }
4603
4604     return;
4605 }
4606
4607 sub vmconfig_delete_or_detach_drive {
4608     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4609
4610     my $drive = parse_drive($opt, $conf->{$opt});
4611
4612     my $rpcenv = PVE::RPCEnvironment::get();
4613     my $authuser = $rpcenv->get_user();
4614
4615     if ($force) {
4616         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4617         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4618     } else {
4619         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4620     }
4621 }
4622
4623
4624
4625 sub vmconfig_apply_pending {
4626     my ($vmid, $conf, $storecfg, $errors) = @_;
4627
4628     my $add_apply_error = sub {
4629         my ($opt, $msg) = @_;
4630         my $err_msg = "unable to apply pending change $opt : $msg";
4631         $errors->{$opt} = $err_msg;
4632         warn $err_msg;
4633     };
4634
4635     # cold plug
4636
4637     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4638     foreach my $opt (sort keys %$pending_delete_hash) {
4639         my $force = $pending_delete_hash->{$opt}->{force};
4640         eval {
4641             if ($opt =~ m/^unused/) {
4642                 die "internal error";
4643             } elsif (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4644                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4645             }
4646         };
4647         if (my $err = $@) {
4648             $add_apply_error->($opt, $err);
4649         } else {
4650             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4651             delete $conf->{$opt};
4652         }
4653     }
4654
4655     PVE::QemuConfig->cleanup_pending($conf);
4656
4657     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4658         next if $opt eq 'delete'; # just to be sure
4659         eval {
4660             if (defined($conf->{$opt}) && is_valid_drivename($opt)) {
4661                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4662             }
4663         };
4664         if (my $err = $@) {
4665             $add_apply_error->($opt, $err);
4666         } else {
4667             $conf->{$opt} = delete $conf->{pending}->{$opt};
4668         }
4669     }
4670
4671     # write all changes at once to avoid unnecessary i/o
4672     PVE::QemuConfig->write_config($vmid, $conf);
4673 }
4674
4675 sub vmconfig_update_net {
4676     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4677
4678     my $newnet = parse_net($value);
4679
4680     if ($conf->{$opt}) {
4681         my $oldnet = parse_net($conf->{$opt});
4682
4683         if (safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4684             safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4685             safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4686             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4687
4688             # for non online change, we try to hot-unplug
4689             die "skip\n" if !$hotplug;
4690             vm_deviceunplug($vmid, $conf, $opt);
4691         } else {
4692
4693             die "internal error" if $opt !~ m/net(\d+)/;
4694             my $iface = "tap${vmid}i$1";
4695
4696             if (safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4697                 safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4698                 safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4699                 safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4700                 PVE::Network::tap_unplug($iface);
4701
4702                 if ($have_sdn) {
4703                     PVE::Network::SDN::Zones::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4704                 } else {
4705                     PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4706                 }
4707             } elsif (safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4708                 # Rate can be applied on its own but any change above needs to
4709                 # include the rate in tap_plug since OVS resets everything.
4710                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4711             }
4712
4713             if (safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4714                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4715             }
4716
4717             return 1;
4718         }
4719     }
4720
4721     if ($hotplug) {
4722         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
4723     } else {
4724         die "skip\n";
4725     }
4726 }
4727
4728 sub vmconfig_update_disk {
4729     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
4730
4731     my $drive = parse_drive($opt, $value);
4732
4733     if ($conf->{$opt} && (my $old_drive = parse_drive($opt, $conf->{$opt}))) {
4734         my $media = $drive->{media} || 'disk';
4735         my $oldmedia = $old_drive->{media} || 'disk';
4736         die "unable to change media type\n" if $media ne $oldmedia;
4737
4738         if (!drive_is_cdrom($old_drive)) {
4739
4740             if ($drive->{file} ne $old_drive->{file}) {
4741
4742                 die "skip\n" if !$hotplug;
4743
4744                 # unplug and register as unused
4745                 vm_deviceunplug($vmid, $conf, $opt);
4746                 vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4747
4748             } else {
4749                 # update existing disk
4750
4751                 # skip non hotpluggable value
4752                 if (safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4753                     safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4754                     safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4755                     safe_string_ne($drive->{cache}, $old_drive->{cache}) ||
4756                     safe_string_ne($drive->{ssd}, $old_drive->{ssd})) {
4757                     die "skip\n";
4758                 }
4759
4760                 # apply throttle
4761                 if (safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4762                     safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4763                     safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4764                     safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4765                     safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4766                     safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4767                     safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4768                     safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4769                     safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4770                     safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4771                     safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4772                     safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4773                     safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4774                     safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4775                     safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4776                     safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4777                     safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4778                     safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4779
4780                     qemu_block_set_io_throttle(
4781                         $vmid,"drive-$opt",
4782                         ($drive->{mbps} || 0)*1024*1024,
4783                         ($drive->{mbps_rd} || 0)*1024*1024,
4784                         ($drive->{mbps_wr} || 0)*1024*1024,
4785                          $drive->{iops} || 0,
4786                          $drive->{iops_rd} || 0,
4787                          $drive->{iops_wr} || 0,
4788                         ($drive->{mbps_max} || 0)*1024*1024,
4789                         ($drive->{mbps_rd_max} || 0)*1024*1024,
4790                         ($drive->{mbps_wr_max} || 0)*1024*1024,
4791                          $drive->{iops_max} || 0,
4792                          $drive->{iops_rd_max} || 0,
4793                          $drive->{iops_wr_max} || 0,
4794                          $drive->{bps_max_length} || 1,
4795                          $drive->{bps_rd_max_length} || 1,
4796                          $drive->{bps_wr_max_length} || 1,
4797                          $drive->{iops_max_length} || 1,
4798                          $drive->{iops_rd_max_length} || 1,
4799                          $drive->{iops_wr_max_length} || 1,
4800                     );
4801
4802                 }
4803
4804                 return 1;
4805             }
4806
4807         } else { # cdrom
4808
4809             if ($drive->{file} eq 'none') {
4810                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4811                 if (drive_is_cloudinit($old_drive)) {
4812                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4813                 }
4814             } else {
4815                 my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4816
4817                 # force eject if locked
4818                 mon_cmd($vmid, "eject", force => JSON::true, id => "$opt");
4819
4820                 if ($path) {
4821                     mon_cmd($vmid, "blockdev-change-medium",
4822                         id => "$opt", filename => "$path");
4823                 }
4824             }
4825
4826             return 1;
4827         }
4828     }
4829
4830     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4831     # hotplug new disks
4832     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4833     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
4834 }
4835
4836 # called in locked context by incoming migration
4837 sub vm_migrate_get_nbd_disks {
4838     my ($storecfg, $conf, $replicated_volumes) = @_;
4839
4840     my $local_volumes = {};
4841     PVE::QemuConfig->foreach_volume($conf, sub {
4842         my ($ds, $drive) = @_;
4843
4844         return if drive_is_cdrom($drive);
4845
4846         my $volid = $drive->{file};
4847
4848         return if !$volid;
4849
4850         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4851
4852         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4853         return if $scfg->{shared};
4854
4855         # replicated disks re-use existing state via bitmap
4856         my $use_existing = $replicated_volumes->{$volid} ? 1 : 0;
4857         $local_volumes->{$ds} = [$volid, $storeid, $volname, $drive, $use_existing];
4858     });
4859     return $local_volumes;
4860 }
4861
4862 # called in locked context by incoming migration
4863 sub vm_migrate_alloc_nbd_disks {
4864     my ($storecfg, $vmid, $source_volumes, $storagemap) = @_;
4865
4866     my $format = undef;
4867
4868     my $nbd = {};
4869     foreach my $opt (sort keys %$source_volumes) {
4870         my ($volid, $storeid, $volname, $drive, $use_existing) = @{$source_volumes->{$opt}};
4871
4872         if ($use_existing) {
4873             $nbd->{$opt}->{drivestr} = print_drive($drive);
4874             $nbd->{$opt}->{volid} = $volid;
4875             $nbd->{$opt}->{replicated} = 1;
4876             next;
4877         }
4878
4879         # If a remote storage is specified and the format of the original
4880         # volume is not available there, fall back to the default format.
4881         # Otherwise use the same format as the original.
4882         if (!$storagemap->{identity}) {
4883             $storeid = map_storage($storagemap, $storeid);
4884             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4885             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4886             my $fileFormat = qemu_img_format($scfg, $volname);
4887             $format = (grep {$fileFormat eq $_} @{$validFormats}) ? $fileFormat : $defFormat;
4888         } else {
4889             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4890             $format = qemu_img_format($scfg, $volname);
4891         }
4892
4893         my $size = $drive->{size} / 1024;
4894         my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, $size);
4895         my $newdrive = $drive;
4896         $newdrive->{format} = $format;
4897         $newdrive->{file} = $newvolid;
4898         my $drivestr = print_drive($newdrive);
4899         $nbd->{$opt}->{drivestr} = $drivestr;
4900         $nbd->{$opt}->{volid} = $newvolid;
4901     }
4902
4903     return $nbd;
4904 }
4905
4906 # see vm_start_nolock for parameters, additionally:
4907 # migrate_opts:
4908 #   storagemap = parsed storage map for allocating NBD disks
4909 sub vm_start {
4910     my ($storecfg, $vmid, $params, $migrate_opts) = @_;
4911
4912     return PVE::QemuConfig->lock_config($vmid, sub {
4913         my $conf = PVE::QemuConfig->load_config($vmid, $migrate_opts->{migratedfrom});
4914
4915         die "you can't start a vm if it's a template\n"
4916             if !$params->{skiptemplate} && PVE::QemuConfig->is_template($conf);
4917
4918         my $has_suspended_lock = PVE::QemuConfig->has_lock($conf, 'suspended');
4919         my $has_backup_lock = PVE::QemuConfig->has_lock($conf, 'backup');
4920
4921         my $running = check_running($vmid, undef, $migrate_opts->{migratedfrom});
4922
4923         if ($has_backup_lock && $running) {
4924             # a backup is currently running, attempt to start the guest in the
4925             # existing QEMU instance
4926             return vm_resume($vmid);
4927         }
4928
4929         PVE::QemuConfig->check_lock($conf)
4930             if !($params->{skiplock} || $has_suspended_lock);
4931
4932         $params->{resume} = $has_suspended_lock || defined($conf->{vmstate});
4933
4934         die "VM $vmid already running\n" if $running;
4935
4936         if (my $storagemap = $migrate_opts->{storagemap}) {
4937             my $replicated = $migrate_opts->{replicated_volumes};
4938             my $disks = vm_migrate_get_nbd_disks($storecfg, $conf, $replicated);
4939             $migrate_opts->{nbd} = vm_migrate_alloc_nbd_disks($storecfg, $vmid, $disks, $storagemap);
4940
4941             foreach my $opt (keys %{$migrate_opts->{nbd}}) {
4942                 $conf->{$opt} = $migrate_opts->{nbd}->{$opt}->{drivestr};
4943             }
4944         }
4945
4946         return vm_start_nolock($storecfg, $vmid, $conf, $params, $migrate_opts);
4947     });
4948 }
4949
4950
4951 # params:
4952 #   statefile => 'tcp', 'unix' for migration or path/volid for RAM state
4953 #   skiplock => 0/1, skip checking for config lock
4954 #   skiptemplate => 0/1, skip checking whether VM is template
4955 #   forcemachine => to force Qemu machine (rollback/migration)
4956 #   forcecpu => a QEMU '-cpu' argument string to override get_cpu_options
4957 #   timeout => in seconds
4958 #   paused => start VM in paused state (backup)
4959 #   resume => resume from hibernation
4960 # migrate_opts:
4961 #   nbd => volumes for NBD exports (vm_migrate_alloc_nbd_disks)
4962 #   migratedfrom => source node
4963 #   spice_ticket => used for spice migration, passed via tunnel/stdin
4964 #   network => CIDR of migration network
4965 #   type => secure/insecure - tunnel over encrypted connection or plain-text
4966 #   nbd_proto_version => int, 0 for TCP, 1 for UNIX
4967 #   replicated_volumes = which volids should be re-used with bitmaps for nbd migration
4968 sub vm_start_nolock {
4969     my ($storecfg, $vmid, $conf, $params, $migrate_opts) = @_;
4970
4971     my $statefile = $params->{statefile};
4972     my $resume = $params->{resume};
4973
4974     my $migratedfrom = $migrate_opts->{migratedfrom};
4975     my $migration_type = $migrate_opts->{type};
4976
4977     my $res = {};
4978
4979     # clean up leftover reboot request files
4980     eval { clear_reboot_request($vmid); };
4981     warn $@ if $@;
4982
4983     if (!$statefile && scalar(keys %{$conf->{pending}})) {
4984         vmconfig_apply_pending($vmid, $conf, $storecfg);
4985         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4986     }
4987
4988     PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4989
4990     my $defaults = load_defaults();
4991
4992     # set environment variable useful inside network script
4993     $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4994
4995     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
4996
4997     my $forcemachine = $params->{forcemachine};
4998     my $forcecpu = $params->{forcecpu};
4999     if ($resume) {
5000         # enforce machine and CPU type on suspended vm to ensure HW compatibility
5001         $forcemachine = $conf->{runningmachine};
5002         $forcecpu = $conf->{runningcpu};
5003         print "Resuming suspended VM\n";
5004     }
5005
5006     my ($cmd, $vollist, $spice_port) =
5007         config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine, $forcecpu);
5008
5009     my $migration_ip;
5010     my $get_migration_ip = sub {
5011         my ($nodename) = @_;
5012
5013         return $migration_ip if defined($migration_ip);
5014
5015         my $cidr = $migrate_opts->{network};
5016
5017         if (!defined($cidr)) {
5018             my $dc_conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5019             $cidr = $dc_conf->{migration}->{network};
5020         }
5021
5022         if (defined($cidr)) {
5023             my $ips = PVE::Network::get_local_ip_from_cidr($cidr);
5024
5025             die "could not get IP: no address configured on local " .
5026                 "node for network '$cidr'\n" if scalar(@$ips) == 0;
5027
5028             die "could not get IP: multiple addresses configured on local " .
5029                 "node for network '$cidr'\n" if scalar(@$ips) > 1;
5030
5031             $migration_ip = @$ips[0];
5032         }
5033
5034         $migration_ip = PVE::Cluster::remote_node_ip($nodename, 1)
5035             if !defined($migration_ip);
5036
5037         return $migration_ip;
5038     };
5039
5040     my $migrate_uri;
5041     if ($statefile) {
5042         if ($statefile eq 'tcp') {
5043             my $localip = "localhost";
5044             my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5045             my $nodename = nodename();
5046
5047             if (!defined($migration_type)) {
5048                 if (defined($datacenterconf->{migration}->{type})) {
5049                     $migration_type = $datacenterconf->{migration}->{type};
5050                 } else {
5051                     $migration_type = 'secure';
5052                 }
5053             }
5054
5055             if ($migration_type eq 'insecure') {
5056                 $localip = $get_migration_ip->($nodename);
5057                 $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5058             }
5059
5060             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5061             my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5062             $migrate_uri = "tcp:${localip}:${migrate_port}";
5063             push @$cmd, '-incoming', $migrate_uri;
5064             push @$cmd, '-S';
5065
5066         } elsif ($statefile eq 'unix') {
5067             # should be default for secure migrations as a ssh TCP forward
5068             # tunnel is not deterministic reliable ready and fails regurarly
5069             # to set up in time, so use UNIX socket forwards
5070             my $socket_addr = "/run/qemu-server/$vmid.migrate";
5071             unlink $socket_addr;
5072
5073             $migrate_uri = "unix:$socket_addr";
5074
5075             push @$cmd, '-incoming', $migrate_uri;
5076             push @$cmd, '-S';
5077
5078         } elsif (-e $statefile) {
5079             push @$cmd, '-loadstate', $statefile;
5080         } else {
5081             my $statepath = PVE::Storage::path($storecfg, $statefile);
5082             push @$vollist, $statefile;
5083             push @$cmd, '-loadstate', $statepath;
5084         }
5085     } elsif ($params->{paused}) {
5086         push @$cmd, '-S';
5087     }
5088
5089     # host pci devices
5090     for (my $i = 0; $i < $PVE::QemuServer::PCI::MAX_HOSTPCI_DEVICES; $i++)  {
5091       my $d = parse_hostpci($conf->{"hostpci$i"});
5092       next if !$d;
5093       my $pcidevices = $d->{pciid};
5094       foreach my $pcidevice (@$pcidevices) {
5095             my $pciid = $pcidevice->{id};
5096
5097             my $info = PVE::SysFSTools::pci_device_info("$pciid");
5098             die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5099             die "no pci device info for device '$pciid'\n" if !$info;
5100
5101             if ($d->{mdev}) {
5102                 my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5103                 PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5104             } else {
5105                 die "can't unbind/bind PCI group to VFIO '$pciid'\n"
5106                     if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5107                 die "can't reset PCI device '$pciid'\n"
5108                     if $info->{has_fl_reset} && !PVE::SysFSTools::pci_dev_reset($info);
5109             }
5110       }
5111     }
5112
5113     PVE::Storage::activate_volumes($storecfg, $vollist);
5114
5115     eval {
5116         run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5117             outfunc => sub {}, errfunc => sub {});
5118     };
5119     # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5120     # timeout should be more than enough here...
5121     PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5122
5123     my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5124                                               : $defaults->{cpuunits};
5125
5126     my $start_timeout = $params->{timeout} // config_aware_timeout($conf, $resume);
5127     my %run_params = (
5128         timeout => $statefile ? undef : $start_timeout,
5129         umask => 0077,
5130         noerr => 1,
5131     );
5132
5133     # when migrating, prefix QEMU output so other side can pick up any
5134     # errors that might occur and show the user
5135     if ($migratedfrom) {
5136         $run_params{quiet} = 1;
5137         $run_params{logfunc} = sub { print "QEMU: $_[0]\n" };
5138     }
5139
5140     my %properties = (
5141         Slice => 'qemu.slice',
5142         KillMode => 'none'
5143     );
5144
5145     if (PVE::CGroup::cgroup_mode() == 2) {
5146         $properties{CPUWeight} = $cpuunits;
5147     } else {
5148         $properties{CPUShares} = $cpuunits;
5149     }
5150
5151     if (my $cpulimit = $conf->{cpulimit}) {
5152         $properties{CPUQuota} = int($cpulimit * 100);
5153     }
5154     $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5155
5156     my $run_qemu = sub {
5157         PVE::Tools::run_fork sub {
5158             PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5159
5160             my $exitcode = run_command($cmd, %run_params);
5161             die "QEMU exited with code $exitcode\n" if $exitcode;
5162         };
5163     };
5164
5165     if ($conf->{hugepages}) {
5166
5167         my $code = sub {
5168             my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5169             my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5170
5171             PVE::QemuServer::Memory::hugepages_mount();
5172             PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5173
5174             eval { $run_qemu->() };
5175             if (my $err = $@) {
5176                 PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology)
5177                     if !$conf->{keephugepages};
5178                 die $err;
5179             }
5180
5181             PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology)
5182                 if !$conf->{keephugepages};
5183         };
5184         eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5185
5186     } else {
5187         eval { $run_qemu->() };
5188     }
5189
5190     if (my $err = $@) {
5191         # deactivate volumes if start fails
5192         eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5193         die "start failed: $err";
5194     }
5195
5196     print "migration listens on $migrate_uri\n" if $migrate_uri;
5197     $res->{migrate_uri} = $migrate_uri;
5198
5199     if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5200         eval { mon_cmd($vmid, "cont"); };
5201         warn $@ if $@;
5202     }
5203
5204     #start nbd server for storage migration
5205     if (my $nbd = $migrate_opts->{nbd}) {
5206         my $nbd_protocol_version = $migrate_opts->{nbd_proto_version} // 0;
5207
5208         my $migrate_storage_uri;
5209         # nbd_protocol_version > 0 for unix socket support
5210         if ($nbd_protocol_version > 0 && $migration_type eq 'secure') {
5211             my $socket_path = "/run/qemu-server/$vmid\_nbd.migrate";
5212             mon_cmd($vmid, "nbd-server-start", addr => { type => 'unix', data => { path => $socket_path } } );
5213             $migrate_storage_uri = "nbd:unix:$socket_path";
5214         } else {
5215             my $nodename = nodename();
5216             my $localip = $get_migration_ip->($nodename);
5217             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5218             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5219
5220             mon_cmd($vmid, "nbd-server-start", addr => {
5221                 type => 'inet',
5222                 data => {
5223                     host => "${localip}",
5224                     port => "${storage_migrate_port}",
5225                 },
5226             });
5227             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5228             $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}";
5229         }
5230
5231         $res->{migrate_storage_uri} = $migrate_storage_uri;
5232
5233         foreach my $opt (sort keys %$nbd) {
5234             my $drivestr = $nbd->{$opt}->{drivestr};
5235             my $volid = $nbd->{$opt}->{volid};
5236             mon_cmd($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5237             my $nbd_uri = "$migrate_storage_uri:exportname=drive-$opt";
5238             print "storage migration listens on $nbd_uri volume:$drivestr\n";
5239             print "re-using replicated volume: $opt - $volid\n"
5240                 if $nbd->{$opt}->{replicated};
5241
5242             $res->{drives}->{$opt} = $nbd->{$opt};
5243             $res->{drives}->{$opt}->{nbd_uri} = $nbd_uri;
5244         }
5245     }
5246
5247     if ($migratedfrom) {
5248         eval {
5249             set_migration_caps($vmid);
5250         };
5251         warn $@ if $@;
5252
5253         if ($spice_port) {
5254             print "spice listens on port $spice_port\n";
5255             $res->{spice_port} = $spice_port;
5256             if ($migrate_opts->{spice_ticket}) {
5257                 mon_cmd($vmid, "set_password", protocol => 'spice', password =>
5258                     $migrate_opts->{spice_ticket});
5259                 mon_cmd($vmid, "expire_password", protocol => 'spice', time => "+30");
5260             }
5261         }
5262
5263     } else {
5264         mon_cmd($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5265             if !$statefile && $conf->{balloon};
5266
5267         foreach my $opt (keys %$conf) {
5268             next if $opt !~  m/^net\d+$/;
5269             my $nicconf = parse_net($conf->{$opt});
5270             qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5271         }
5272     }
5273
5274     mon_cmd($vmid, 'qom-set',
5275                 path => "machine/peripheral/balloon0",
5276                 property => "guest-stats-polling-interval",
5277                 value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5278
5279     if ($resume) {
5280         print "Resumed VM, removing state\n";
5281         if (my $vmstate = $conf->{vmstate}) {
5282             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5283             PVE::Storage::vdisk_free($storecfg, $vmstate);
5284         }
5285         delete $conf->@{qw(lock vmstate runningmachine runningcpu)};
5286         PVE::QemuConfig->write_config($vmid, $conf);
5287     }
5288
5289     PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5290
5291     return $res;
5292 }
5293
5294 sub vm_commandline {
5295     my ($storecfg, $vmid, $snapname) = @_;
5296
5297     my $conf = PVE::QemuConfig->load_config($vmid);
5298     my $forcemachine;
5299     my $forcecpu;
5300
5301     if ($snapname) {
5302         my $snapshot = $conf->{snapshots}->{$snapname};
5303         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5304
5305         # check for machine or CPU overrides in snapshot
5306         $forcemachine = $snapshot->{runningmachine};
5307         $forcecpu = $snapshot->{runningcpu};
5308
5309         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5310
5311         $conf = $snapshot;
5312     }
5313
5314     my $defaults = load_defaults();
5315
5316     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults,
5317         $forcemachine, $forcecpu);
5318
5319     return PVE::Tools::cmd2string($cmd);
5320 }
5321
5322 sub vm_reset {
5323     my ($vmid, $skiplock) = @_;
5324
5325     PVE::QemuConfig->lock_config($vmid, sub {
5326
5327         my $conf = PVE::QemuConfig->load_config($vmid);
5328
5329         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5330
5331         mon_cmd($vmid, "system_reset");
5332     });
5333 }
5334
5335 sub get_vm_volumes {
5336     my ($conf) = @_;
5337
5338     my $vollist = [];
5339     foreach_volid($conf, sub {
5340         my ($volid, $attr) = @_;
5341
5342         return if $volid =~ m|^/|;
5343
5344         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5345         return if !$sid;
5346
5347         push @$vollist, $volid;
5348     });
5349
5350     return $vollist;
5351 }
5352
5353 sub vm_stop_cleanup {
5354     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5355
5356     eval {
5357
5358         if (!$keepActive) {
5359             my $vollist = get_vm_volumes($conf);
5360             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5361         }
5362
5363         foreach my $ext (qw(mon qmp pid vnc qga)) {
5364             unlink "/var/run/qemu-server/${vmid}.$ext";
5365         }
5366
5367         if ($conf->{ivshmem}) {
5368             my $ivshmem = parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5369             # just delete it for now, VMs which have this already open do not
5370             # are affected, but new VMs will get a separated one. If this
5371             # becomes an issue we either add some sort of ref-counting or just
5372             # add a "don't delete on stop" flag to the ivshmem format.
5373             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5374         }
5375
5376         foreach my $key (keys %$conf) {
5377             next if $key !~ m/^hostpci(\d+)$/;
5378             my $hostpciindex = $1;
5379             my $d = parse_hostpci($conf->{$key});
5380             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5381
5382             foreach my $pci (@{$d->{pciid}}) {
5383                 my $pciid = $pci->{id};
5384                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5385             }
5386         }
5387
5388         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5389     };
5390     warn $@ if $@; # avoid errors - just warn
5391 }
5392
5393 # call only in locked context
5394 sub _do_vm_stop {
5395     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5396
5397     my $pid = check_running($vmid, $nocheck);
5398     return if !$pid;
5399
5400     my $conf;
5401     if (!$nocheck) {
5402         $conf = PVE::QemuConfig->load_config($vmid);
5403         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5404         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5405             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5406             $timeout = $opts->{down} if $opts->{down};
5407         }
5408         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5409     }
5410
5411     eval {
5412         if ($shutdown) {
5413             if (defined($conf) && get_qga_key($conf, 'enabled')) {
5414                 mon_cmd($vmid, "guest-shutdown", timeout => $timeout);
5415             } else {
5416                 mon_cmd($vmid, "system_powerdown");
5417             }
5418         } else {
5419             mon_cmd($vmid, "quit");
5420         }
5421     };
5422     my $err = $@;
5423
5424     if (!$err) {
5425         $timeout = 60 if !defined($timeout);
5426
5427         my $count = 0;
5428         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5429             $count++;
5430             sleep 1;
5431         }
5432
5433         if ($count >= $timeout) {
5434             if ($force) {
5435                 warn "VM still running - terminating now with SIGTERM\n";
5436                 kill 15, $pid;
5437             } else {
5438                 die "VM quit/powerdown failed - got timeout\n";
5439             }
5440         } else {
5441             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5442             return;
5443         }
5444     } else {
5445        if (!check_running($vmid, $nocheck)) {
5446             warn "Unexpected: VM shutdown command failed, but VM not running anymore..\n";
5447             return;
5448        }
5449        if ($force) {
5450             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5451             kill 15, $pid;
5452         } else {
5453             die "VM quit/powerdown failed\n";
5454         }
5455     }
5456
5457     # wait again
5458     $timeout = 10;
5459
5460     my $count = 0;
5461     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5462         $count++;
5463         sleep 1;
5464     }
5465
5466     if ($count >= $timeout) {
5467         warn "VM still running - terminating now with SIGKILL\n";
5468         kill 9, $pid;
5469         sleep 1;
5470     }
5471
5472     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5473 }
5474
5475 # Note: use $nocheck to skip tests if VM configuration file exists.
5476 # We need that when migration VMs to other nodes (files already moved)
5477 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5478 sub vm_stop {
5479     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5480
5481     $force = 1 if !defined($force) && !$shutdown;
5482
5483     if ($migratedfrom){
5484         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5485         kill 15, $pid if $pid;
5486         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5487         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5488         return;
5489     }
5490
5491     PVE::QemuConfig->lock_config($vmid, sub {
5492         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5493    });
5494 }
5495
5496 sub vm_reboot {
5497     my ($vmid, $timeout) = @_;
5498
5499     PVE::QemuConfig->lock_config($vmid, sub {
5500         eval {
5501
5502             # only reboot if running, as qmeventd starts it again on a stop event
5503             return if !check_running($vmid);
5504
5505             create_reboot_request($vmid);
5506
5507             my $storecfg = PVE::Storage::config();
5508             _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5509
5510         };
5511         if (my $err = $@) {
5512             # avoid that the next normal shutdown will be confused for a reboot
5513             clear_reboot_request($vmid);
5514             die $err;
5515         }
5516    });
5517 }
5518
5519 # note: if using the statestorage parameter, the caller has to check privileges
5520 sub vm_suspend {
5521     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5522
5523     my $conf;
5524     my $path;
5525     my $storecfg;
5526     my $vmstate;
5527
5528     PVE::QemuConfig->lock_config($vmid, sub {
5529
5530         $conf = PVE::QemuConfig->load_config($vmid);
5531
5532         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5533         PVE::QemuConfig->check_lock($conf)
5534             if !($skiplock || $is_backing_up);
5535
5536         die "cannot suspend to disk during backup\n"
5537             if $is_backing_up && $includestate;
5538
5539         if ($includestate) {
5540             $conf->{lock} = 'suspending';
5541             my $date = strftime("%Y-%m-%d", localtime(time()));
5542             $storecfg = PVE::Storage::config();
5543             if (!$statestorage) {
5544                 $statestorage = find_vmstate_storage($conf, $storecfg);
5545                 # check permissions for the storage
5546                 my $rpcenv = PVE::RPCEnvironment::get();
5547                 if ($rpcenv->{type} ne 'cli') {
5548                     my $authuser = $rpcenv->get_user();
5549                     $rpcenv->check($authuser, "/storage/$statestorage", ['Datastore.AllocateSpace']);
5550                 }
5551             }
5552
5553
5554             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate(
5555                 $vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5556             $path = PVE::Storage::path($storecfg, $vmstate);
5557             PVE::QemuConfig->write_config($vmid, $conf);
5558         } else {
5559             mon_cmd($vmid, "stop");
5560         }
5561     });
5562
5563     if ($includestate) {
5564         # save vm state
5565         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5566
5567         eval {
5568             mon_cmd($vmid, "savevm-start", statefile => $path);
5569             for(;;) {
5570                 my $state = mon_cmd($vmid, "query-savevm");
5571                 if (!$state->{status}) {
5572                     die "savevm not active\n";
5573                 } elsif ($state->{status} eq 'active') {
5574                     sleep(1);
5575                     next;
5576                 } elsif ($state->{status} eq 'completed') {
5577                     print "State saved, quitting\n";
5578                     last;
5579                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5580                     die "query-savevm failed with error '$state->{error}'\n"
5581                 } else {
5582                     die "query-savevm returned status '$state->{status}'\n";
5583                 }
5584             }
5585         };
5586         my $err = $@;
5587
5588         PVE::QemuConfig->lock_config($vmid, sub {
5589             $conf = PVE::QemuConfig->load_config($vmid);
5590             if ($err) {
5591                 # cleanup, but leave suspending lock, to indicate something went wrong
5592                 eval {
5593                     mon_cmd($vmid, "savevm-end");
5594                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5595                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5596                     delete $conf->@{qw(vmstate runningmachine runningcpu)};
5597                     PVE::QemuConfig->write_config($vmid, $conf);
5598                 };
5599                 warn $@ if $@;
5600                 die $err;
5601             }
5602
5603             die "lock changed unexpectedly\n"
5604                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5605
5606             mon_cmd($vmid, "quit");
5607             $conf->{lock} = 'suspended';
5608             PVE::QemuConfig->write_config($vmid, $conf);
5609         });
5610     }
5611 }
5612
5613 sub vm_resume {
5614     my ($vmid, $skiplock, $nocheck) = @_;
5615
5616     PVE::QemuConfig->lock_config($vmid, sub {
5617         my $res = mon_cmd($vmid, 'query-status');
5618         my $resume_cmd = 'cont';
5619         my $reset = 0;
5620
5621         if ($res->{status}) {
5622             return if $res->{status} eq 'running'; # job done, go home
5623             $resume_cmd = 'system_wakeup' if $res->{status} eq 'suspended';
5624             $reset = 1 if $res->{status} eq 'shutdown';
5625         }
5626
5627         if (!$nocheck) {
5628
5629             my $conf = PVE::QemuConfig->load_config($vmid);
5630
5631             PVE::QemuConfig->check_lock($conf)
5632                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5633         }
5634
5635         if ($reset) {
5636             # required if a VM shuts down during a backup and we get a resume
5637             # request before the backup finishes for example
5638             mon_cmd($vmid, "system_reset");
5639         }
5640         mon_cmd($vmid, $resume_cmd);
5641     });
5642 }
5643
5644 sub vm_sendkey {
5645     my ($vmid, $skiplock, $key) = @_;
5646
5647     PVE::QemuConfig->lock_config($vmid, sub {
5648
5649         my $conf = PVE::QemuConfig->load_config($vmid);
5650
5651         # there is no qmp command, so we use the human monitor command
5652         my $res = PVE::QemuServer::Monitor::hmp_cmd($vmid, "sendkey $key");
5653         die $res if $res ne '';
5654     });
5655 }
5656
5657 # vzdump restore implementaion
5658
5659 sub tar_archive_read_firstfile {
5660     my $archive = shift;
5661
5662     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5663
5664     # try to detect archive type first
5665     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5666         die "unable to open file '$archive'\n";
5667     my $firstfile = <$fh>;
5668     kill 15, $pid;
5669     close $fh;
5670
5671     die "ERROR: archive contaions no data\n" if !$firstfile;
5672     chomp $firstfile;
5673
5674     return $firstfile;
5675 }
5676
5677 sub tar_restore_cleanup {
5678     my ($storecfg, $statfile) = @_;
5679
5680     print STDERR "starting cleanup\n";
5681
5682     if (my $fd = IO::File->new($statfile, "r")) {
5683         while (defined(my $line = <$fd>)) {
5684             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5685                 my $volid = $2;
5686                 eval {
5687                     if ($volid =~ m|^/|) {
5688                         unlink $volid || die 'unlink failed\n';
5689                     } else {
5690                         PVE::Storage::vdisk_free($storecfg, $volid);
5691                     }
5692                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5693                 };
5694                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5695             } else {
5696                 print STDERR "unable to parse line in statfile - $line";
5697             }
5698         }
5699         $fd->close();
5700     }
5701 }
5702
5703 sub restore_file_archive {
5704     my ($archive, $vmid, $user, $opts) = @_;
5705
5706     return restore_vma_archive($archive, $vmid, $user, $opts)
5707         if $archive eq '-';
5708
5709     my $info = PVE::Storage::archive_info($archive);
5710     my $format = $opts->{format} // $info->{format};
5711     my $comp = $info->{compression};
5712
5713     # try to detect archive format
5714     if ($format eq 'tar') {
5715         return restore_tar_archive($archive, $vmid, $user, $opts);
5716     } else {
5717         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5718     }
5719 }
5720
5721 # hepler to remove disks that will not be used after restore
5722 my $restore_cleanup_oldconf = sub {
5723     my ($storecfg, $vmid, $oldconf, $virtdev_hash) = @_;
5724
5725     PVE::QemuConfig->foreach_volume($oldconf, sub {
5726         my ($ds, $drive) = @_;
5727
5728         return if drive_is_cdrom($drive, 1);
5729
5730         my $volid = $drive->{file};
5731         return if !$volid || $volid =~ m|^/|;
5732
5733         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
5734         return if !$path || !$owner || ($owner != $vmid);
5735
5736         # Note: only delete disk we want to restore
5737         # other volumes will become unused
5738         if ($virtdev_hash->{$ds}) {
5739             eval { PVE::Storage::vdisk_free($storecfg, $volid); };
5740             if (my $err = $@) {
5741                 warn $err;
5742             }
5743         }
5744     });
5745
5746     # delete vmstate files, after the restore we have no snapshots anymore
5747     foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5748         my $snap = $oldconf->{snapshots}->{$snapname};
5749         if ($snap->{vmstate}) {
5750             eval { PVE::Storage::vdisk_free($storecfg, $snap->{vmstate}); };
5751             if (my $err = $@) {
5752                 warn $err;
5753             }
5754         }
5755     }
5756 };
5757
5758 # Helper to parse vzdump backup device hints
5759 #
5760 # $rpcenv: Environment, used to ckeck storage permissions
5761 # $user: User ID, to check storage permissions
5762 # $storecfg: Storage configuration
5763 # $fh: the file handle for reading the configuration
5764 # $devinfo: should contain device sizes for all backu-up'ed devices
5765 # $options: backup options (pool, default storage)
5766 #
5767 # Return: $virtdev_hash, updates $devinfo (add devname, virtdev, format, storeid)
5768 my $parse_backup_hints = sub {
5769     my ($rpcenv, $user, $storecfg, $fh, $devinfo, $options) = @_;
5770
5771     my $virtdev_hash = {};
5772
5773     while (defined(my $line = <$fh>)) {
5774         if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5775             my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5776             die "archive does not contain data for drive '$virtdev'\n"
5777                 if !$devinfo->{$devname};
5778
5779             if (defined($options->{storage})) {
5780                 $storeid = $options->{storage} || 'local';
5781             } elsif (!$storeid) {
5782                 $storeid = 'local';
5783             }
5784             $format = 'raw' if !$format;
5785             $devinfo->{$devname}->{devname} = $devname;
5786             $devinfo->{$devname}->{virtdev} = $virtdev;
5787             $devinfo->{$devname}->{format} = $format;
5788             $devinfo->{$devname}->{storeid} = $storeid;
5789
5790             # check permission on storage
5791             my $pool = $options->{pool}; # todo: do we need that?
5792             if ($user ne 'root@pam') {
5793                 $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5794             }
5795
5796             $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5797         } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
5798             my $virtdev = $1;
5799             my $drive = parse_drive($virtdev, $2);
5800             if (drive_is_cloudinit($drive)) {
5801                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5802                 $storeid = $options->{storage} if defined ($options->{storage});
5803                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5804                 my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
5805
5806                 $virtdev_hash->{$virtdev} = {
5807                     format => $format,
5808                     storeid => $storeid,
5809                     size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
5810                     is_cloudinit => 1,
5811                 };
5812             }
5813         }
5814     }
5815
5816     return $virtdev_hash;
5817 };
5818
5819 # Helper to allocate and activate all volumes required for a restore
5820 #
5821 # $storecfg: Storage configuration
5822 # $virtdev_hash: as returned by parse_backup_hints()
5823 #
5824 # Returns: { $virtdev => $volid }
5825 my $restore_allocate_devices = sub {
5826     my ($storecfg, $virtdev_hash, $vmid) = @_;
5827
5828     my $map = {};
5829     foreach my $virtdev (sort keys %$virtdev_hash) {
5830         my $d = $virtdev_hash->{$virtdev};
5831         my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5832         my $storeid = $d->{storeid};
5833         my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5834
5835         # test if requested format is supported
5836         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5837         my $supported = grep { $_ eq $d->{format} } @$validFormats;
5838         $d->{format} = $defFormat if !$supported;
5839
5840         my $name;
5841         if ($d->{is_cloudinit}) {
5842             $name = "vm-$vmid-cloudinit";
5843             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5844             if ($scfg->{path}) {
5845                 $name .= ".$d->{format}";
5846             }
5847         }
5848
5849         my $volid = PVE::Storage::vdisk_alloc(
5850             $storecfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
5851
5852         print STDERR "new volume ID is '$volid'\n";
5853         $d->{volid} = $volid;
5854
5855         PVE::Storage::activate_volumes($storecfg, [$volid]);
5856
5857         $map->{$virtdev} = $volid;
5858     }
5859
5860     return $map;
5861 };
5862
5863 my $restore_update_config_line = sub {
5864     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5865
5866     return if $line =~ m/^\#qmdump\#/;
5867     return if $line =~ m/^\#vzdump\#/;
5868     return if $line =~ m/^lock:/;
5869     return if $line =~ m/^unused\d+:/;
5870     return if $line =~ m/^parent:/;
5871
5872     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5873     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5874         # try to convert old 1.X settings
5875         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5876         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5877             my ($model, $macaddr) = split(/\=/, $devconfig);
5878             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5879             my $net = {
5880                 model => $model,
5881                 bridge => "vmbr$ind",
5882                 macaddr => $macaddr,
5883             };
5884             my $netstr = print_net($net);
5885
5886             print $outfd "net$cookie->{netcount}: $netstr\n";
5887             $cookie->{netcount}++;
5888         }
5889     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5890         my ($id, $netstr) = ($1, $2);
5891         my $net = parse_net($netstr);
5892         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5893         $netstr = print_net($net);
5894         print $outfd "$id: $netstr\n";
5895     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5896         my $virtdev = $1;
5897         my $value = $3;
5898         my $di = parse_drive($virtdev, $value);
5899         if (defined($di->{backup}) && !$di->{backup}) {
5900             print $outfd "#$line";
5901         } elsif ($map->{$virtdev}) {
5902             delete $di->{format}; # format can change on restore
5903             $di->{file} = $map->{$virtdev};
5904             $value = print_drive($di);
5905             print $outfd "$virtdev: $value\n";
5906         } else {
5907             print $outfd $line;
5908         }
5909     } elsif (($line =~ m/^vmgenid: (.*)/)) {
5910         my $vmgenid = $1;
5911         if ($vmgenid ne '0') {
5912             # always generate a new vmgenid if there was a valid one setup
5913             $vmgenid = generate_uuid();
5914         }
5915         print $outfd "vmgenid: $vmgenid\n";
5916     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5917         my ($uuid, $uuid_str);
5918         UUID::generate($uuid);
5919         UUID::unparse($uuid, $uuid_str);
5920         my $smbios1 = parse_smbios1($2);
5921         $smbios1->{uuid} = $uuid_str;
5922         print $outfd $1.print_smbios1($smbios1)."\n";
5923     } else {
5924         print $outfd $line;
5925     }
5926 };
5927
5928 my $restore_deactivate_volumes = sub {
5929     my ($storecfg, $devinfo) = @_;
5930
5931     my $vollist = [];
5932     foreach my $devname (keys %$devinfo) {
5933         my $volid = $devinfo->{$devname}->{volid};
5934         push @$vollist, $volid if $volid;
5935     }
5936
5937     PVE::Storage::deactivate_volumes($storecfg, $vollist);
5938 };
5939
5940 my $restore_destroy_volumes = sub {
5941     my ($storecfg, $devinfo) = @_;
5942
5943     foreach my $devname (keys %$devinfo) {
5944         my $volid = $devinfo->{$devname}->{volid};
5945         next if !$volid;
5946         eval {
5947             if ($volid =~ m|^/|) {
5948                 unlink $volid || die 'unlink failed\n';
5949             } else {
5950                 PVE::Storage::vdisk_free($storecfg, $volid);
5951             }
5952             print STDERR "temporary volume '$volid' sucessfuly removed\n";
5953         };
5954         print STDERR "unable to cleanup '$volid' - $@" if $@;
5955     }
5956 };
5957
5958 sub scan_volids {
5959     my ($cfg, $vmid) = @_;
5960
5961     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5962
5963     my $volid_hash = {};
5964     foreach my $storeid (keys %$info) {
5965         foreach my $item (@{$info->{$storeid}}) {
5966             next if !($item->{volid} && $item->{size});
5967             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5968             $volid_hash->{$item->{volid}} = $item;
5969         }
5970     }
5971
5972     return $volid_hash;
5973 }
5974
5975 sub update_disk_config {
5976     my ($vmid, $conf, $volid_hash) = @_;
5977
5978     my $changes;
5979     my $prefix = "VM $vmid";
5980
5981     # used and unused disks
5982     my $referenced = {};
5983
5984     # Note: it is allowed to define multiple storages with same path (alias), so
5985     # we need to check both 'volid' and real 'path' (two different volid can point
5986     # to the same path).
5987
5988     my $referencedpath = {};
5989
5990     # update size info
5991     PVE::QemuConfig->foreach_volume($conf, sub {
5992         my ($opt, $drive) = @_;
5993
5994         my $volid = $drive->{file};
5995         return if !$volid;
5996         my $volume = $volid_hash->{$volid};
5997
5998         # mark volid as "in-use" for next step
5999         $referenced->{$volid} = 1;
6000         if ($volume && (my $path = $volume->{path})) {
6001             $referencedpath->{$path} = 1;
6002         }
6003
6004         return if drive_is_cdrom($drive);
6005         return if !$volume;
6006
6007         my ($updated, $msg) = PVE::QemuServer::Drive::update_disksize($drive, $volume->{size});
6008         if (defined($updated)) {
6009             $changes = 1;
6010             $conf->{$opt} = print_drive($updated);
6011             print "$prefix ($opt): $msg\n";
6012         }
6013     });
6014
6015     # remove 'unusedX' entry if volume is used
6016     PVE::QemuConfig->foreach_unused_volume($conf, sub {
6017         my ($opt, $drive) = @_;
6018
6019         my $volid = $drive->{file};
6020         return if !$volid;
6021
6022         my $path;
6023         $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6024         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6025             print "$prefix remove entry '$opt', its volume '$volid' is in use\n";
6026             $changes = 1;
6027             delete $conf->{$opt};
6028         }
6029
6030         $referenced->{$volid} = 1;
6031         $referencedpath->{$path} = 1 if $path;
6032     });
6033
6034     foreach my $volid (sort keys %$volid_hash) {
6035         next if $volid =~ m/vm-$vmid-state-/;
6036         next if $referenced->{$volid};
6037         my $path = $volid_hash->{$volid}->{path};
6038         next if !$path; # just to be sure
6039         next if $referencedpath->{$path};
6040         $changes = 1;
6041         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6042         print "$prefix add unreferenced volume '$volid' as '$key' to config\n";
6043         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6044     }
6045
6046     return $changes;
6047 }
6048
6049 sub rescan {
6050     my ($vmid, $nolock, $dryrun) = @_;
6051
6052     my $cfg = PVE::Storage::config();
6053
6054     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6055     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6056     foreach my $stor (keys %{$cfg->{ids}}) {
6057         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6058     }
6059
6060     print "rescan volumes...\n";
6061     my $volid_hash = scan_volids($cfg, $vmid);
6062
6063     my $updatefn =  sub {
6064         my ($vmid) = @_;
6065
6066         my $conf = PVE::QemuConfig->load_config($vmid);
6067
6068         PVE::QemuConfig->check_lock($conf);
6069
6070         my $vm_volids = {};
6071         foreach my $volid (keys %$volid_hash) {
6072             my $info = $volid_hash->{$volid};
6073             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6074         }
6075
6076         my $changes = update_disk_config($vmid, $conf, $vm_volids);
6077
6078         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6079     };
6080
6081     if (defined($vmid)) {
6082         if ($nolock) {
6083             &$updatefn($vmid);
6084         } else {
6085             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6086         }
6087     } else {
6088         my $vmlist = config_list();
6089         foreach my $vmid (keys %$vmlist) {
6090             if ($nolock) {
6091                 &$updatefn($vmid);
6092             } else {
6093                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6094             }
6095         }
6096     }
6097 }
6098
6099 sub restore_proxmox_backup_archive {
6100     my ($archive, $vmid, $user, $options) = @_;
6101
6102     my $storecfg = PVE::Storage::config();
6103
6104     my ($storeid, $volname) = PVE::Storage::parse_volume_id($archive);
6105     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6106
6107     my $fingerprint = $scfg->{fingerprint};
6108     my $keyfile = PVE::Storage::PBSPlugin::pbs_encryption_key_file_name($storecfg, $storeid);
6109
6110     my $repo = PVE::PBSClient::get_repository($scfg);
6111
6112     # This is only used for `pbs-restore`!
6113     my $password = PVE::Storage::PBSPlugin::pbs_get_password($scfg, $storeid);
6114     local $ENV{PBS_PASSWORD} = $password;
6115     local $ENV{PBS_FINGERPRINT} = $fingerprint if defined($fingerprint);
6116
6117     my ($vtype, $pbs_backup_name, undef, undef, undef, undef, $format) =
6118         PVE::Storage::parse_volname($storecfg, $archive);
6119
6120     die "got unexpected vtype '$vtype'\n" if $vtype ne 'backup';
6121
6122     die "got unexpected backup format '$format'\n" if $format ne 'pbs-vm';
6123
6124     my $tmpdir = "/var/tmp/vzdumptmp$$";
6125     rmtree $tmpdir;
6126     mkpath $tmpdir;
6127
6128     my $conffile = PVE::QemuConfig->config_file($vmid);
6129     my $tmpfn = "$conffile.$$.tmp";
6130      # disable interrupts (always do cleanups)
6131     local $SIG{INT} =
6132         local $SIG{TERM} =
6133         local $SIG{QUIT} =
6134         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6135
6136     # Note: $oldconf is undef if VM does not exists
6137     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6138     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6139
6140     my $rpcenv = PVE::RPCEnvironment::get();
6141     my $devinfo = {};
6142
6143     eval {
6144         # enable interrupts
6145         local $SIG{INT} =
6146             local $SIG{TERM} =
6147             local $SIG{QUIT} =
6148             local $SIG{HUP} =
6149             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6150
6151         my $cfgfn = "$tmpdir/qemu-server.conf";
6152         my $firewall_config_fn = "$tmpdir/fw.conf";
6153         my $index_fn = "$tmpdir/index.json";
6154
6155         my $cmd = "restore";
6156
6157         my $param = [$pbs_backup_name, "index.json", $index_fn];
6158         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6159         my $index = PVE::Tools::file_get_contents($index_fn);
6160         $index = decode_json($index);
6161
6162         # print Dumper($index);
6163         foreach my $info (@{$index->{files}}) {
6164             if ($info->{filename} =~ m/^(drive-\S+).img.fidx$/) {
6165                 my $devname = $1;
6166                 if ($info->{size} =~ m/^(\d+)$/) { # untaint size
6167                     $devinfo->{$devname}->{size} = $1;
6168                 } else {
6169                     die "unable to parse file size in 'index.json' - got '$info->{size}'\n";
6170                 }
6171             }
6172         }
6173
6174         my $is_qemu_server_backup = scalar(
6175             grep { $_->{filename} eq 'qemu-server.conf.blob' } @{$index->{files}}
6176         );
6177         if (!$is_qemu_server_backup) {
6178             die "backup does not look like a qemu-server backup (missing 'qemu-server.conf' file)\n";
6179         }
6180         my $has_firewall_config = scalar(grep { $_->{filename} eq 'fw.conf.blob' } @{$index->{files}});
6181
6182         $param = [$pbs_backup_name, "qemu-server.conf", $cfgfn];
6183         PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6184
6185         if ($has_firewall_config) {
6186             $param = [$pbs_backup_name, "fw.conf", $firewall_config_fn];
6187             PVE::Storage::PBSPlugin::run_raw_client_cmd($scfg, $storeid, $cmd, $param);
6188
6189             my $pve_firewall_dir = '/etc/pve/firewall';
6190             mkdir $pve_firewall_dir; # make sure the dir exists
6191             PVE::Tools::file_copy($firewall_config_fn, "${pve_firewall_dir}/$vmid.fw");
6192         }
6193
6194         my $fh = IO::File->new($cfgfn, "r") ||
6195             die "unable to read qemu-server.conf - $!\n";
6196
6197         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $storecfg, $fh, $devinfo, $options);
6198
6199         # fixme: rate limit?
6200
6201         # create empty/temp config
6202         PVE::Tools::file_set_contents($conffile, "memory: 128\nlock: create");
6203
6204         $restore_cleanup_oldconf->($storecfg, $vmid, $oldconf, $virtdev_hash) if $oldconf;
6205
6206         # allocate volumes
6207         my $map = $restore_allocate_devices->($storecfg, $virtdev_hash, $vmid);
6208
6209         foreach my $virtdev (sort keys %$virtdev_hash) {
6210             my $d = $virtdev_hash->{$virtdev};
6211             next if $d->{is_cloudinit}; # no need to restore cloudinit
6212
6213             my $volid = $d->{volid};
6214
6215             my $path = PVE::Storage::path($storecfg, $volid);
6216
6217             # This is the ONLY user of the PBS_ env vars set on top of this function!
6218             my $pbs_restore_cmd = [
6219                 '/usr/bin/pbs-restore',
6220                 '--repository', $repo,
6221                 $pbs_backup_name,
6222                 "$d->{devname}.img.fidx",
6223                 $path,
6224                 '--verbose',
6225                 ];
6226
6227             push @$pbs_restore_cmd, '--format', $d->{format} if $d->{format};
6228             push @$pbs_restore_cmd, '--keyfile', $keyfile if -e $keyfile;
6229
6230             if (PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $volid)) {
6231                 push @$pbs_restore_cmd, '--skip-zero';
6232             }
6233
6234             my $dbg_cmdstring = PVE::Tools::cmd2string($pbs_restore_cmd);
6235             print "restore proxmox backup image: $dbg_cmdstring\n";
6236             run_command($pbs_restore_cmd);
6237         }
6238
6239         $fh->seek(0, 0) || die "seek failed - $!\n";
6240
6241         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6242
6243         my $cookie = { netcount => 0 };
6244         while (defined(my $line = <$fh>)) {
6245             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $options->{unique});
6246         }
6247
6248         $fh->close();
6249         $outfd->close();
6250     };
6251     my $err = $@;
6252
6253     $restore_deactivate_volumes->($storecfg, $devinfo);
6254
6255     rmtree $tmpdir;
6256
6257     if ($err) {
6258         unlink $tmpfn;
6259         $restore_destroy_volumes->($storecfg, $devinfo);
6260         die $err;
6261     }
6262
6263     rename($tmpfn, $conffile) ||
6264         die "unable to commit configuration file '$conffile'\n";
6265
6266     PVE::Cluster::cfs_update(); # make sure we read new file
6267
6268     eval { rescan($vmid, 1); };
6269     warn $@ if $@;
6270 }
6271
6272 sub restore_vma_archive {
6273     my ($archive, $vmid, $user, $opts, $comp) = @_;
6274
6275     my $readfrom = $archive;
6276
6277     my $cfg = PVE::Storage::config();
6278     my $commands = [];
6279     my $bwlimit = $opts->{bwlimit};
6280
6281     my $dbg_cmdstring = '';
6282     my $add_pipe = sub {
6283         my ($cmd) = @_;
6284         push @$commands, $cmd;
6285         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6286         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6287         $readfrom = '-';
6288     };
6289
6290     my $input = undef;
6291     if ($archive eq '-') {
6292         $input = '<&STDIN';
6293     } else {
6294         # If we use a backup from a PVE defined storage we also consider that
6295         # storage's rate limit:
6296         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6297         if (defined($volid)) {
6298             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6299             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6300             if ($readlimit) {
6301                 print STDERR "applying read rate limit: $readlimit\n";
6302                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6303                 $add_pipe->($cstream);
6304             }
6305         }
6306     }
6307
6308     if ($comp) {
6309         my $info = PVE::Storage::decompressor_info('vma', $comp);
6310         my $cmd = $info->{decompressor};
6311         push @$cmd, $readfrom;
6312         $add_pipe->($cmd);
6313     }
6314
6315     my $tmpdir = "/var/tmp/vzdumptmp$$";
6316     rmtree $tmpdir;
6317
6318     # disable interrupts (always do cleanups)
6319     local $SIG{INT} =
6320         local $SIG{TERM} =
6321         local $SIG{QUIT} =
6322         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6323
6324     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6325     POSIX::mkfifo($mapfifo, 0600);
6326     my $fifofh;
6327     my $openfifo = sub { open($fifofh, '>', $mapfifo) or die $! };
6328
6329     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6330
6331     my $oldtimeout;
6332     my $timeout = 5;
6333
6334     my $devinfo = {};
6335
6336     my $rpcenv = PVE::RPCEnvironment::get();
6337
6338     my $conffile = PVE::QemuConfig->config_file($vmid);
6339     my $tmpfn = "$conffile.$$.tmp";
6340
6341     # Note: $oldconf is undef if VM does not exist
6342     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6343     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6344
6345     my %storage_limits;
6346
6347     my $print_devmap = sub {
6348         my $cfgfn = "$tmpdir/qemu-server.conf";
6349
6350         # we can read the config - that is already extracted
6351         my $fh = IO::File->new($cfgfn, "r") ||
6352             die "unable to read qemu-server.conf - $!\n";
6353
6354         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6355         if (-f $fwcfgfn) {
6356             my $pve_firewall_dir = '/etc/pve/firewall';
6357             mkdir $pve_firewall_dir; # make sure the dir exists
6358             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6359         }
6360
6361         my $virtdev_hash = $parse_backup_hints->($rpcenv, $user, $cfg, $fh, $devinfo, $opts);
6362
6363         foreach my $key (keys %storage_limits) {
6364             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6365             next if !$limit;
6366             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6367             $storage_limits{$key} = $limit * 1024;
6368         }
6369
6370         foreach my $devname (keys %$devinfo) {
6371             die "found no device mapping information for device '$devname'\n"
6372                 if !$devinfo->{$devname}->{virtdev};
6373         }
6374
6375         # create empty/temp config
6376         if ($oldconf) {
6377             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6378             $restore_cleanup_oldconf->($cfg, $vmid, $oldconf, $virtdev_hash);
6379         }
6380
6381         # allocate volumes
6382         my $map = $restore_allocate_devices->($cfg, $virtdev_hash, $vmid);
6383
6384         # print restore information to $fifofh
6385         foreach my $virtdev (sort keys %$virtdev_hash) {
6386             my $d = $virtdev_hash->{$virtdev};
6387             next if $d->{is_cloudinit}; # no need to restore cloudinit
6388
6389             my $storeid = $d->{storeid};
6390             my $volid = $d->{volid};
6391
6392             my $map_opts = '';
6393             if (my $limit = $storage_limits{$storeid}) {
6394                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6395             }
6396
6397             my $write_zeros = 1;
6398             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6399                 $write_zeros = 0;
6400             }
6401
6402             my $path = PVE::Storage::path($cfg, $volid);
6403
6404             print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6405
6406             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6407         }
6408
6409         $fh->seek(0, 0) || die "seek failed - $!\n";
6410
6411         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6412
6413         my $cookie = { netcount => 0 };
6414         while (defined(my $line = <$fh>)) {
6415             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6416         }
6417
6418         $fh->close();
6419         $outfd->close();
6420     };
6421
6422     eval {
6423         # enable interrupts
6424         local $SIG{INT} =
6425             local $SIG{TERM} =
6426             local $SIG{QUIT} =
6427             local $SIG{HUP} =
6428             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6429         local $SIG{ALRM} = sub { die "got timeout\n"; };
6430
6431         $oldtimeout = alarm($timeout);
6432
6433         my $parser = sub {
6434             my $line = shift;
6435
6436             print "$line\n";
6437
6438             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6439                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6440                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6441             } elsif ($line =~ m/^CTIME: /) {
6442                 # we correctly received the vma config, so we can disable
6443                 # the timeout now for disk allocation (set to 10 minutes, so
6444                 # that we always timeout if something goes wrong)
6445                 alarm(600);
6446                 &$print_devmap();
6447                 print $fifofh "done\n";
6448                 my $tmp = $oldtimeout || 0;
6449                 $oldtimeout = undef;
6450                 alarm($tmp);
6451                 close($fifofh);
6452                 $fifofh = undef;
6453             }
6454         };
6455
6456         print "restore vma archive: $dbg_cmdstring\n";
6457         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6458     };
6459     my $err = $@;
6460
6461     alarm($oldtimeout) if $oldtimeout;
6462
6463     $restore_deactivate_volumes->($cfg, $devinfo);
6464
6465     close($fifofh) if $fifofh;
6466     unlink $mapfifo;
6467     rmtree $tmpdir;
6468
6469     if ($err) {
6470         unlink $tmpfn;
6471         $restore_destroy_volumes->($cfg, $devinfo);
6472         die $err;
6473     }
6474
6475     rename($tmpfn, $conffile) ||
6476         die "unable to commit configuration file '$conffile'\n";
6477
6478     PVE::Cluster::cfs_update(); # make sure we read new file
6479
6480     eval { rescan($vmid, 1); };
6481     warn $@ if $@;
6482 }
6483
6484 sub restore_tar_archive {
6485     my ($archive, $vmid, $user, $opts) = @_;
6486
6487     if ($archive ne '-') {
6488         my $firstfile = tar_archive_read_firstfile($archive);
6489         die "ERROR: file '$archive' does not look like a QemuServer vzdump backup\n"
6490             if $firstfile ne 'qemu-server.conf';
6491     }
6492
6493     my $storecfg = PVE::Storage::config();
6494
6495     # avoid zombie disks when restoring over an existing VM -> cleanup first
6496     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6497     # skiplock=1 because qmrestore has set the 'create' lock itself already
6498     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6499     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6500
6501     my $tocmd = "/usr/lib/qemu-server/qmextract";
6502
6503     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6504     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6505     $tocmd .= ' --prealloc' if $opts->{prealloc};
6506     $tocmd .= ' --info' if $opts->{info};
6507
6508     # tar option "xf" does not autodetect compression when read from STDIN,
6509     # so we pipe to zcat
6510     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6511         PVE::Tools::shellquote("--to-command=$tocmd");
6512
6513     my $tmpdir = "/var/tmp/vzdumptmp$$";
6514     mkpath $tmpdir;
6515
6516     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6517     local $ENV{VZDUMP_VMID} = $vmid;
6518     local $ENV{VZDUMP_USER} = $user;
6519
6520     my $conffile = PVE::QemuConfig->config_file($vmid);
6521     my $tmpfn = "$conffile.$$.tmp";
6522
6523     # disable interrupts (always do cleanups)
6524     local $SIG{INT} =
6525         local $SIG{TERM} =
6526         local $SIG{QUIT} =
6527         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6528
6529     eval {
6530         # enable interrupts
6531         local $SIG{INT} =
6532             local $SIG{TERM} =
6533             local $SIG{QUIT} =
6534             local $SIG{HUP} =
6535             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6536
6537         if ($archive eq '-') {
6538             print "extracting archive from STDIN\n";
6539             run_command($cmd, input => "<&STDIN");
6540         } else {
6541             print "extracting archive '$archive'\n";
6542             run_command($cmd);
6543         }
6544
6545         return if $opts->{info};
6546
6547         # read new mapping
6548         my $map = {};
6549         my $statfile = "$tmpdir/qmrestore.stat";
6550         if (my $fd = IO::File->new($statfile, "r")) {
6551             while (defined (my $line = <$fd>)) {
6552                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6553                     $map->{$1} = $2 if $1;
6554                 } else {
6555                     print STDERR "unable to parse line in statfile - $line\n";
6556                 }
6557             }
6558             $fd->close();
6559         }
6560
6561         my $confsrc = "$tmpdir/qemu-server.conf";
6562
6563         my $srcfd = IO::File->new($confsrc, "r") || die "unable to open file '$confsrc'\n";
6564
6565         my $outfd = IO::File->new($tmpfn, "w") || die "unable to write config for VM $vmid\n";
6566
6567         my $cookie = { netcount => 0 };
6568         while (defined (my $line = <$srcfd>)) {
6569             $restore_update_config_line->($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6570         }
6571
6572         $srcfd->close();
6573         $outfd->close();
6574     };
6575     if (my $err = $@) {
6576         unlink $tmpfn;
6577         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6578         die $err;
6579     }
6580
6581     rmtree $tmpdir;
6582
6583     rename $tmpfn, $conffile ||
6584         die "unable to commit configuration file '$conffile'\n";
6585
6586     PVE::Cluster::cfs_update(); # make sure we read new file
6587
6588     eval { rescan($vmid, 1); };
6589     warn $@ if $@;
6590 };
6591
6592 sub foreach_storage_used_by_vm {
6593     my ($conf, $func) = @_;
6594
6595     my $sidhash = {};
6596
6597     PVE::QemuConfig->foreach_volume($conf, sub {
6598         my ($ds, $drive) = @_;
6599         return if drive_is_cdrom($drive);
6600
6601         my $volid = $drive->{file};
6602
6603         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6604         $sidhash->{$sid} = $sid if $sid;
6605     });
6606
6607     foreach my $sid (sort keys %$sidhash) {
6608         &$func($sid);
6609     }
6610 }
6611
6612 my $qemu_snap_storage = {
6613     rbd => 1,
6614 };
6615 sub do_snapshots_with_qemu {
6616     my ($storecfg, $volid) = @_;
6617
6618     my $storage_name = PVE::Storage::parse_volume_id($volid);
6619     my $scfg = $storecfg->{ids}->{$storage_name};
6620     die "could not find storage '$storage_name'\n" if !defined($scfg);
6621
6622     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6623         return 1;
6624     }
6625
6626     if ($volid =~ m/\.(qcow2|qed)$/){
6627         return 1;
6628     }
6629
6630     return;
6631 }
6632
6633 sub qga_check_running {
6634     my ($vmid, $nowarn) = @_;
6635
6636     eval { mon_cmd($vmid, "guest-ping", timeout => 3); };
6637     if ($@) {
6638         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6639         return 0;
6640     }
6641     return 1;
6642 }
6643
6644 sub template_create {
6645     my ($vmid, $conf, $disk) = @_;
6646
6647     my $storecfg = PVE::Storage::config();
6648
6649     PVE::QemuConfig->foreach_volume($conf, sub {
6650         my ($ds, $drive) = @_;
6651
6652         return if drive_is_cdrom($drive);
6653         return if $disk && $ds ne $disk;
6654
6655         my $volid = $drive->{file};
6656         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6657
6658         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6659         $drive->{file} = $voliddst;
6660         $conf->{$ds} = print_drive($drive);
6661         PVE::QemuConfig->write_config($vmid, $conf);
6662     });
6663 }
6664
6665 sub convert_iscsi_path {
6666     my ($path) = @_;
6667
6668     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6669         my $portal = $1;
6670         my $target = $2;
6671         my $lun = $3;
6672
6673         my $initiator_name = get_initiator_name();
6674
6675         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6676                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6677     }
6678
6679     die "cannot convert iscsi path '$path', unkown format\n";
6680 }
6681
6682 sub qemu_img_convert {
6683     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6684
6685     my $storecfg = PVE::Storage::config();
6686     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6687     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6688
6689     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6690
6691     my $cachemode;
6692     my $src_path;
6693     my $src_is_iscsi = 0;
6694     my $src_format;
6695
6696     if ($src_storeid) {
6697         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6698         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6699         $src_format = qemu_img_format($src_scfg, $src_volname);
6700         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6701         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6702         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6703     } elsif (-f $src_volid) {
6704         $src_path = $src_volid;
6705         if ($src_path =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6706             $src_format = $1;
6707         }
6708     }
6709
6710     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6711
6712     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6713     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6714     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6715     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6716
6717     my $cmd = [];
6718     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6719     push @$cmd, '-l', "snapshot.name=$snapname"
6720         if $snapname && $src_format && $src_format eq "qcow2";
6721     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6722     push @$cmd, '-T', $cachemode if defined($cachemode);
6723
6724     if ($src_is_iscsi) {
6725         push @$cmd, '--image-opts';
6726         $src_path = convert_iscsi_path($src_path);
6727     } elsif ($src_format) {
6728         push @$cmd, '-f', $src_format;
6729     }
6730
6731     if ($dst_is_iscsi) {
6732         push @$cmd, '--target-image-opts';
6733         $dst_path = convert_iscsi_path($dst_path);
6734     } else {
6735         push @$cmd, '-O', $dst_format;
6736     }
6737
6738     push @$cmd, $src_path;
6739
6740     if (!$dst_is_iscsi && $is_zero_initialized) {
6741         push @$cmd, "zeroinit:$dst_path";
6742     } else {
6743         push @$cmd, $dst_path;
6744     }
6745
6746     my $parser = sub {
6747         my $line = shift;
6748         if($line =~ m/\((\S+)\/100\%\)/){
6749             my $percent = $1;
6750             my $transferred = int($size * $percent / 100);
6751             my $remaining = $size - $transferred;
6752
6753             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6754         }
6755
6756     };
6757
6758     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6759     my $err = $@;
6760     die "copy failed: $err" if $err;
6761 }
6762
6763 sub qemu_img_format {
6764     my ($scfg, $volname) = @_;
6765
6766     if ($scfg->{path} && $volname =~ m/\.($PVE::QemuServer::Drive::QEMU_FORMAT_RE)$/) {
6767         return $1;
6768     } else {
6769         return "raw";
6770     }
6771 }
6772
6773 sub qemu_drive_mirror {
6774     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $completion, $qga, $bwlimit, $src_bitmap) = @_;
6775
6776     $jobs = {} if !$jobs;
6777
6778     my $qemu_target;
6779     my $format;
6780     $jobs->{"drive-$drive"} = {};
6781
6782     if ($dst_volid =~ /^nbd:/) {
6783         $qemu_target = $dst_volid;
6784         $format = "nbd";
6785     } else {
6786         my $storecfg = PVE::Storage::config();
6787         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6788
6789         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6790
6791         $format = qemu_img_format($dst_scfg, $dst_volname);
6792
6793         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6794
6795         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6796     }
6797
6798     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6799     $opts->{format} = $format if $format;
6800
6801     if (defined($src_bitmap)) {
6802         $opts->{sync} = 'incremental';
6803         $opts->{bitmap} = $src_bitmap;
6804         print "drive mirror re-using dirty bitmap '$src_bitmap'\n";
6805     }
6806
6807     if (defined($bwlimit)) {
6808         $opts->{speed} = $bwlimit * 1024;
6809         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6810     } else {
6811         print "drive mirror is starting for drive-$drive\n";
6812     }
6813
6814     # if a job already runs for this device we get an error, catch it for cleanup
6815     eval { mon_cmd($vmid, "drive-mirror", %$opts); };
6816     if (my $err = $@) {
6817         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6818         warn "$@\n" if $@;
6819         die "mirroring error: $err\n";
6820     }
6821
6822     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $completion, $qga);
6823 }
6824
6825 # $completion can be either
6826 # 'complete': wait until all jobs are ready, block-job-complete them (default)
6827 # 'cancel': wait until all jobs are ready, block-job-cancel them
6828 # 'skip': wait until all jobs are ready, return with block jobs in ready state
6829 sub qemu_drive_mirror_monitor {
6830     my ($vmid, $vmiddst, $jobs, $completion, $qga) = @_;
6831
6832     $completion //= 'complete';
6833
6834     eval {
6835         my $err_complete = 0;
6836
6837         while (1) {
6838             die "storage migration timed out\n" if $err_complete > 300;
6839
6840             my $stats = mon_cmd($vmid, "query-block-jobs");
6841
6842             my $running_mirror_jobs = {};
6843             foreach my $stat (@$stats) {
6844                 next if $stat->{type} ne 'mirror';
6845                 $running_mirror_jobs->{$stat->{device}} = $stat;
6846             }
6847
6848             my $readycounter = 0;
6849
6850             foreach my $job (keys %$jobs) {
6851
6852                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6853                     print "$job : finished\n";
6854                     delete $jobs->{$job};
6855                     next;
6856                 }
6857
6858                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6859
6860                 my $busy = $running_mirror_jobs->{$job}->{busy};
6861                 my $ready = $running_mirror_jobs->{$job}->{ready};
6862                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6863                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6864                     my $remaining = $total - $transferred;
6865                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6866
6867                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6868                 }
6869
6870                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6871             }
6872
6873             last if scalar(keys %$jobs) == 0;
6874
6875             if ($readycounter == scalar(keys %$jobs)) {
6876                 print "all mirroring jobs are ready \n";
6877                 last if $completion eq 'skip'; #do the complete later
6878
6879                 if ($vmiddst && $vmiddst != $vmid) {
6880                     my $agent_running = $qga && qga_check_running($vmid);
6881                     if ($agent_running) {
6882                         print "freeze filesystem\n";
6883                         eval { mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6884                     } else {
6885                         print "suspend vm\n";
6886                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6887                     }
6888
6889                     # if we clone a disk for a new target vm, we don't switch the disk
6890                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6891
6892                     if ($agent_running) {
6893                         print "unfreeze filesystem\n";
6894                         eval { mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6895                     } else {
6896                         print "resume vm\n";
6897                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6898                     }
6899
6900                     last;
6901                 } else {
6902
6903                     foreach my $job (keys %$jobs) {
6904                         # try to switch the disk if source and destination are on the same guest
6905                         print "$job: Completing block job...\n";
6906
6907                         my $op;
6908                         if ($completion eq 'complete') {
6909                             $op = 'block-job-complete';
6910                         } elsif ($completion eq 'cancel') {
6911                             $op = 'block-job-cancel';
6912                         } else {
6913                             die "invalid completion value: $completion\n";
6914                         }
6915                         eval { mon_cmd($vmid, $op, device => $job) };
6916                         if ($@ =~ m/cannot be completed/) {
6917                             print "$job: Block job cannot be completed, try again.\n";
6918                             $err_complete++;
6919                         }else {
6920                             print "$job: Completed successfully.\n";
6921                             $jobs->{$job}->{complete} = 1;
6922                         }
6923                     }
6924                 }
6925             }
6926             sleep 1;
6927         }
6928     };
6929     my $err = $@;
6930
6931     if ($err) {
6932         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6933         die "mirroring error: $err";
6934     }
6935
6936 }
6937
6938 sub qemu_blockjobs_cancel {
6939     my ($vmid, $jobs) = @_;
6940
6941     foreach my $job (keys %$jobs) {
6942         print "$job: Cancelling block job\n";
6943         eval { mon_cmd($vmid, "block-job-cancel", device => $job); };
6944         $jobs->{$job}->{cancel} = 1;
6945     }
6946
6947     while (1) {
6948         my $stats = mon_cmd($vmid, "query-block-jobs");
6949
6950         my $running_jobs = {};
6951         foreach my $stat (@$stats) {
6952             $running_jobs->{$stat->{device}} = $stat;
6953         }
6954
6955         foreach my $job (keys %$jobs) {
6956
6957             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6958                 print "$job: Done.\n";
6959                 delete $jobs->{$job};
6960             }
6961         }
6962
6963         last if scalar(keys %$jobs) == 0;
6964
6965         sleep 1;
6966     }
6967 }
6968
6969 sub clone_disk {
6970     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6971         $newvmid, $storage, $format, $full, $newvollist, $jobs, $completion, $qga, $bwlimit, $conf) = @_;
6972
6973     my $newvolid;
6974
6975     if (!$full) {
6976         print "create linked clone of drive $drivename ($drive->{file})\n";
6977         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6978         push @$newvollist, $newvolid;
6979     } else {
6980
6981         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6982         $storeid = $storage if $storage;
6983
6984         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6985
6986         print "create full clone of drive $drivename ($drive->{file})\n";
6987         my $name = undef;
6988         my $size = undef;
6989         if (drive_is_cloudinit($drive)) {
6990             $name = "vm-$newvmid-cloudinit";
6991             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6992             if ($scfg->{path}) {
6993                 $name .= ".$dst_format";
6994             }
6995             $snapname = undef;
6996             $size = PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE;
6997         } elsif ($drivename eq 'efidisk0') {
6998             $size = get_efivars_size($conf);
6999         } else {
7000             ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 10);
7001         }
7002         $newvolid = PVE::Storage::vdisk_alloc(
7003             $storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024)
7004         );
7005         push @$newvollist, $newvolid;
7006
7007         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7008
7009         if (drive_is_cloudinit($drive)) {
7010             # when cloning multiple disks (e.g. during clone_vm) it might be the last disk
7011             # if this is the case, we have to complete any block-jobs still there from
7012             # previous drive-mirrors
7013             if (($completion eq 'complete') && (scalar(keys %$jobs) > 0)) {
7014                 qemu_drive_mirror_monitor($vmid, $newvmid, $jobs, $completion, $qga);
7015             }
7016             goto no_data_clone;
7017         }
7018
7019         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7020         if (!$running || $snapname) {
7021             # TODO: handle bwlimits
7022             if ($drivename eq 'efidisk0') {
7023                 # the relevant data on the efidisk may be smaller than the source
7024                 # e.g. on RBD/ZFS, so we use dd to copy only the amount
7025                 # that is given by the OVMF_VARS.fd
7026                 my $src_path = PVE::Storage::path($storecfg, $drive->{file});
7027                 my $dst_path = PVE::Storage::path($storecfg, $newvolid);
7028
7029                 # better for Ceph if block size is not too small, see bug #3324
7030                 my $bs = 1024*1024;
7031
7032                 run_command(['qemu-img', 'dd', '-n', '-O', $dst_format, "bs=$bs", "osize=$size",
7033                     "if=$src_path", "of=$dst_path"]);
7034             } else {
7035                 qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7036             }
7037         } else {
7038
7039             my $kvmver = get_running_qemu_version ($vmid);
7040             if (!min_version($kvmver, 2, 7)) {
7041                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7042                     if $drive->{iothread};
7043             }
7044
7045             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs,
7046                 $completion, $qga, $bwlimit);
7047         }
7048     }
7049
7050 no_data_clone:
7051     my ($size) = eval { PVE::Storage::volume_size_info($storecfg, $newvolid, 10) };
7052
7053     my $disk = $drive;
7054     $disk->{format} = undef;
7055     $disk->{file} = $newvolid;
7056     $disk->{size} = $size if defined($size);
7057
7058     return $disk;
7059 }
7060
7061 sub get_running_qemu_version {
7062     my ($vmid) = @_;
7063     my $res = mon_cmd($vmid, "query-version");
7064     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7065 }
7066
7067 sub qemu_use_old_bios_files {
7068     my ($machine_type) = @_;
7069
7070     return if !$machine_type;
7071
7072     my $use_old_bios_files = undef;
7073
7074     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7075         $machine_type = $1;
7076         $use_old_bios_files = 1;
7077     } else {
7078         my $version = extract_version($machine_type, kvm_user_version());
7079         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7080         # load new efi bios files on migration. So this hack is required to allow
7081         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7082         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7083         $use_old_bios_files = !min_version($version, 2, 4);
7084     }
7085
7086     return ($use_old_bios_files, $machine_type);
7087 }
7088
7089 sub get_efivars_size {
7090     my ($conf) = @_;
7091     my $arch = get_vm_arch($conf);
7092     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7093     die "uefi vars image '$ovmf_vars' not found\n" if ! -f $ovmf_vars;
7094     return -s $ovmf_vars;
7095 }
7096
7097 sub update_efidisk_size {
7098     my ($conf) = @_;
7099
7100     return if !defined($conf->{efidisk0});
7101
7102     my $disk = PVE::QemuServer::parse_drive('efidisk0', $conf->{efidisk0});
7103     $disk->{size} = get_efivars_size($conf);
7104     $conf->{efidisk0} = print_drive($disk);
7105
7106     return;
7107 }
7108
7109 sub create_efidisk($$$$$) {
7110     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7111
7112     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7113     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7114
7115     my $vars_size_b = -s $ovmf_vars;
7116     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7117     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7118     PVE::Storage::activate_volumes($storecfg, [$volid]);
7119
7120     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7121     my ($size) = PVE::Storage::volume_size_info($storecfg, $volid, 3);
7122
7123     return ($volid, $size/1024);
7124 }
7125
7126 sub vm_iothreads_list {
7127     my ($vmid) = @_;
7128
7129     my $res = mon_cmd($vmid, 'query-iothreads');
7130
7131     my $iothreads = {};
7132     foreach my $iothread (@$res) {
7133         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7134     }
7135
7136     return $iothreads;
7137 }
7138
7139 sub scsihw_infos {
7140     my ($conf, $drive) = @_;
7141
7142     my $maxdev = 0;
7143
7144     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7145         $maxdev = 7;
7146     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7147         $maxdev = 1;
7148     } else {
7149         $maxdev = 256;
7150     }
7151
7152     my $controller = int($drive->{index} / $maxdev);
7153     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single')
7154         ? "virtioscsi"
7155         : "scsihw";
7156
7157     return ($maxdev, $controller, $controller_prefix);
7158 }
7159
7160 sub windows_version {
7161     my ($ostype) = @_;
7162
7163     return 0 if !$ostype;
7164
7165     my $winversion = 0;
7166
7167     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7168         $winversion = 5;
7169     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7170         $winversion = 6;
7171     } elsif ($ostype =~ m/^win(\d+)$/) {
7172         $winversion = $1;
7173     }
7174
7175     return $winversion;
7176 }
7177
7178 sub resolve_dst_disk_format {
7179         my ($storecfg, $storeid, $src_volname, $format) = @_;
7180         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7181
7182         if (!$format) {
7183             # if no target format is specified, use the source disk format as hint
7184             if ($src_volname) {
7185                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7186                 $format = qemu_img_format($scfg, $src_volname);
7187             } else {
7188                 return $defFormat;
7189             }
7190         }
7191
7192         # test if requested format is supported - else use default
7193         my $supported = grep { $_ eq $format } @$validFormats;
7194         $format = $defFormat if !$supported;
7195         return $format;
7196 }
7197
7198 # NOTE: if this logic changes, please update docs & possibly gui logic
7199 sub find_vmstate_storage {
7200     my ($conf, $storecfg) = @_;
7201
7202     # first, return storage from conf if set
7203     return $conf->{vmstatestorage} if $conf->{vmstatestorage};
7204
7205     my ($target, $shared, $local);
7206
7207     foreach_storage_used_by_vm($conf, sub {
7208         my ($sid) = @_;
7209         my $scfg = PVE::Storage::storage_config($storecfg, $sid);
7210         my $dst = $scfg->{shared} ? \$shared : \$local;
7211         $$dst = $sid if !$$dst || $scfg->{path}; # prefer file based storage
7212     });
7213
7214     # second, use shared storage where VM has at least one disk
7215     # third, use local storage where VM has at least one disk
7216     # fall back to local storage
7217     $target = $shared // $local // 'local';
7218
7219     return $target;
7220 }
7221
7222 sub generate_uuid {
7223     my ($uuid, $uuid_str);
7224     UUID::generate($uuid);
7225     UUID::unparse($uuid, $uuid_str);
7226     return $uuid_str;
7227 }
7228
7229 sub generate_smbios1_uuid {
7230     return "uuid=".generate_uuid();
7231 }
7232
7233 sub nbd_stop {
7234     my ($vmid) = @_;
7235
7236     mon_cmd($vmid, 'nbd-server-stop');
7237 }
7238
7239 sub create_reboot_request {
7240     my ($vmid) = @_;
7241     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7242         or die "failed to create reboot trigger file: $!\n";
7243     close($fh);
7244 }
7245
7246 sub clear_reboot_request {
7247     my ($vmid) = @_;
7248     my $path = "/run/qemu-server/$vmid.reboot";
7249     my $res = 0;
7250
7251     $res = unlink($path);
7252     die "could not remove reboot request for $vmid: $!"
7253         if !$res && $! != POSIX::ENOENT;
7254
7255     return $res;
7256 }
7257
7258 sub bootorder_from_legacy {
7259     my ($conf, $bootcfg) = @_;
7260
7261     my $boot = $bootcfg->{legacy} || $boot_fmt->{legacy}->{default};
7262     my $bootindex_hash = {};
7263     my $i = 1;
7264     foreach my $o (split(//, $boot)) {
7265         $bootindex_hash->{$o} = $i*100;
7266         $i++;
7267     }
7268
7269     my $bootorder = {};
7270
7271     PVE::QemuConfig->foreach_volume($conf, sub {
7272         my ($ds, $drive) = @_;
7273
7274         if (drive_is_cdrom ($drive, 1)) {
7275             if ($bootindex_hash->{d}) {
7276                 $bootorder->{$ds} = $bootindex_hash->{d};
7277                 $bootindex_hash->{d} += 1;
7278             }
7279         } elsif ($bootindex_hash->{c}) {
7280             $bootorder->{$ds} = $bootindex_hash->{c}
7281                 if $conf->{bootdisk} && $conf->{bootdisk} eq $ds;
7282             $bootindex_hash->{c} += 1;
7283         }
7284     });
7285
7286     if ($bootindex_hash->{n}) {
7287         for (my $i = 0; $i < $MAX_NETS; $i++) {
7288             my $netname = "net$i";
7289             next if !$conf->{$netname};
7290             $bootorder->{$netname} = $bootindex_hash->{n};
7291             $bootindex_hash->{n} += 1;
7292         }
7293     }
7294
7295     return $bootorder;
7296 }
7297
7298 # Generate default device list for 'boot: order=' property. Matches legacy
7299 # default boot order, but with explicit device names. This is important, since
7300 # the fallback for when neither 'order' nor the old format is specified relies
7301 # on 'bootorder_from_legacy' above, and it would be confusing if this diverges.
7302 sub get_default_bootdevices {
7303     my ($conf) = @_;
7304
7305     my @ret = ();
7306
7307     # harddisk
7308     my $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 0);
7309     push @ret, $first if $first;
7310
7311     # cdrom
7312     $first = PVE::QemuServer::Drive::resolve_first_disk($conf, 1);
7313     push @ret, $first if $first;
7314
7315     # network
7316     for (my $i = 0; $i < $MAX_NETS; $i++) {
7317         my $netname = "net$i";
7318         next if !$conf->{$netname};
7319         push @ret, $netname;
7320         last;
7321     }
7322
7323     return \@ret;
7324 }
7325
7326 sub device_bootorder {
7327     my ($conf) = @_;
7328
7329     return bootorder_from_legacy($conf) if !defined($conf->{boot});
7330
7331     my $boot = parse_property_string($boot_fmt, $conf->{boot});
7332
7333     my $bootorder = {};
7334     if (!defined($boot) || $boot->{legacy}) {
7335         $bootorder = bootorder_from_legacy($conf, $boot);
7336     } elsif ($boot->{order}) {
7337         my $i = 100; # start at 100 to allow user to insert devices before us with -args
7338         for my $dev (PVE::Tools::split_list($boot->{order})) {
7339             $bootorder->{$dev} = $i++;
7340         }
7341     }
7342
7343     return $bootorder;
7344 }
7345
7346 # bash completion helper
7347
7348 sub complete_backup_archives {
7349     my ($cmdname, $pname, $cvalue) = @_;
7350
7351     my $cfg = PVE::Storage::config();
7352
7353     my $storeid;
7354
7355     if ($cvalue =~ m/^([^:]+):/) {
7356         $storeid = $1;
7357     }
7358
7359     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7360
7361     my $res = [];
7362     foreach my $id (keys %$data) {
7363         foreach my $item (@{$data->{$id}}) {
7364             next if $item->{format} !~ m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
7365             push @$res, $item->{volid} if defined($item->{volid});
7366         }
7367     }
7368
7369     return $res;
7370 }
7371
7372 my $complete_vmid_full = sub {
7373     my ($running) = @_;
7374
7375     my $idlist = vmstatus();
7376
7377     my $res = [];
7378
7379     foreach my $id (keys %$idlist) {
7380         my $d = $idlist->{$id};
7381         if (defined($running)) {
7382             next if $d->{template};
7383             next if $running && $d->{status} ne 'running';
7384             next if !$running && $d->{status} eq 'running';
7385         }
7386         push @$res, $id;
7387
7388     }
7389     return $res;
7390 };
7391
7392 sub complete_vmid {
7393     return &$complete_vmid_full();
7394 }
7395
7396 sub complete_vmid_stopped {
7397     return &$complete_vmid_full(0);
7398 }
7399
7400 sub complete_vmid_running {
7401     return &$complete_vmid_full(1);
7402 }
7403
7404 sub complete_storage {
7405
7406     my $cfg = PVE::Storage::config();
7407     my $ids = $cfg->{ids};
7408
7409     my $res = [];
7410     foreach my $sid (keys %$ids) {
7411         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7412         next if !$ids->{$sid}->{content}->{images};
7413         push @$res, $sid;
7414     }
7415
7416     return $res;
7417 }
7418
7419 sub complete_migration_storage {
7420     my ($cmd, $param, $current_value, $all_args) = @_;
7421
7422     my $targetnode = @$all_args[1];
7423
7424     my $cfg = PVE::Storage::config();
7425     my $ids = $cfg->{ids};
7426
7427     my $res = [];
7428     foreach my $sid (keys %$ids) {
7429         next if !PVE::Storage::storage_check_enabled($cfg, $sid, $targetnode, 1);
7430         next if !$ids->{$sid}->{content}->{images};
7431         push @$res, $sid;
7432     }
7433
7434     return $res;
7435 }
7436
7437 sub vm_is_paused {
7438     my ($vmid) = @_;
7439     my $qmpstatus = eval {
7440         PVE::QemuConfig::assert_config_exists_on_node($vmid);
7441         mon_cmd($vmid, "query-status");
7442     };
7443     warn "$@\n" if $@;
7444     return $qmpstatus && $qmpstatus->{status} eq "paused";
7445 }
7446
7447 1;