PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my @supported_cpu_flags = (
 171     'pcid',
 172     'spec-ctrl',
 173     'ibpb',
 174     'ssbd',
 175     'virt-ssbd',
 176     'amd-ssbd',
 177     'amd-no-ssb',
 178     'pdpe1gb',
 179     'md-clear',
 180     'hv-tlbflush',
 181     'hv-evmcs',
 182     'aes'
 183 );
 184 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 185
 186 my $cpu_fmt = {
 187     cputype => {
 188         description => "Emulated CPU type.",
 189         type => 'string',
 190         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 191         default => 'kvm64',
 192         default_key => 1,
 193     },
 194     hidden => {
 195         description => "Do not identify as a KVM virtual machine.",
 196         type => 'boolean',
 197         optional => 1,
 198         default => 0
 199     },
 200     'hv-vendor-id' => {
 201         type => 'string',
 202         pattern => qr/[a-zA-Z0-9]{1,12}/,
 203         format_description => 'vendor-id',
 204         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 205         optional => 1,
 206     },
 207     flags => {
 208         description => "List of additional CPU flags separated by ';'."
 209                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 210                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 211         format_description => '+FLAG[;-FLAG...]',
 212         type => 'string',
 213         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 214         optional => 1,
 215     },
 216 };
 217
 218 my $watchdog_fmt = {
 219     model => {
 220         default_key => 1,
 221         type => 'string',
 222         enum => [qw(i6300esb ib700)],
 223         description => "Watchdog type to emulate.",
 224         default => 'i6300esb',
 225         optional => 1,
 226     },
 227     action => {
 228         type => 'string',
 229         enum => [qw(reset shutdown poweroff pause debug none)],
 230         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 231         optional => 1,
 232     },
 233 };
 234 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 235
 236 my $agent_fmt = {
 237     enabled => {
 238         description => "Enable/disable Qemu GuestAgent.",
 239         type => 'boolean',
 240         default => 0,
 241         default_key => 1,
 242     },
 243     fstrim_cloned_disks => {
 244         description => "Run fstrim after cloning/moving a disk.",
 245         type => 'boolean',
 246         optional => 1,
 247         default => 0
 248     },
 249 };
 250
 251 my $vga_fmt = {
 252     type => {
 253         description => "Select the VGA type.",
 254         type => 'string',
 255         default => 'std',
 256         optional => 1,
 257         default_key => 1,
 258         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 259     },
 260     memory => {
 261         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 262         type => 'integer',
 263         optional => 1,
 264         minimum => 4,
 265         maximum => 512,
 266     },
 267 };
 268
 269 my $ivshmem_fmt = {
 270     size => {
 271         type => 'integer',
 272         minimum => 1,
 273         description => "The size of the file in MB.",
 274     },
 275     name => {
 276         type => 'string',
 277         pattern => '[a-zA-Z0-9\-]+',
 278         optional => 1,
 279         format_description => 'string',
 280         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 281     },
 282 };
 283
 284 my $audio_fmt = {
 285     device => {
 286         type => 'string',
 287         enum => [qw(ich9-intel-hda intel-hda AC97)],
 288         description =>  "Configure an audio device."
 289     },
 290     driver =>  {
 291         type => 'string',
 292         enum => ['spice'],
 293         default => 'spice',
 294         optional => 1,
 295         description => "Driver backend for the audio device."
 296     },
 297 };
 298
 299 my $spice_enhancements_fmt = {
 300     foldersharing => {
 301         type => 'boolean',
 302         optional => 1,
 303         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 304     },
 305     videostreaming =>  {
 306         type => 'string',
 307         enum => ['off', 'all', 'filter'],
 308         optional => 1,
 309         description => "Enable video streaming. Uses compression for detected video streams."
 310     },
 311 };
 312
 313 my $confdesc = {
 314     onboot => {
 315         optional => 1,
 316         type => 'boolean',
 317         description => "Specifies whether a VM will be started during system bootup.",
 318         default => 0,
 319     },
 320     autostart => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Automatic restart after crash (currently ignored).",
 324         default => 0,
 325     },
 326     hotplug => {
 327         optional => 1,
 328         type => 'string', format => 'pve-hotplug-features',
 329         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 330         default => 'network,disk,usb',
 331     },
 332     reboot => {
 333         optional => 1,
 334         type => 'boolean',
 335         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 336         default => 1,
 337     },
 338     lock => {
 339         optional => 1,
 340         type => 'string',
 341         description => "Lock/unlock the VM.",
 342         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 343     },
 344     cpulimit => {
 345         optional => 1,
 346         type => 'number',
 347         description => "Limit of CPU usage.",
 348         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 349         minimum => 0,
 350         maximum => 128,
 351         default => 0,
 352     },
 353     cpuunits => {
 354         optional => 1,
 355         type => 'integer',
 356         description => "CPU weight for a VM.",
 357         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 358         minimum => 2,
 359         maximum => 262144,
 360         default => 1024,
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'integer',
 365         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 366         minimum => 16,
 367         default => 512,
 368     },
 369     balloon => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 373         minimum => 0,
 374     },
 375     shares => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 379         minimum => 0,
 380         maximum => 50000,
 381         default => 1000,
 382     },
 383     keyboard => {
 384         optional => 1,
 385         type => 'string',
 386         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 387                        "It should not be necessary to set it.",
 388         enum => PVE::Tools::kvmkeymaplist(),
 389         default => undef,
 390     },
 391     name => {
 392         optional => 1,
 393         type => 'string', format => 'dns-name',
 394         description => "Set a name for the VM. Only used on the configuration web interface.",
 395     },
 396     scsihw => {
 397         optional => 1,
 398         type => 'string',
 399         description => "SCSI controller model",
 400         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 401         default => 'lsi',
 402     },
 403     description => {
 404         optional => 1,
 405         type => 'string',
 406         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 407     },
 408     ostype => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 412         description => "Specify guest operating system.",
 413         verbose_description => <<EODESC,
 414 Specify guest operating system. This is used to enable special
 415 optimization/features for specific operating systems:
 416
 417 [horizontal]
 418 other;; unspecified OS
 419 wxp;; Microsoft Windows XP
 420 w2k;; Microsoft Windows 2000
 421 w2k3;; Microsoft Windows 2003
 422 w2k8;; Microsoft Windows 2008
 423 wvista;; Microsoft Windows Vista
 424 win7;; Microsoft Windows 7
 425 win8;; Microsoft Windows 8/2012/2012r2
 426 win10;; Microsoft Windows 10/2016
 427 l24;; Linux 2.4 Kernel
 428 l26;; Linux 2.6/3.X Kernel
 429 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 430 EODESC
 431     },
 432     boot => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 436         pattern => '[acdn]{1,4}',
 437         default => 'cdn',
 438     },
 439     bootdisk => {
 440         optional => 1,
 441         type => 'string', format => 'pve-qm-bootdisk',
 442         description => "Enable booting from specified disk.",
 443         pattern => '(ide|sata|scsi|virtio)\d+',
 444     },
 445     smp => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of CPUs. Please use option -sockets instead.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     sockets => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPU sockets.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     cores => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of cores per socket.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     numa => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Enable/disable NUMA.",
 470         default => 0,
 471     },
 472     hugepages => {
 473         optional => 1,
 474         type => 'string',
 475         description => "Enable/disable hugepages memory.",
 476         enum => [qw(any 2 1024)],
 477     },
 478     vcpus => {
 479         optional => 1,
 480         type => 'integer',
 481         description => "Number of hotplugged vcpus.",
 482         minimum => 1,
 483         default => 0,
 484     },
 485     acpi => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Enable/disable ACPI.",
 489         default => 1,
 490     },
 491     agent => {
 492         optional => 1,
 493         description => "Enable/disable Qemu GuestAgent and its properties.",
 494         type => 'string',
 495         format => $agent_fmt,
 496     },
 497     kvm => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Enable/disable KVM hardware virtualization.",
 501         default => 1,
 502     },
 503     tdf => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable time drift fix.",
 507         default => 0,
 508     },
 509     localtime => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 513     },
 514     freeze => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 518     },
 519     vga => {
 520         optional => 1,
 521         type => 'string', format => $vga_fmt,
 522         description => "Configure the VGA hardware.",
 523         verbose_description => "Configure the VGA Hardware. If you want to use ".
 524             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 525             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 526             "is 'std' for all OS types besides some Windows versions (XP and " .
 527             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 528             "display server. For win* OS you can select how many independent " .
 529             "displays you want, Linux guests can add displays them self.\n".
 530             "You can also run without any graphic card, using a serial device as terminal.",
 531     },
 532     watchdog => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-watchdog',
 535         description => "Create a virtual hardware watchdog device.",
 536         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 537             " (by a guest action), the watchdog must be periodically polled " .
 538             "by an agent inside the guest or else the watchdog will reset " .
 539             "the guest (or execute the respective action specified)",
 540     },
 541     startdate => {
 542         optional => 1,
 543         type => 'string',
 544         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 545         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 546         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 547         default => 'now',
 548     },
 549     startup =>  get_standard_option('pve-startup-order'),
 550     template => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Enable/disable Template.",
 554         default => 0,
 555     },
 556     args => {
 557         optional => 1,
 558         type => 'string',
 559         description => "Arbitrary arguments passed to kvm.",
 560         verbose_description => <<EODESCR,
 561 Arbitrary arguments passed to kvm, for example:
 562
 563 args: -no-reboot -no-hpet
 564
 565 NOTE: this option is for experts only.
 566 EODESCR
 567     },
 568     tablet => {
 569         optional => 1,
 570         type => 'boolean',
 571         default => 1,
 572         description => "Enable/disable the USB tablet device.",
 573         verbose_description => "Enable/disable the USB tablet device. This device is " .
 574             "usually needed to allow absolute mouse positioning with VNC. " .
 575             "Else the mouse runs out of sync with normal VNC clients. " .
 576             "If you're running lots of console-only guests on one host, " .
 577             "you may consider disabling this to save some context switches. " .
 578             "This is turned off by default if you use spice (-vga=qxl).",
 579     },
 580     migrate_speed => {
 581         optional => 1,
 582         type => 'integer',
 583         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 584         minimum => 0,
 585         default => 0,
 586     },
 587     migrate_downtime => {
 588         optional => 1,
 589         type => 'number',
 590         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 591         minimum => 0,
 592         default => 0.1,
 593     },
 594     cdrom => {
 595         optional => 1,
 596         type => 'string', format => 'pve-qm-ide',
 597         typetext => '<volume>',
 598         description => "This is an alias for option -ide2",
 599     },
 600     cpu => {
 601         optional => 1,
 602         description => "Emulated CPU type.",
 603         type => 'string',
 604         format => $cpu_fmt,
 605     },
 606     parent => get_standard_option('pve-snapshot-name', {
 607         optional => 1,
 608         description => "Parent snapshot name. This is used internally, and should not be modified.",
 609     }),
 610     snaptime => {
 611         optional => 1,
 612         description => "Timestamp for snapshots.",
 613         type => 'integer',
 614         minimum => 0,
 615     },
 616     vmstate => {
 617         optional => 1,
 618         type => 'string', format => 'pve-volume-id',
 619         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 620     },
 621     vmstatestorage => get_standard_option('pve-storage-id', {
 622         description => "Default storage for VM state volumes/files.",
 623         optional => 1,
 624     }),
 625     runningmachine => get_standard_option('pve-qemu-machine', {
 626         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 627     }),
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 645         default => 0,
 646     },
 647     bios => {
 648         optional => 1,
 649         type => 'string',
 650         enum => [ qw(seabios ovmf) ],
 651         description => "Select BIOS implementation.",
 652         default => 'seabios',
 653     },
 654     vmgenid => {
 655         type => 'string',
 656         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 657         format_description => 'UUID',
 658         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 659         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 660             " 128-bit integer value identifier to the guest OS. This allows to".
 661             " notify the guest operating system when the virtual machine is".
 662             " executed with a different configuration (e.g. snapshot execution".
 663             " or creation from a template). The guest operating system notices".
 664             " the change, and is then able to react as appropriate by marking".
 665             " its copies of distributed databases as dirty, re-initializing its".
 666             " random number generator, etc.\n".
 667             "Note that auto-creation only works when done throug API/CLI create".
 668             " or update methods, but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 682         optional => 1,
 683     },
 684     audio0 => {
 685         type => 'string',
 686         format => $audio_fmt,
 687         description => "Configure a audio device, useful in combination with QXL/Spice.",
 688         optional => 1
 689     },
 690     spice_enhancements => {
 691         type => 'string',
 692         format => $spice_enhancements_fmt,
 693         description => "Configure additional enhancements for SPICE.",
 694         optional => 1
 695     },
 696 };
 697
 698 my $cicustom_fmt = {
 699     meta => {
 700         type => 'string',
 701         optional => 1,
 702         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     network => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     user => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720 };
 721 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 722
 723 my $confdesc_cloudinit = {
 724     citype => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 728         enum => ['configdrive2', 'nocloud'],
 729     },
 730     ciuser => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 734     },
 735     cipassword => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 739     },
 740     cicustom => {
 741         optional => 1,
 742         type => 'string',
 743         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 750     },
 751     nameserver => {
 752         optional => 1,
 753         type => 'string', format => 'address-list',
 754         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 755     },
 756     sshkeys => {
 757         optional => 1,
 758         type => 'string',
 759         format => 'urlencoded',
 760         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 761     },
 762 };
 763
 764 # what about other qemu settings ?
 765 #cpu => 'string',
 766 #machine => 'string',
 767 #fda => 'file',
 768 #fdb => 'file',
 769 #mtdblock => 'file',
 770 #sd => 'file',
 771 #pflash => 'file',
 772 #snapshot => 'bool',
 773 #bootp => 'file',
 774 ##tftp => 'dir',
 775 ##smb => 'dir',
 776 #kernel => 'file',
 777 #append => 'string',
 778 #initrd => 'file',
 779 ##soundhw => 'string',
 780
 781 while (my ($k, $v) = each %$confdesc) {
 782     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 783 }
 784
 785 my $MAX_IDE_DISKS = 4;
 786 my $MAX_SCSI_DISKS = 14;
 787 my $MAX_VIRTIO_DISKS = 16;
 788 my $MAX_SATA_DISKS = 6;
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_UNUSED_DISKS = 256;
 792 my $MAX_HOSTPCI_DEVICES = 4;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 836                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 837                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 839
 840 my $net_fmt_bridge_descr = <<__EOD__;
 841 Bridge to attach the network device to. The Proxmox VE standard bridge
 842 is called 'vmbr0'.
 843
 844 If you do not specify a bridge, we create a kvm user (NATed) network
 845 device, which provides DHCP and DNS services. The following addresses
 846 are used:
 847
 848  10.0.2.2   Gateway
 849  10.0.2.3   DNS Server
 850  10.0.2.4   SMB Server
 851
 852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 853 __EOD__
 854
 855 my $net_fmt = {
 856     macaddr  => get_standard_option('mac-addr', {
 857         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 858     }),
 859     model => {
 860         type => 'string',
 861         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 862         enum => $nic_model_list,
 863         default_key => 1,
 864     },
 865     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 866     bridge => {
 867         type => 'string',
 868         description => $net_fmt_bridge_descr,
 869         format_description => 'bridge',
 870         optional => 1,
 871     },
 872     queues => {
 873         type => 'integer',
 874         minimum => 0, maximum => 16,
 875         description => 'Number of packet queues to be used on the device.',
 876         optional => 1,
 877     },
 878     rate => {
 879         type => 'number',
 880         minimum => 0,
 881         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 882         optional => 1,
 883     },
 884     tag => {
 885         type => 'integer',
 886         minimum => 1, maximum => 4094,
 887         description => 'VLAN tag to apply to packets on this interface.',
 888         optional => 1,
 889     },
 890     trunks => {
 891         type => 'string',
 892         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 893         description => 'VLAN trunks to pass through this interface.',
 894         format_description => 'vlanid[;vlanid...]',
 895         optional => 1,
 896     },
 897     firewall => {
 898         type => 'boolean',
 899         description => 'Whether this interface should be protected by the firewall.',
 900         optional => 1,
 901     },
 902     link_down => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be disconnected (like pulling the plug).',
 905         optional => 1,
 906     },
 907 };
 908
 909 my $netdesc = {
 910     optional => 1,
 911     type => 'string', format => $net_fmt,
 912     description => "Specify network devices.",
 913 };
 914
 915 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 916
 917 my $ipconfig_fmt = {
 918     ip => {
 919         type => 'string',
 920         format => 'pve-ipv4-config',
 921         format_description => 'IPv4Format/CIDR',
 922         description => 'IPv4 address in CIDR format.',
 923         optional => 1,
 924         default => 'dhcp',
 925     },
 926     gw => {
 927         type => 'string',
 928         format => 'ipv4',
 929         format_description => 'GatewayIPv4',
 930         description => 'Default gateway for IPv4 traffic.',
 931         optional => 1,
 932         requires => 'ip',
 933     },
 934     ip6 => {
 935         type => 'string',
 936         format => 'pve-ipv6-config',
 937         format_description => 'IPv6Format/CIDR',
 938         description => 'IPv6 address in CIDR format.',
 939         optional => 1,
 940         default => 'dhcp',
 941     },
 942     gw6 => {
 943         type => 'string',
 944         format => 'ipv6',
 945         format_description => 'GatewayIPv6',
 946         description => 'Default gateway for IPv6 traffic.',
 947         optional => 1,
 948         requires => 'ip6',
 949     },
 950 };
 951 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 952 my $ipconfigdesc = {
 953     optional => 1,
 954     type => 'string', format => 'pve-qm-ipconfig',
 955     description => <<'EODESCR',
 956 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 957
 958 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 959
 960 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 961 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 962
 963 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 964 EODESCR
 965 };
 966 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 967
 968 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 969     $confdesc->{"net$i"} = $netdesc;
 970     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 971 }
 972
 973 foreach my $key (keys %$confdesc_cloudinit) {
 974     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 975 }
 976
 977 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 978 sub verify_volume_id_or_qm_path {
 979     my ($volid, $noerr) = @_;
 980
 981     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 982         return $volid;
 983     }
 984
 985     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 986     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 987     if ($@) {
 988         return undef if $noerr;
 989         die $@;
 990     }
 991     return $volid;
 992 }
 993
 994 my $drivename_hash;
 995
 996 my %drivedesc_base = (
 997     volume => { alias => 'file' },
 998     file => {
 999         type => 'string',
1000         format => 'pve-volume-id-or-qm-path',
1001         default_key => 1,
1002         format_description => 'volume',
1003         description => "The drive's backing volume.",
1004     },
1005     media => {
1006         type => 'string',
1007         enum => [qw(cdrom disk)],
1008         description => "The drive's media type.",
1009         default => 'disk',
1010         optional => 1
1011     },
1012     cyls => {
1013         type => 'integer',
1014         description => "Force the drive's physical geometry to have a specific cylinder count.",
1015         optional => 1
1016     },
1017     heads => {
1018         type => 'integer',
1019         description => "Force the drive's physical geometry to have a specific head count.",
1020         optional => 1
1021     },
1022     secs => {
1023         type => 'integer',
1024         description => "Force the drive's physical geometry to have a specific sector count.",
1025         optional => 1
1026     },
1027     trans => {
1028         type => 'string',
1029         enum => [qw(none lba auto)],
1030         description => "Force disk geometry bios translation mode.",
1031         optional => 1,
1032     },
1033     snapshot => {
1034         type => 'boolean',
1035         description => "Controls qemu's snapshot mode feature."
1036             . " If activated, changes made to the disk are temporary and will"
1037             . " be discarded when the VM is shutdown.",
1038         optional => 1,
1039     },
1040     cache => {
1041         type => 'string',
1042         enum => [qw(none writethrough writeback unsafe directsync)],
1043         description => "The drive's cache mode",
1044         optional => 1,
1045     },
1046     format => get_standard_option('pve-qm-image-format'),
1047     size => {
1048         type => 'string',
1049         format => 'disk-size',
1050         format_description => 'DiskSize',
1051         description => "Disk size. This is purely informational and has no effect.",
1052         optional => 1,
1053     },
1054     backup => {
1055         type => 'boolean',
1056         description => "Whether the drive should be included when making backups.",
1057         optional => 1,
1058     },
1059     replicate => {
1060         type => 'boolean',
1061         description => 'Whether the drive should considered for replication jobs.',
1062         optional => 1,
1063         default => 1,
1064     },
1065     rerror => {
1066         type => 'string',
1067         enum => [qw(ignore report stop)],
1068         description => 'Read error action.',
1069         optional => 1,
1070     },
1071     werror => {
1072         type => 'string',
1073         enum => [qw(enospc ignore report stop)],
1074         description => 'Write error action.',
1075         optional => 1,
1076     },
1077     aio => {
1078         type => 'string',
1079         enum => [qw(native threads)],
1080         description => 'AIO type to use.',
1081         optional => 1,
1082     },
1083     discard => {
1084         type => 'string',
1085         enum => [qw(ignore on)],
1086         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1087         optional => 1,
1088     },
1089     detect_zeroes => {
1090         type => 'boolean',
1091         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1092         optional => 1,
1093     },
1094     serial => {
1095         type => 'string',
1096         format => 'urlencoded',
1097         format_description => 'serial',
1098         maxLength => 20*3, # *3 since it's %xx url enoded
1099         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1100         optional => 1,
1101     },
1102     shared => {
1103         type => 'boolean',
1104         description => 'Mark this locally-managed volume as available on all nodes',
1105         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1106         optional => 1,
1107         default => 0,
1108     }
1109 );
1110
1111 my %iothread_fmt = ( iothread => {
1112         type => 'boolean',
1113         description => "Whether to use iothreads for this drive",
1114         optional => 1,
1115 });
1116
1117 my %model_fmt = (
1118     model => {
1119         type => 'string',
1120         format => 'urlencoded',
1121         format_description => 'model',
1122         maxLength => 40*3, # *3 since it's %xx url enoded
1123         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1124         optional => 1,
1125     },
1126 );
1127
1128 my %queues_fmt = (
1129     queues => {
1130         type => 'integer',
1131         description => "Number of queues.",
1132         minimum => 2,
1133         optional => 1
1134     }
1135 );
1136
1137 my %scsiblock_fmt = (
1138     scsiblock => {
1139         type => 'boolean',
1140         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1141         optional => 1,
1142         default => 0,
1143     },
1144 );
1145
1146 my %ssd_fmt = (
1147     ssd => {
1148         type => 'boolean',
1149         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1150         optional => 1,
1151     },
1152 );
1153
1154 my %wwn_fmt = (
1155     wwn => {
1156         type => 'string',
1157         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1158         format_description => 'wwn',
1159         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1160         optional => 1,
1161     },
1162 );
1163
1164 my $add_throttle_desc = sub {
1165     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1166     my $d = {
1167         type => $type,
1168         format_description => $unit,
1169         description => "Maximum $what in $longunit.",
1170         optional => 1,
1171     };
1172     $d->{minimum} = $minimum if defined($minimum);
1173     $drivedesc_base{$key} = $d;
1174 };
1175 # throughput: (leaky bucket)
1176 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1177 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1178 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1179 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1180 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1181 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1182 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1183 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1184 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1185
1186 # pools: (pool of IO before throttling starts taking effect)
1187 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1189 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1190 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1191 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1192 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1193
1194 # burst lengths
1195 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1196 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1197 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1198 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1199 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1200 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1201
1202 # legacy support
1203 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1204 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1205 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1206 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1207
1208 my $ide_fmt = {
1209     %drivedesc_base,
1210     %model_fmt,
1211     %ssd_fmt,
1212     %wwn_fmt,
1213 };
1214 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1215
1216 my $idedesc = {
1217     optional => 1,
1218     type => 'string', format => $ide_fmt,
1219     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1220 };
1221 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1222
1223 my $scsi_fmt = {
1224     %drivedesc_base,
1225     %iothread_fmt,
1226     %queues_fmt,
1227     %scsiblock_fmt,
1228     %ssd_fmt,
1229     %wwn_fmt,
1230 };
1231 my $scsidesc = {
1232     optional => 1,
1233     type => 'string', format => $scsi_fmt,
1234     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1235 };
1236 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1237
1238 my $sata_fmt = {
1239     %drivedesc_base,
1240     %ssd_fmt,
1241     %wwn_fmt,
1242 };
1243 my $satadesc = {
1244     optional => 1,
1245     type => 'string', format => $sata_fmt,
1246     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1247 };
1248 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1249
1250 my $virtio_fmt = {
1251     %drivedesc_base,
1252     %iothread_fmt,
1253 };
1254 my $virtiodesc = {
1255     optional => 1,
1256     type => 'string', format => $virtio_fmt,
1257     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1260
1261 my $alldrive_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264     %model_fmt,
1265     %queues_fmt,
1266     %scsiblock_fmt,
1267     %ssd_fmt,
1268     %wwn_fmt,
1269 };
1270
1271 my $efidisk_fmt = {
1272     volume => { alias => 'file' },
1273     file => {
1274         type => 'string',
1275         format => 'pve-volume-id-or-qm-path',
1276         default_key => 1,
1277         format_description => 'volume',
1278         description => "The drive's backing volume.",
1279     },
1280     format => get_standard_option('pve-qm-image-format'),
1281     size => {
1282         type => 'string',
1283         format => 'disk-size',
1284         format_description => 'DiskSize',
1285         description => "Disk size. This is purely informational and has no effect.",
1286         optional => 1,
1287     },
1288 };
1289
1290 my $efidisk_desc = {
1291     optional => 1,
1292     type => 'string', format => $efidisk_fmt,
1293     description => "Configure a Disk for storing EFI vars",
1294 };
1295
1296 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1297
1298 my $usb_fmt = {
1299     host => {
1300         default_key => 1,
1301         type => 'string', format => 'pve-qm-usb-device',
1302         format_description => 'HOSTUSBDEVICE|spice',
1303         description => <<EODESCR,
1304 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1305
1306  'bus-port(.port)*' (decimal numbers) or
1307  'vendor_id:product_id' (hexadeciaml numbers) or
1308  'spice'
1309
1310 You can use the 'lsusb -t' command to list existing usb devices.
1311
1312 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1313
1314 The value 'spice' can be used to add a usb redirection devices for spice.
1315 EODESCR
1316     },
1317     usb3 => {
1318         optional => 1,
1319         type => 'boolean',
1320         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1321         default => 0,
1322     },
1323 };
1324
1325 my $usbdesc = {
1326     optional => 1,
1327     type => 'string', format => $usb_fmt,
1328     description => "Configure an USB device (n is 0 to 4).",
1329 };
1330 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1331
1332 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1333 my $hostpci_fmt = {
1334     host => {
1335         default_key => 1,
1336         type => 'string',
1337         pattern => qr/$PCIRE(;$PCIRE)*/,
1338         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1339         description => <<EODESCR,
1340 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1341 of PCI virtual functions of the host. HOSTPCIID syntax is:
1342
1343 'bus:dev.func' (hexadecimal numbers)
1344
1345 You can us the 'lspci' command to list existing PCI devices.
1346 EODESCR
1347     },
1348     rombar => {
1349         type => 'boolean',
1350         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1351         optional => 1,
1352         default => 1,
1353     },
1354     romfile => {
1355         type => 'string',
1356         pattern => '[^,;]+',
1357         format_description => 'string',
1358         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1359         optional => 1,
1360     },
1361     pcie => {
1362         type => 'boolean',
1363         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1364         optional => 1,
1365         default => 0,
1366     },
1367     'x-vga' => {
1368         type => 'boolean',
1369         description =>  "Enable vfio-vga device support.",
1370         optional => 1,
1371         default => 0,
1372     },
1373     'mdev' => {
1374         type => 'string',
1375         format_description => 'string',
1376         pattern => '[^/\.:]+',
1377         optional => 1,
1378         description => <<EODESCR
1379 The type of mediated device to use.
1380 An instance of this type will be created on startup of the VM and
1381 will be cleaned up when the VM stops.
1382 EODESCR
1383     }
1384 };
1385 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1386
1387 my $hostpcidesc = {
1388         optional => 1,
1389         type => 'string', format => 'pve-qm-hostpci',
1390         description => "Map host PCI devices into guest.",
1391         verbose_description =>  <<EODESCR,
1392 Map host PCI devices into guest.
1393
1394 NOTE: This option allows direct access to host hardware. So it is no longer
1395 possible to migrate such machines - use with special care.
1396
1397 CAUTION: Experimental! User reported problems with this option.
1398 EODESCR
1399 };
1400 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1401
1402 my $serialdesc = {
1403         optional => 1,
1404         type => 'string',
1405         pattern => '(/dev/.+|socket)',
1406         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1407         verbose_description =>  <<EODESCR,
1408 Create a serial device inside the VM (n is 0 to 3), and pass through a
1409 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1410 host side (use 'qm terminal' to open a terminal connection).
1411
1412 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1413
1414 CAUTION: Experimental! User reported problems with this option.
1415 EODESCR
1416 };
1417
1418 my $paralleldesc= {
1419         optional => 1,
1420         type => 'string',
1421         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1422         description =>  "Map host parallel devices (n is 0 to 2).",
1423         verbose_description =>  <<EODESCR,
1424 Map host parallel devices (n is 0 to 2).
1425
1426 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1427
1428 CAUTION: Experimental! User reported problems with this option.
1429 EODESCR
1430 };
1431
1432 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1433     $confdesc->{"parallel$i"} = $paralleldesc;
1434 }
1435
1436 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1437     $confdesc->{"serial$i"} = $serialdesc;
1438 }
1439
1440 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1441     $confdesc->{"hostpci$i"} = $hostpcidesc;
1442 }
1443
1444 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1445     $drivename_hash->{"ide$i"} = 1;
1446     $confdesc->{"ide$i"} = $idedesc;
1447 }
1448
1449 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1450     $drivename_hash->{"sata$i"} = 1;
1451     $confdesc->{"sata$i"} = $satadesc;
1452 }
1453
1454 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1455     $drivename_hash->{"scsi$i"} = 1;
1456     $confdesc->{"scsi$i"} = $scsidesc ;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1460     $drivename_hash->{"virtio$i"} = 1;
1461     $confdesc->{"virtio$i"} = $virtiodesc;
1462 }
1463
1464 $drivename_hash->{efidisk0} = 1;
1465 $confdesc->{efidisk0} = $efidisk_desc;
1466
1467 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1468     $confdesc->{"usb$i"} = $usbdesc;
1469 }
1470
1471 my $unuseddesc = {
1472     optional => 1,
1473     type => 'string', format => 'pve-volume-id',
1474     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1475 };
1476
1477 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1478     $confdesc->{"unused$i"} = $unuseddesc;
1479 }
1480
1481 my $kvm_api_version = 0;
1482
1483 sub kvm_version {
1484     return $kvm_api_version if $kvm_api_version;
1485
1486     open my $fh, '<', '/dev/kvm'
1487         or return undef;
1488
1489     # 0xae00 => KVM_GET_API_VERSION
1490     $kvm_api_version = ioctl($fh, 0xae00, 0);
1491
1492     return $kvm_api_version;
1493 }
1494
1495 my $kvm_user_version = {};
1496 my $kvm_mtime = {};
1497
1498 sub kvm_user_version {
1499     my ($binary) = @_;
1500
1501     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1502     my $st = stat($binary);
1503
1504     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1505     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1506         $cachedmtime == $st->mtime;
1507
1508     $kvm_user_version->{$binary} = 'unknown';
1509     $kvm_mtime->{$binary} = $st->mtime;
1510
1511     my $code = sub {
1512         my $line = shift;
1513         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1514             $kvm_user_version->{$binary} = $2;
1515         }
1516     };
1517
1518     eval { run_command([$binary, '--version'], outfunc => $code); };
1519     warn $@ if $@;
1520
1521     return $kvm_user_version->{$binary};
1522
1523 }
1524
1525 sub kernel_has_vhost_net {
1526     return -c '/dev/vhost-net';
1527 }
1528
1529 sub valid_drive_names {
1530     # order is important - used to autoselect boot disk
1531     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1532             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1533             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1534             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1535             'efidisk0');
1536 }
1537
1538 sub is_valid_drivename {
1539     my $dev = shift;
1540
1541     return defined($drivename_hash->{$dev});
1542 }
1543
1544 sub option_exists {
1545     my $key = shift;
1546     return defined($confdesc->{$key});
1547 }
1548
1549 sub nic_models {
1550     return $nic_model_list;
1551 }
1552
1553 sub os_list_description {
1554
1555     return {
1556         other => 'Other',
1557         wxp => 'Windows XP',
1558         w2k => 'Windows 2000',
1559         w2k3 =>, 'Windows 2003',
1560         w2k8 => 'Windows 2008',
1561         wvista => 'Windows Vista',
1562         win7 => 'Windows 7',
1563         win8 => 'Windows 8/2012',
1564         win10 => 'Windows 10/2016',
1565         l24 => 'Linux 2.4',
1566         l26 => 'Linux 2.6',
1567     };
1568 }
1569
1570 my $cdrom_path;
1571
1572 sub get_cdrom_path {
1573
1574     return  $cdrom_path if $cdrom_path;
1575
1576     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1577     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1578     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1579 }
1580
1581 sub get_iso_path {
1582     my ($storecfg, $vmid, $cdrom) = @_;
1583
1584     if ($cdrom eq 'cdrom') {
1585         return get_cdrom_path();
1586     } elsif ($cdrom eq 'none') {
1587         return '';
1588     } elsif ($cdrom =~ m|^/|) {
1589         return $cdrom;
1590     } else {
1591         return PVE::Storage::path($storecfg, $cdrom);
1592     }
1593 }
1594
1595 # try to convert old style file names to volume IDs
1596 sub filename_to_volume_id {
1597     my ($vmid, $file, $media) = @_;
1598
1599      if (!($file eq 'none' || $file eq 'cdrom' ||
1600           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1601
1602         return undef if $file =~ m|/|;
1603
1604         if ($media && $media eq 'cdrom') {
1605             $file = "local:iso/$file";
1606         } else {
1607             $file = "local:$vmid/$file";
1608         }
1609     }
1610
1611     return $file;
1612 }
1613
1614 sub verify_media_type {
1615     my ($opt, $vtype, $media) = @_;
1616
1617     return if !$media;
1618
1619     my $etype;
1620     if ($media eq 'disk') {
1621         $etype = 'images';
1622     } elsif ($media eq 'cdrom') {
1623         $etype = 'iso';
1624     } else {
1625         die "internal error";
1626     }
1627
1628     return if ($vtype eq $etype);
1629
1630     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1631 }
1632
1633 sub cleanup_drive_path {
1634     my ($opt, $storecfg, $drive) = @_;
1635
1636     # try to convert filesystem paths to volume IDs
1637
1638     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1639         ($drive->{file} !~ m|^/dev/.+|) &&
1640         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1641         ($drive->{file} !~ m/^\d+$/)) {
1642         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1643         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1644         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1645         verify_media_type($opt, $vtype, $drive->{media});
1646         $drive->{file} = $volid;
1647     }
1648
1649     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1650 }
1651
1652 sub parse_hotplug_features {
1653     my ($data) = @_;
1654
1655     my $res = {};
1656
1657     return $res if $data eq '0';
1658
1659     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1660
1661     foreach my $feature (PVE::Tools::split_list($data)) {
1662         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1663             $res->{$1} = 1;
1664         } else {
1665             die "invalid hotplug feature '$feature'\n";
1666         }
1667     }
1668     return $res;
1669 }
1670
1671 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1672 sub pve_verify_hotplug_features {
1673     my ($value, $noerr) = @_;
1674
1675     return $value if parse_hotplug_features($value);
1676
1677     return undef if $noerr;
1678
1679     die "unable to parse hotplug option\n";
1680 }
1681
1682 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1683 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1684 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1685 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1686 #        [,iothread=on][,serial=serial][,model=model]
1687
1688 sub parse_drive {
1689     my ($key, $data) = @_;
1690
1691     my ($interface, $index);
1692
1693     if ($key =~ m/^([^\d]+)(\d+)$/) {
1694         $interface = $1;
1695         $index = $2;
1696     } else {
1697         return undef;
1698     }
1699
1700     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1701                                      : $confdesc->{$key}->{format};
1702     if (!$desc) {
1703         warn "invalid drive key: $key\n";
1704         return undef;
1705     }
1706     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1707     return undef if !$res;
1708     $res->{interface} = $interface;
1709     $res->{index} = $index;
1710
1711     my $error = 0;
1712     foreach my $opt (qw(bps bps_rd bps_wr)) {
1713         if (my $bps = defined(delete $res->{$opt})) {
1714             if (defined($res->{"m$opt"})) {
1715                 warn "both $opt and m$opt specified\n";
1716                 ++$error;
1717                 next;
1718             }
1719             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1720         }
1721     }
1722
1723     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1724     for my $requirement (
1725         [mbps_max => 'mbps'],
1726         [mbps_rd_max => 'mbps_rd'],
1727         [mbps_wr_max => 'mbps_wr'],
1728         [miops_max => 'miops'],
1729         [miops_rd_max => 'miops_rd'],
1730         [miops_wr_max => 'miops_wr'],
1731         [bps_max_length => 'mbps_max'],
1732         [bps_rd_max_length => 'mbps_rd_max'],
1733         [bps_wr_max_length => 'mbps_wr_max'],
1734         [iops_max_length => 'iops_max'],
1735         [iops_rd_max_length => 'iops_rd_max'],
1736         [iops_wr_max_length => 'iops_wr_max']) {
1737         my ($option, $requires) = @$requirement;
1738         if ($res->{$option} && !$res->{$requires}) {
1739             warn "$option requires $requires\n";
1740             ++$error;
1741         }
1742     }
1743
1744     return undef if $error;
1745
1746     return undef if $res->{mbps_rd} && $res->{mbps};
1747     return undef if $res->{mbps_wr} && $res->{mbps};
1748     return undef if $res->{iops_rd} && $res->{iops};
1749     return undef if $res->{iops_wr} && $res->{iops};
1750
1751     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1752         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1753         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1754         return undef if $res->{interface} eq 'virtio';
1755     }
1756
1757     if (my $size = $res->{size}) {
1758         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1759     }
1760
1761     return $res;
1762 }
1763
1764 sub print_drive {
1765     my ($vmid, $drive) = @_;
1766     my $data = { %$drive };
1767     delete $data->{$_} for qw(index interface);
1768     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1769 }
1770
1771 sub scsi_inquiry {
1772     my($fh, $noerr) = @_;
1773
1774     my $SG_IO = 0x2285;
1775     my $SG_GET_VERSION_NUM = 0x2282;
1776
1777     my $versionbuf = "\x00" x 8;
1778     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1779     if (!$ret) {
1780         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1781         return undef;
1782     }
1783     my $version = unpack("I", $versionbuf);
1784     if ($version < 30000) {
1785         die "scsi generic interface too old\n"  if !$noerr;
1786         return undef;
1787     }
1788
1789     my $buf = "\x00" x 36;
1790     my $sensebuf = "\x00" x 8;
1791     my $cmd = pack("C x3 C x1", 0x12, 36);
1792
1793     # see /usr/include/scsi/sg.h
1794     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1795
1796     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1797                       length($sensebuf), 0, length($buf), $buf,
1798                       $cmd, $sensebuf, 6000);
1799
1800     $ret = ioctl($fh, $SG_IO, $packet);
1801     if (!$ret) {
1802         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1803         return undef;
1804     }
1805
1806     my @res = unpack($sg_io_hdr_t, $packet);
1807     if ($res[17] || $res[18]) {
1808         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1809         return undef;
1810     }
1811
1812     my $res = {};
1813     (my $byte0, my $byte1, $res->{vendor},
1814      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1815
1816     $res->{removable} = $byte1 & 128 ? 1 : 0;
1817     $res->{type} = $byte0 & 31;
1818
1819     return $res;
1820 }
1821
1822 sub path_is_scsi {
1823     my ($path) = @_;
1824
1825     my $fh = IO::File->new("+<$path") || return undef;
1826     my $res = scsi_inquiry($fh, 1);
1827     close($fh);
1828
1829     return $res;
1830 }
1831
1832 sub machine_type_is_q35 {
1833     my ($conf) = @_;
1834
1835     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1836 }
1837
1838 sub print_tabletdevice_full {
1839     my ($conf, $arch) = @_;
1840
1841     my $q35 = machine_type_is_q35($conf);
1842
1843     # we use uhci for old VMs because tablet driver was buggy in older qemu
1844     my $usbbus;
1845     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1846         $usbbus = 'ehci';
1847     } else {
1848         $usbbus = 'uhci';
1849     }
1850
1851     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1852 }
1853
1854 sub print_keyboarddevice_full {
1855     my ($conf, $arch, $machine) = @_;
1856
1857     return undef if $arch ne 'aarch64';
1858
1859     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1860 }
1861
1862 sub print_drivedevice_full {
1863     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1864
1865     my $device = '';
1866     my $maxdev = 0;
1867
1868     if ($drive->{interface} eq 'virtio') {
1869         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1870         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1871         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1872     } elsif ($drive->{interface} eq 'scsi') {
1873
1874         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1875         my $unit = $drive->{index} % $maxdev;
1876         my $devicetype = 'hd';
1877         my $path = '';
1878         if (drive_is_cdrom($drive)) {
1879             $devicetype = 'cd';
1880         } else {
1881             if ($drive->{file} =~ m|^/|) {
1882                 $path = $drive->{file};
1883                 if (my $info = path_is_scsi($path)) {
1884                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1885                         $devicetype = 'block';
1886                     } elsif ($info->{type} == 1) { # tape
1887                         $devicetype = 'generic';
1888                     }
1889                 }
1890             } else {
1891                  $path = PVE::Storage::path($storecfg, $drive->{file});
1892             }
1893
1894             if($path =~ m/^iscsi\:\/\//){
1895                 $devicetype = 'generic';
1896             }
1897         }
1898
1899         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1900             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1901         } else {
1902             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1903         }
1904
1905         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1906             $device .= ",rotation_rate=1";
1907         }
1908         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1909
1910     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1911         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1912         my $controller = int($drive->{index} / $maxdev);
1913         my $unit = $drive->{index} % $maxdev;
1914         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1915
1916         $device = "ide-$devicetype";
1917         if ($drive->{interface} eq 'ide') {
1918             $device .= ",bus=ide.$controller,unit=$unit";
1919         } else {
1920             $device .= ",bus=ahci$controller.$unit";
1921         }
1922         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1923
1924         if ($devicetype eq 'hd') {
1925             if (my $model = $drive->{model}) {
1926                 $model = URI::Escape::uri_unescape($model);
1927                 $device .= ",model=$model";
1928             }
1929             if ($drive->{ssd}) {
1930                 $device .= ",rotation_rate=1";
1931             }
1932         }
1933         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1934     } elsif ($drive->{interface} eq 'usb') {
1935         die "implement me";
1936         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1937     } else {
1938         die "unsupported interface type";
1939     }
1940
1941     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1942
1943     if (my $serial = $drive->{serial}) {
1944         $serial = URI::Escape::uri_unescape($serial);
1945         $device .= ",serial=$serial";
1946     }
1947
1948
1949     return $device;
1950 }
1951
1952 sub get_initiator_name {
1953     my $initiator;
1954
1955     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1956     while (defined(my $line = <$fh>)) {
1957         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1958         $initiator = $1;
1959         last;
1960     }
1961     $fh->close();
1962
1963     return $initiator;
1964 }
1965
1966 sub print_drive_full {
1967     my ($storecfg, $vmid, $drive) = @_;
1968
1969     my $path;
1970     my $volid = $drive->{file};
1971     my $format;
1972
1973     if (drive_is_cdrom($drive)) {
1974         $path = get_iso_path($storecfg, $vmid, $volid);
1975     } else {
1976         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1977         if ($storeid) {
1978             $path = PVE::Storage::path($storecfg, $volid);
1979             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1980             $format = qemu_img_format($scfg, $volname);
1981         } else {
1982             $path = $volid;
1983             $format = "raw";
1984         }
1985    }
1986
1987     my $opts = '';
1988     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1989     foreach my $o (@qemu_drive_options) {
1990         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1991     }
1992
1993     # snapshot only accepts on|off
1994     if (defined($drive->{snapshot})) {
1995         my $v = $drive->{snapshot} ? 'on' : 'off';
1996         $opts .= ",snapshot=$v";
1997     }
1998
1999     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2000         my ($dir, $qmpname) = @$type;
2001         if (my $v = $drive->{"mbps$dir"}) {
2002             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2003         }
2004         if (my $v = $drive->{"mbps${dir}_max"}) {
2005             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2006         }
2007         if (my $v = $drive->{"bps${dir}_max_length"}) {
2008             $opts .= ",throttling.bps$qmpname-max-length=$v";
2009         }
2010         if (my $v = $drive->{"iops${dir}"}) {
2011             $opts .= ",throttling.iops$qmpname=$v";
2012         }
2013         if (my $v = $drive->{"iops${dir}_max"}) {
2014             $opts .= ",throttling.iops$qmpname-max=$v";
2015         }
2016         if (my $v = $drive->{"iops${dir}_max_length"}) {
2017             $opts .= ",throttling.iops$qmpname-max-length=$v";
2018         }
2019     }
2020
2021     $opts .= ",format=$format" if $format && !$drive->{format};
2022
2023     my $cache_direct = 0;
2024
2025     if (my $cache = $drive->{cache}) {
2026         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2027     } elsif (!drive_is_cdrom($drive)) {
2028         $opts .= ",cache=none";
2029         $cache_direct = 1;
2030     }
2031
2032     # aio native works only with O_DIRECT
2033     if (!$drive->{aio}) {
2034         if($cache_direct) {
2035             $opts .= ",aio=native";
2036         } else {
2037             $opts .= ",aio=threads";
2038         }
2039     }
2040
2041     if (!drive_is_cdrom($drive)) {
2042         my $detectzeroes;
2043         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2044             $detectzeroes = 'off';
2045         } elsif ($drive->{discard}) {
2046             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2047         } else {
2048             # This used to be our default with discard not being specified:
2049             $detectzeroes = 'on';
2050         }
2051         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2052     }
2053
2054     my $pathinfo = $path ? "file=$path," : '';
2055
2056     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2057 }
2058
2059 sub print_netdevice_full {
2060     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2061
2062     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2063
2064     my $device = $net->{model};
2065     if ($net->{model} eq 'virtio') {
2066          $device = 'virtio-net-pci';
2067      };
2068
2069     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2070     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2071     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2072         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2073         my $vectors = $net->{queues} * 2 + 2;
2074         $tmpstr .= ",vectors=$vectors,mq=on";
2075     }
2076     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2077
2078     if ($use_old_bios_files) {
2079         my $romfile;
2080         if ($device eq 'virtio-net-pci') {
2081             $romfile = 'pxe-virtio.rom';
2082         } elsif ($device eq 'e1000') {
2083             $romfile = 'pxe-e1000.rom';
2084         } elsif ($device eq 'ne2k') {
2085             $romfile = 'pxe-ne2k_pci.rom';
2086         } elsif ($device eq 'pcnet') {
2087             $romfile = 'pxe-pcnet.rom';
2088         } elsif ($device eq 'rtl8139') {
2089             $romfile = 'pxe-rtl8139.rom';
2090         }
2091         $tmpstr .= ",romfile=$romfile" if $romfile;
2092     }
2093
2094     return $tmpstr;
2095 }
2096
2097 sub print_netdev_full {
2098     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2099
2100     my $i = '';
2101     if ($netid =~ m/^net(\d+)$/) {
2102         $i = int($1);
2103     }
2104
2105     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2106
2107     my $ifname = "tap${vmid}i$i";
2108
2109     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2110     die "interface name '$ifname' is too long (max 15 character)\n"
2111         if length($ifname) >= 16;
2112
2113     my $vhostparam = '';
2114     if (is_native($arch)) {
2115         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2116     }
2117
2118     my $vmname = $conf->{name} || "vm$vmid";
2119
2120     my $netdev = "";
2121     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2122
2123     if ($net->{bridge}) {
2124         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2125     } else {
2126         $netdev = "type=user,id=$netid,hostname=$vmname";
2127     }
2128
2129     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2130
2131     return $netdev;
2132 }
2133
2134
2135 sub print_cpu_device {
2136     my ($conf, $id) = @_;
2137
2138     my $kvm = $conf->{kvm} // 1;
2139     my $cpu = $kvm ? "kvm64" : "qemu64";
2140     if (my $cputype = $conf->{cpu}) {
2141         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2142             or die "Cannot parse cpu description: $cputype\n";
2143         $cpu = $cpuconf->{cputype};
2144     }
2145
2146     my $cores = $conf->{cores} || 1;
2147
2148     my $current_core = ($id - 1) % $cores;
2149     my $current_socket = int(($id - 1 - $current_core)/$cores);
2150
2151     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2152 }
2153
2154 my $vga_map = {
2155     'cirrus' => 'cirrus-vga',
2156     'std' => 'VGA',
2157     'vmware' => 'vmware-svga',
2158     'virtio' => 'virtio-vga',
2159 };
2160
2161 sub print_vga_device {
2162     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2163
2164     my $type = $vga_map->{$vga->{type}};
2165     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2166         $type = 'virtio-gpu';
2167     }
2168     my $vgamem_mb = $vga->{memory};
2169     if ($qxlnum) {
2170         $type = $id ? 'qxl' : 'qxl-vga';
2171     }
2172     die "no devicetype for $vga->{type}\n" if !$type;
2173
2174     my $memory = "";
2175     if ($vgamem_mb) {
2176         if ($vga->{type} eq 'virtio') {
2177             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2178             $memory = ",max_hostmem=$bytes";
2179         } elsif ($qxlnum) {
2180             # from https://www.spice-space.org/multiple-monitors.html
2181             $memory = ",vgamem_mb=$vga->{memory}";
2182             my $ram = $vgamem_mb * 4;
2183             my $vram = $vgamem_mb * 2;
2184             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2185         } else {
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187         }
2188     } elsif ($qxlnum && $id) {
2189         $memory = ",ram_size=67108864,vram_size=33554432";
2190     }
2191
2192     my $q35 = machine_type_is_q35($conf);
2193     my $vgaid = "vga" . ($id // '');
2194     my $pciaddr;
2195
2196     if ($q35 && $vgaid eq 'vga') {
2197         # the first display uses pcie.0 bus on q35 machines
2198         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2199     } else {
2200         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2201     }
2202
2203     return "$type,id=${vgaid}${memory}${pciaddr}";
2204 }
2205
2206 sub drive_is_cloudinit {
2207     my ($drive) = @_;
2208     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2209 }
2210
2211 sub drive_is_cdrom {
2212     my ($drive, $exclude_cloudinit) = @_;
2213
2214     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2215
2216     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2217
2218 }
2219
2220 sub parse_number_sets {
2221     my ($set) = @_;
2222     my $res = [];
2223     foreach my $part (split(/;/, $set)) {
2224         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2225             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2226             push @$res, [ $1, $2 ];
2227         } else {
2228             die "invalid range: $part\n";
2229         }
2230     }
2231     return $res;
2232 }
2233
2234 sub parse_numa {
2235     my ($data) = @_;
2236
2237     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2238     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2239     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2240     return $res;
2241 }
2242
2243 sub parse_hostpci {
2244     my ($value) = @_;
2245
2246     return undef if !$value;
2247
2248     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2249
2250     my @idlist = split(/;/, $res->{host});
2251     delete $res->{host};
2252     foreach my $id (@idlist) {
2253         if ($id =~ m/\./) { # full id 00:00.1
2254             push @{$res->{pciid}}, {
2255                 id => $id,
2256             };
2257         } else { # partial id 00:00
2258             $res->{pciid} = PVE::SysFSTools::lspci($id);
2259         }
2260     }
2261     return $res;
2262 }
2263
2264 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2265 sub parse_net {
2266     my ($data) = @_;
2267
2268     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2269     if ($@) {
2270         warn $@;
2271         return undef;
2272     }
2273     if (!defined($res->{macaddr})) {
2274         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2275         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2276     }
2277     return $res;
2278 }
2279
2280 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2281 sub parse_ipconfig {
2282     my ($data) = @_;
2283
2284     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2285     if ($@) {
2286         warn $@;
2287         return undef;
2288     }
2289
2290     if ($res->{gw} && !$res->{ip}) {
2291         warn 'gateway specified without specifying an IP address';
2292         return undef;
2293     }
2294     if ($res->{gw6} && !$res->{ip6}) {
2295         warn 'IPv6 gateway specified without specifying an IPv6 address';
2296         return undef;
2297     }
2298     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2299         warn 'gateway specified together with DHCP';
2300         return undef;
2301     }
2302     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2303         # gw6 + auto/dhcp
2304         warn "IPv6 gateway specified together with $res->{ip6} address";
2305         return undef;
2306     }
2307
2308     if (!$res->{ip} && !$res->{ip6}) {
2309         return { ip => 'dhcp', ip6 => 'dhcp' };
2310     }
2311
2312     return $res;
2313 }
2314
2315 sub print_net {
2316     my $net = shift;
2317
2318     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2319 }
2320
2321 sub add_random_macs {
2322     my ($settings) = @_;
2323
2324     foreach my $opt (keys %$settings) {
2325         next if $opt !~ m/^net(\d+)$/;
2326         my $net = parse_net($settings->{$opt});
2327         next if !$net;
2328         $settings->{$opt} = print_net($net);
2329     }
2330 }
2331
2332 sub vm_is_volid_owner {
2333     my ($storecfg, $vmid, $volid) = @_;
2334
2335     if ($volid !~  m|^/|) {
2336         my ($path, $owner);
2337         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2338         if ($owner && ($owner == $vmid)) {
2339             return 1;
2340         }
2341     }
2342
2343     return undef;
2344 }
2345
2346 sub split_flagged_list {
2347     my $text = shift || '';
2348     $text =~ s/[,;]/ /g;
2349     $text =~ s/^\s+//;
2350     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2351 }
2352
2353 sub join_flagged_list {
2354     my ($how, $lst) = @_;
2355     join $how, map { $lst->{$_} . $_ } keys %$lst;
2356 }
2357
2358 sub vmconfig_delete_pending_option {
2359     my ($conf, $key, $force) = @_;
2360
2361     delete $conf->{pending}->{$key};
2362     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2363     $pending_delete_hash->{$key} = $force ? '!' : '';
2364     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2365 }
2366
2367 sub vmconfig_undelete_pending_option {
2368     my ($conf, $key) = @_;
2369
2370     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2371     delete $pending_delete_hash->{$key};
2372
2373     if (%$pending_delete_hash) {
2374         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2375     } else {
2376         delete $conf->{pending}->{delete};
2377     }
2378 }
2379
2380 sub vmconfig_register_unused_drive {
2381     my ($storecfg, $vmid, $conf, $drive) = @_;
2382
2383     if (drive_is_cloudinit($drive)) {
2384         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2385         warn $@ if $@;
2386     } elsif (!drive_is_cdrom($drive)) {
2387         my $volid = $drive->{file};
2388         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2389             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2390         }
2391     }
2392 }
2393
2394 sub vmconfig_cleanup_pending {
2395     my ($conf) = @_;
2396
2397     # remove pending changes when nothing changed
2398     my $changes;
2399     foreach my $opt (keys %{$conf->{pending}}) {
2400         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2401             $changes = 1;
2402             delete $conf->{pending}->{$opt};
2403         }
2404     }
2405
2406     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2407     my $pending_delete_hash = {};
2408     while (my ($opt, $force) = each %$current_delete_hash) {
2409         if (defined($conf->{$opt})) {
2410             $pending_delete_hash->{$opt} = $force;
2411         } else {
2412             $changes = 1;
2413         }
2414     }
2415
2416     if (%$pending_delete_hash) {
2417         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2418     } else {
2419         delete $conf->{pending}->{delete};
2420     }
2421
2422     return $changes;
2423 }
2424
2425 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2426 my $smbios1_fmt = {
2427     uuid => {
2428         type => 'string',
2429         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2430         format_description => 'UUID',
2431         description => "Set SMBIOS1 UUID.",
2432         optional => 1,
2433     },
2434     version => {
2435         type => 'string',
2436         pattern => '[A-Za-z0-9+\/]+={0,2}',
2437         format_description => 'Base64 encoded string',
2438         description => "Set SMBIOS1 version.",
2439         optional => 1,
2440     },
2441     serial => {
2442         type => 'string',
2443         pattern => '[A-Za-z0-9+\/]+={0,2}',
2444         format_description => 'Base64 encoded string',
2445         description => "Set SMBIOS1 serial number.",
2446         optional => 1,
2447     },
2448     manufacturer => {
2449         type => 'string',
2450         pattern => '[A-Za-z0-9+\/]+={0,2}',
2451         format_description => 'Base64 encoded string',
2452         description => "Set SMBIOS1 manufacturer.",
2453         optional => 1,
2454     },
2455     product => {
2456         type => 'string',
2457         pattern => '[A-Za-z0-9+\/]+={0,2}',
2458         format_description => 'Base64 encoded string',
2459         description => "Set SMBIOS1 product ID.",
2460         optional => 1,
2461     },
2462     sku => {
2463         type => 'string',
2464         pattern => '[A-Za-z0-9+\/]+={0,2}',
2465         format_description => 'Base64 encoded string',
2466         description => "Set SMBIOS1 SKU string.",
2467         optional => 1,
2468     },
2469     family => {
2470         type => 'string',
2471         pattern => '[A-Za-z0-9+\/]+={0,2}',
2472         format_description => 'Base64 encoded string',
2473         description => "Set SMBIOS1 family string.",
2474         optional => 1,
2475     },
2476     base64 => {
2477         type => 'boolean',
2478         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2479         optional => 1,
2480     },
2481 };
2482
2483 sub parse_smbios1 {
2484     my ($data) = @_;
2485
2486     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2487     warn $@ if $@;
2488     return $res;
2489 }
2490
2491 sub print_smbios1 {
2492     my ($smbios1) = @_;
2493     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2494 }
2495
2496 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2497
2498 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2499 sub verify_bootdisk {
2500     my ($value, $noerr) = @_;
2501
2502     return $value if is_valid_drivename($value);
2503
2504     return undef if $noerr;
2505
2506     die "invalid boot disk '$value'\n";
2507 }
2508
2509 sub parse_watchdog {
2510     my ($value) = @_;
2511
2512     return undef if !$value;
2513
2514     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2515     warn $@ if $@;
2516     return $res;
2517 }
2518
2519 sub parse_guest_agent {
2520     my ($value) = @_;
2521
2522     return {} if !defined($value->{agent});
2523
2524     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2525     warn $@ if $@;
2526
2527     # if the agent is disabled ignore the other potentially set properties
2528     return {} if !$res->{enabled};
2529     return $res;
2530 }
2531
2532 sub parse_vga {
2533     my ($value) = @_;
2534
2535     return {} if !$value;
2536     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2537     warn $@ if $@;
2538     return $res;
2539 }
2540
2541 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2542 sub verify_usb_device {
2543     my ($value, $noerr) = @_;
2544
2545     return $value if parse_usb_device($value);
2546
2547     return undef if $noerr;
2548
2549     die "unable to parse usb device\n";
2550 }
2551
2552 # add JSON properties for create and set function
2553 sub json_config_properties {
2554     my $prop = shift;
2555
2556     foreach my $opt (keys %$confdesc) {
2557         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2558         $prop->{$opt} = $confdesc->{$opt};
2559     }
2560
2561     return $prop;
2562 }
2563
2564 # return copy of $confdesc_cloudinit to generate documentation
2565 sub cloudinit_config_properties {
2566
2567     return dclone($confdesc_cloudinit);
2568 }
2569
2570 sub check_type {
2571     my ($key, $value) = @_;
2572
2573     die "unknown setting '$key'\n" if !$confdesc->{$key};
2574
2575     my $type = $confdesc->{$key}->{type};
2576
2577     if (!defined($value)) {
2578         die "got undefined value\n";
2579     }
2580
2581     if ($value =~ m/[\n\r]/) {
2582         die "property contains a line feed\n";
2583     }
2584
2585     if ($type eq 'boolean') {
2586         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2587         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2588         die "type check ('boolean') failed - got '$value'\n";
2589     } elsif ($type eq 'integer') {
2590         return int($1) if $value =~ m/^(\d+)$/;
2591         die "type check ('integer') failed - got '$value'\n";
2592     } elsif ($type eq 'number') {
2593         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2594         die "type check ('number') failed - got '$value'\n";
2595     } elsif ($type eq 'string') {
2596         if (my $fmt = $confdesc->{$key}->{format}) {
2597             PVE::JSONSchema::check_format($fmt, $value);
2598             return $value;
2599         }
2600         $value =~ s/^\"(.*)\"$/$1/;
2601         return $value;
2602     } else {
2603         die "internal error"
2604     }
2605 }
2606
2607 sub touch_config {
2608     my ($vmid) = @_;
2609
2610     my $conf = PVE::QemuConfig->config_file($vmid);
2611     utime undef, undef, $conf;
2612 }
2613
2614 sub destroy_vm {
2615     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2616
2617     my $conffile = PVE::QemuConfig->config_file($vmid);
2618
2619     my $conf = PVE::QemuConfig->load_config($vmid);
2620
2621     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2622
2623     if ($conf->{template}) {
2624         # check if any base image is still used by a linked clone
2625         foreach_drive($conf, sub {
2626                 my ($ds, $drive) = @_;
2627
2628                 return if drive_is_cdrom($drive);
2629
2630                 my $volid = $drive->{file};
2631
2632                 return if !$volid || $volid =~ m|^/|;
2633
2634                 die "base volume '$volid' is still in use by linked cloned\n"
2635                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2636
2637         });
2638     }
2639
2640     # only remove disks owned by this VM
2641     foreach_drive($conf, sub {
2642         my ($ds, $drive) = @_;
2643
2644         return if drive_is_cdrom($drive, 1);
2645
2646         my $volid = $drive->{file};
2647
2648         return if !$volid || $volid =~ m|^/|;
2649
2650         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2651         return if !$path || !$owner || ($owner != $vmid);
2652
2653         eval {
2654             PVE::Storage::vdisk_free($storecfg, $volid);
2655         };
2656         warn "Could not remove disk '$volid', check manually: $@" if $@;
2657
2658     });
2659
2660     if ($keep_empty_config) {
2661         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2662     } else {
2663         unlink $conffile;
2664     }
2665
2666     # also remove unused disk
2667     eval {
2668         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2669
2670         eval {
2671             PVE::Storage::foreach_volid($dl, sub {
2672                 my ($volid, $sid, $volname, $d) = @_;
2673                 PVE::Storage::vdisk_free($storecfg, $volid);
2674             });
2675         };
2676         warn $@ if $@;
2677
2678     };
2679     warn $@ if $@;
2680 }
2681
2682 sub parse_vm_config {
2683     my ($filename, $raw) = @_;
2684
2685     return undef if !defined($raw);
2686
2687     my $res = {
2688         digest => Digest::SHA::sha1_hex($raw),
2689         snapshots => {},
2690         pending => {},
2691     };
2692
2693     $filename =~ m|/qemu-server/(\d+)\.conf$|
2694         || die "got strange filename '$filename'";
2695
2696     my $vmid = $1;
2697
2698     my $conf = $res;
2699     my $descr;
2700     my $section = '';
2701
2702     my @lines = split(/\n/, $raw);
2703     foreach my $line (@lines) {
2704         next if $line =~ m/^\s*$/;
2705
2706         if ($line =~ m/^\[PENDING\]\s*$/i) {
2707             $section = 'pending';
2708             if (defined($descr)) {
2709                 $descr =~ s/\s+$//;
2710                 $conf->{description} = $descr;
2711             }
2712             $descr = undef;
2713             $conf = $res->{$section} = {};
2714             next;
2715
2716         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2717             $section = $1;
2718             if (defined($descr)) {
2719                 $descr =~ s/\s+$//;
2720                 $conf->{description} = $descr;
2721             }
2722             $descr = undef;
2723             $conf = $res->{snapshots}->{$section} = {};
2724             next;
2725         }
2726
2727         if ($line =~ m/^\#(.*)\s*$/) {
2728             $descr = '' if !defined($descr);
2729             $descr .= PVE::Tools::decode_text($1) . "\n";
2730             next;
2731         }
2732
2733         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2734             $descr = '' if !defined($descr);
2735             $descr .= PVE::Tools::decode_text($2);
2736         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2737             $conf->{snapstate} = $1;
2738         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2739             my $key = $1;
2740             my $value = $2;
2741             $conf->{$key} = $value;
2742         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2743             my $value = $1;
2744             if ($section eq 'pending') {
2745                 $conf->{delete} = $value; # we parse this later
2746             } else {
2747                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2748             }
2749         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2750             my $key = $1;
2751             my $value = $2;
2752             eval { $value = check_type($key, $value); };
2753             if ($@) {
2754                 warn "vm $vmid - unable to parse value of '$key' - $@";
2755             } else {
2756                 $key = 'ide2' if $key eq 'cdrom';
2757                 my $fmt = $confdesc->{$key}->{format};
2758                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2759                     my $v = parse_drive($key, $value);
2760                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2761                         $v->{file} = $volid;
2762                         $value = print_drive($vmid, $v);
2763                     } else {
2764                         warn "vm $vmid - unable to parse value of '$key'\n";
2765                         next;
2766                     }
2767                 }
2768
2769                 $conf->{$key} = $value;
2770             }
2771         }
2772     }
2773
2774     if (defined($descr)) {
2775         $descr =~ s/\s+$//;
2776         $conf->{description} = $descr;
2777     }
2778     delete $res->{snapstate}; # just to be sure
2779
2780     return $res;
2781 }
2782
2783 sub write_vm_config {
2784     my ($filename, $conf) = @_;
2785
2786     delete $conf->{snapstate}; # just to be sure
2787
2788     if ($conf->{cdrom}) {
2789         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2790         $conf->{ide2} = $conf->{cdrom};
2791         delete $conf->{cdrom};
2792     }
2793
2794     # we do not use 'smp' any longer
2795     if ($conf->{sockets}) {
2796         delete $conf->{smp};
2797     } elsif ($conf->{smp}) {
2798         $conf->{sockets} = $conf->{smp};
2799         delete $conf->{cores};
2800         delete $conf->{smp};
2801     }
2802
2803     my $used_volids = {};
2804
2805     my $cleanup_config = sub {
2806         my ($cref, $pending, $snapname) = @_;
2807
2808         foreach my $key (keys %$cref) {
2809             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2810                 $key eq 'snapstate' || $key eq 'pending';
2811             my $value = $cref->{$key};
2812             if ($key eq 'delete') {
2813                 die "propertry 'delete' is only allowed in [PENDING]\n"
2814                     if !$pending;
2815                 # fixme: check syntax?
2816                 next;
2817             }
2818             eval { $value = check_type($key, $value); };
2819             die "unable to parse value of '$key' - $@" if $@;
2820
2821             $cref->{$key} = $value;
2822
2823             if (!$snapname && is_valid_drivename($key)) {
2824                 my $drive = parse_drive($key, $value);
2825                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2826             }
2827         }
2828     };
2829
2830     &$cleanup_config($conf);
2831
2832     &$cleanup_config($conf->{pending}, 1);
2833
2834     foreach my $snapname (keys %{$conf->{snapshots}}) {
2835         die "internal error" if $snapname eq 'pending';
2836         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2837     }
2838
2839     # remove 'unusedX' settings if we re-add a volume
2840     foreach my $key (keys %$conf) {
2841         my $value = $conf->{$key};
2842         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2843             delete $conf->{$key};
2844         }
2845     }
2846
2847     my $generate_raw_config = sub {
2848         my ($conf, $pending) = @_;
2849
2850         my $raw = '';
2851
2852         # add description as comment to top of file
2853         if (defined(my $descr = $conf->{description})) {
2854             if ($descr) {
2855                 foreach my $cl (split(/\n/, $descr)) {
2856                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2857                 }
2858             } else {
2859                 $raw .= "#\n" if $pending;
2860             }
2861         }
2862
2863         foreach my $key (sort keys %$conf) {
2864             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2865             $raw .= "$key: $conf->{$key}\n";
2866         }
2867         return $raw;
2868     };
2869
2870     my $raw = &$generate_raw_config($conf);
2871
2872     if (scalar(keys %{$conf->{pending}})){
2873         $raw .= "\n[PENDING]\n";
2874         $raw .= &$generate_raw_config($conf->{pending}, 1);
2875     }
2876
2877     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2878         $raw .= "\n[$snapname]\n";
2879         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2880     }
2881
2882     return $raw;
2883 }
2884
2885 sub load_defaults {
2886
2887     my $res = {};
2888
2889     # we use static defaults from our JSON schema configuration
2890     foreach my $key (keys %$confdesc) {
2891         if (defined(my $default = $confdesc->{$key}->{default})) {
2892             $res->{$key} = $default;
2893         }
2894     }
2895
2896     return $res;
2897 }
2898
2899 sub config_list {
2900     my $vmlist = PVE::Cluster::get_vmlist();
2901     my $res = {};
2902     return $res if !$vmlist || !$vmlist->{ids};
2903     my $ids = $vmlist->{ids};
2904
2905     foreach my $vmid (keys %$ids) {
2906         my $d = $ids->{$vmid};
2907         next if !$d->{node} || $d->{node} ne $nodename;
2908         next if !$d->{type} || $d->{type} ne 'qemu';
2909         $res->{$vmid}->{exists} = 1;
2910     }
2911     return $res;
2912 }
2913
2914 # test if VM uses local resources (to prevent migration)
2915 sub check_local_resources {
2916     my ($conf, $noerr) = @_;
2917
2918     my @loc_res = ();
2919
2920     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2921     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2922
2923     push @loc_res, "ivshmem" if $conf->{ivshmem};
2924
2925     foreach my $k (keys %$conf) {
2926         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2927         # sockets are safe: they will recreated be on the target side post-migrate
2928         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2929         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2930     }
2931
2932     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2933
2934     return \@loc_res;
2935 }
2936
2937 # check if used storages are available on all nodes (use by migrate)
2938 sub check_storage_availability {
2939     my ($storecfg, $conf, $node) = @_;
2940
2941     foreach_drive($conf, sub {
2942         my ($ds, $drive) = @_;
2943
2944         my $volid = $drive->{file};
2945         return if !$volid;
2946
2947         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2948         return if !$sid;
2949
2950         # check if storage is available on both nodes
2951         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2952         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2953    });
2954 }
2955
2956 # list nodes where all VM images are available (used by has_feature API)
2957 sub shared_nodes {
2958     my ($conf, $storecfg) = @_;
2959
2960     my $nodelist = PVE::Cluster::get_nodelist();
2961     my $nodehash = { map { $_ => 1 } @$nodelist };
2962     my $nodename = PVE::INotify::nodename();
2963
2964     foreach_drive($conf, sub {
2965         my ($ds, $drive) = @_;
2966
2967         my $volid = $drive->{file};
2968         return if !$volid;
2969
2970         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2971         if ($storeid) {
2972             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2973             if ($scfg->{disable}) {
2974                 $nodehash = {};
2975             } elsif (my $avail = $scfg->{nodes}) {
2976                 foreach my $node (keys %$nodehash) {
2977                     delete $nodehash->{$node} if !$avail->{$node};
2978                 }
2979             } elsif (!$scfg->{shared}) {
2980                 foreach my $node (keys %$nodehash) {
2981                     delete $nodehash->{$node} if $node ne $nodename
2982                 }
2983             }
2984         }
2985     });
2986
2987     return $nodehash
2988 }
2989
2990 sub check_local_storage_availability {
2991     my ($conf, $storecfg) = @_;
2992
2993     my $nodelist = PVE::Cluster::get_nodelist();
2994     my $nodehash = { map { $_ => {} } @$nodelist };
2995
2996     foreach_drive($conf, sub {
2997         my ($ds, $drive) = @_;
2998
2999         my $volid = $drive->{file};
3000         return if !$volid;
3001
3002         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3003         if ($storeid) {
3004             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3005
3006             if ($scfg->{disable}) {
3007                 foreach my $node (keys %$nodehash) {
3008                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3009                 }
3010             } elsif (my $avail = $scfg->{nodes}) {
3011                 foreach my $node (keys %$nodehash) {
3012                     if (!$avail->{$node}) {
3013                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3014                     }
3015                 }
3016             }
3017         }
3018     });
3019
3020     foreach my $node (values %$nodehash) {
3021         if (my $unavail = $node->{unavailable_storages}) {
3022             $node->{unavailable_storages} = [ sort keys %$unavail ];
3023         }
3024     }
3025
3026     return $nodehash
3027 }
3028
3029 sub check_cmdline {
3030     my ($pidfile, $pid) = @_;
3031
3032     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3033     if (defined($fh)) {
3034         my $line = <$fh>;
3035         $fh->close;
3036         return undef if !$line;
3037         my @param = split(/\0/, $line);
3038
3039         my $cmd = $param[0];
3040         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3041
3042         for (my $i = 0; $i < scalar (@param); $i++) {
3043             my $p = $param[$i];
3044             next if !$p;
3045             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3046                 my $p = $param[$i+1];
3047                 return 1 if $p && ($p eq $pidfile);
3048                 return undef;
3049             }
3050         }
3051     }
3052     return undef;
3053 }
3054
3055 sub check_running {
3056     my ($vmid, $nocheck, $node) = @_;
3057
3058     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3059
3060     die "unable to find configuration file for VM $vmid - no such machine\n"
3061         if !$nocheck && ! -f $filename;
3062
3063     my $pidfile = pidfile_name($vmid);
3064
3065     if (my $fd = IO::File->new("<$pidfile")) {
3066         my $st = stat($fd);
3067         my $line = <$fd>;
3068         close($fd);
3069
3070         my $mtime = $st->mtime;
3071         if ($mtime > time()) {
3072             warn "file '$filename' modified in future\n";
3073         }
3074
3075         if ($line =~ m/^(\d+)$/) {
3076             my $pid = $1;
3077             if (check_cmdline($pidfile, $pid)) {
3078                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3079                     return $pid;
3080                 }
3081             }
3082         }
3083     }
3084
3085     return undef;
3086 }
3087
3088 sub vzlist {
3089
3090     my $vzlist = config_list();
3091
3092     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3093
3094     while (defined(my $de = $fd->read)) {
3095         next if $de !~ m/^(\d+)\.pid$/;
3096         my $vmid = $1;
3097         next if !defined($vzlist->{$vmid});
3098         if (my $pid = check_running($vmid)) {
3099             $vzlist->{$vmid}->{pid} = $pid;
3100         }
3101     }
3102
3103     return $vzlist;
3104 }
3105
3106 sub disksize {
3107     my ($storecfg, $conf) = @_;
3108
3109     my $bootdisk = $conf->{bootdisk};
3110     return undef if !$bootdisk;
3111     return undef if !is_valid_drivename($bootdisk);
3112
3113     return undef if !$conf->{$bootdisk};
3114
3115     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3116     return undef if !defined($drive);
3117
3118     return undef if drive_is_cdrom($drive);
3119
3120     my $volid = $drive->{file};
3121     return undef if !$volid;
3122
3123     return $drive->{size};
3124 }
3125
3126 our $vmstatus_return_properties = {
3127     vmid => get_standard_option('pve-vmid'),
3128     status => {
3129         description => "Qemu process status.",
3130         type => 'string',
3131         enum => ['stopped', 'running'],
3132     },
3133     maxmem => {
3134         description => "Maximum memory in bytes.",
3135         type => 'integer',
3136         optional => 1,
3137         renderer => 'bytes',
3138     },
3139     maxdisk => {
3140         description => "Root disk size in bytes.",
3141         type => 'integer',
3142         optional => 1,
3143         renderer => 'bytes',
3144     },
3145     name => {
3146         description => "VM name.",
3147         type => 'string',
3148         optional => 1,
3149     },
3150     qmpstatus => {
3151         description => "Qemu QMP agent status.",
3152         type => 'string',
3153         optional => 1,
3154     },
3155     pid => {
3156         description => "PID of running qemu process.",
3157         type => 'integer',
3158         optional => 1,
3159     },
3160     uptime => {
3161         description => "Uptime.",
3162         type => 'integer',
3163         optional => 1,
3164         renderer => 'duration',
3165     },
3166     cpus => {
3167         description => "Maximum usable CPUs.",
3168         type => 'number',
3169         optional => 1,
3170     },
3171     lock => {
3172         description => "The current config lock, if any.",
3173         type => 'string',
3174         optional => 1,
3175     }
3176 };
3177
3178 my $last_proc_pid_stat;
3179
3180 # get VM status information
3181 # This must be fast and should not block ($full == false)
3182 # We only query KVM using QMP if $full == true (this can be slow)
3183 sub vmstatus {
3184     my ($opt_vmid, $full) = @_;
3185
3186     my $res = {};
3187
3188     my $storecfg = PVE::Storage::config();
3189
3190     my $list = vzlist();
3191     my $defaults = load_defaults();
3192
3193     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3194
3195     my $cpucount = $cpuinfo->{cpus} || 1;
3196
3197     foreach my $vmid (keys %$list) {
3198         next if $opt_vmid && ($vmid ne $opt_vmid);
3199
3200         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3201         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3202
3203         my $d = { vmid => $vmid };
3204         $d->{pid} = $list->{$vmid}->{pid};
3205
3206         # fixme: better status?
3207         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3208
3209         my $size = disksize($storecfg, $conf);
3210         if (defined($size)) {
3211             $d->{disk} = 0; # no info available
3212             $d->{maxdisk} = $size;
3213         } else {
3214             $d->{disk} = 0;
3215             $d->{maxdisk} = 0;
3216         }
3217
3218         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3219             * ($conf->{cores} || $defaults->{cores});
3220         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3221         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3222
3223         $d->{name} = $conf->{name} || "VM $vmid";
3224         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3225             : $defaults->{memory}*(1024*1024);
3226
3227         if ($conf->{balloon}) {
3228             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3229             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3230                 : $defaults->{shares};
3231         }
3232
3233         $d->{uptime} = 0;
3234         $d->{cpu} = 0;
3235         $d->{mem} = 0;
3236
3237         $d->{netout} = 0;
3238         $d->{netin} = 0;
3239
3240         $d->{diskread} = 0;
3241         $d->{diskwrite} = 0;
3242
3243         $d->{template} = PVE::QemuConfig->is_template($conf);
3244
3245         $d->{serial} = 1 if conf_has_serial($conf);
3246         $d->{lock} = $conf->{lock} if $conf->{lock};
3247
3248         $res->{$vmid} = $d;
3249     }
3250
3251     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3252     foreach my $dev (keys %$netdev) {
3253         next if $dev !~ m/^tap([1-9]\d*)i/;
3254         my $vmid = $1;
3255         my $d = $res->{$vmid};
3256         next if !$d;
3257
3258         $d->{netout} += $netdev->{$dev}->{receive};
3259         $d->{netin} += $netdev->{$dev}->{transmit};
3260
3261         if ($full) {
3262             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3263             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3264         }
3265
3266     }
3267
3268     my $ctime = gettimeofday;
3269
3270     foreach my $vmid (keys %$list) {
3271
3272         my $d = $res->{$vmid};
3273         my $pid = $d->{pid};
3274         next if !$pid;
3275
3276         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3277         next if !$pstat; # not running
3278
3279         my $used = $pstat->{utime} + $pstat->{stime};
3280
3281         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3282
3283         if ($pstat->{vsize}) {
3284             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3285         }
3286
3287         my $old = $last_proc_pid_stat->{$pid};
3288         if (!$old) {
3289             $last_proc_pid_stat->{$pid} = {
3290                 time => $ctime,
3291                 used => $used,
3292                 cpu => 0,
3293             };
3294             next;
3295         }
3296
3297         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3298
3299         if ($dtime > 1000) {
3300             my $dutime = $used -  $old->{used};
3301
3302             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3303             $last_proc_pid_stat->{$pid} = {
3304                 time => $ctime,
3305                 used => $used,
3306                 cpu => $d->{cpu},
3307             };
3308         } else {
3309             $d->{cpu} = $old->{cpu};
3310         }
3311     }
3312
3313     return $res if !$full;
3314
3315     my $qmpclient = PVE::QMPClient->new();
3316
3317     my $ballooncb = sub {
3318         my ($vmid, $resp) = @_;
3319
3320         my $info = $resp->{'return'};
3321         return if !$info->{max_mem};
3322
3323         my $d = $res->{$vmid};
3324
3325         # use memory assigned to VM
3326         $d->{maxmem} = $info->{max_mem};
3327         $d->{balloon} = $info->{actual};
3328
3329         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3330             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3331             $d->{freemem} = $info->{free_mem};
3332         }
3333
3334         $d->{ballooninfo} = $info;
3335     };
3336
3337     my $blockstatscb = sub {
3338         my ($vmid, $resp) = @_;
3339         my $data = $resp->{'return'} || [];
3340         my $totalrdbytes = 0;
3341         my $totalwrbytes = 0;
3342
3343         for my $blockstat (@$data) {
3344             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3345             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3346
3347             $blockstat->{device} =~ s/drive-//;
3348             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3349         }
3350         $res->{$vmid}->{diskread} = $totalrdbytes;
3351         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3352     };
3353
3354     my $statuscb = sub {
3355         my ($vmid, $resp) = @_;
3356
3357         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3358         # this fails if ballon driver is not loaded, so this must be
3359         # the last commnand (following command are aborted if this fails).
3360         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3361
3362         my $status = 'unknown';
3363         if (!defined($status = $resp->{'return'}->{status})) {
3364             warn "unable to get VM status\n";
3365             return;
3366         }
3367
3368         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3369     };
3370
3371     foreach my $vmid (keys %$list) {
3372         next if $opt_vmid && ($vmid ne $opt_vmid);
3373         next if !$res->{$vmid}->{pid}; # not running
3374         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3375     }
3376
3377     $qmpclient->queue_execute(undef, 2);
3378
3379     foreach my $vmid (keys %$list) {
3380         next if $opt_vmid && ($vmid ne $opt_vmid);
3381         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3382     }
3383
3384     return $res;
3385 }
3386
3387 sub foreach_drive {
3388     my ($conf, $func, @param) = @_;
3389
3390     foreach my $ds (valid_drive_names()) {
3391         next if !defined($conf->{$ds});
3392
3393         my $drive = parse_drive($ds, $conf->{$ds});
3394         next if !$drive;
3395
3396         &$func($ds, $drive, @param);
3397     }
3398 }
3399
3400 sub foreach_volid {
3401     my ($conf, $func, @param) = @_;
3402
3403     my $volhash = {};
3404
3405     my $test_volid = sub {
3406         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3407
3408         return if !$volid;
3409
3410         $volhash->{$volid}->{cdrom} //= 1;
3411         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3412
3413         $volhash->{$volid}->{replicate} //= 0;
3414         $volhash->{$volid}->{replicate} = 1 if $replicate;
3415
3416         $volhash->{$volid}->{shared} //= 0;
3417         $volhash->{$volid}->{shared} = 1 if $shared;
3418
3419         $volhash->{$volid}->{referenced_in_config} //= 0;
3420         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3421
3422         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3423             if defined($snapname);
3424         $volhash->{$volid}->{size} = $size if $size;
3425     };
3426
3427     foreach_drive($conf, sub {
3428         my ($ds, $drive) = @_;
3429         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3430     });
3431
3432     foreach my $snapname (keys %{$conf->{snapshots}}) {
3433         my $snap = $conf->{snapshots}->{$snapname};
3434         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3435         foreach_drive($snap, sub {
3436             my ($ds, $drive) = @_;
3437             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3438         });
3439     }
3440
3441     foreach my $volid (keys %$volhash) {
3442         &$func($volid, $volhash->{$volid}, @param);
3443     }
3444 }
3445
3446 sub conf_has_serial {
3447     my ($conf) = @_;
3448
3449     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3450         if ($conf->{"serial$i"}) {
3451             return 1;
3452         }
3453     }
3454
3455     return 0;
3456 }
3457
3458 sub conf_has_audio {
3459     my ($conf, $id) = @_;
3460
3461     $id //= 0;
3462     my $audio = $conf->{"audio$id"};
3463     return undef if !defined($audio);
3464
3465     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3466     my $audiodriver = $audioproperties->{driver} // 'spice';
3467
3468     return {
3469         dev => $audioproperties->{device},
3470         dev_id => "audiodev$id",
3471         backend => $audiodriver,
3472         backend_id => "$audiodriver-backend${id}",
3473     };
3474 }
3475
3476 sub vga_conf_has_spice {
3477     my ($vga) = @_;
3478
3479     my $vgaconf = parse_vga($vga);
3480     my $vgatype = $vgaconf->{type};
3481     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3482
3483     return $1 || 1;
3484 }
3485
3486 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3487 sub get_host_arch() {
3488     $host_arch = (POSIX::uname())[4] if !$host_arch;
3489     return $host_arch;
3490 }
3491
3492 sub is_native($) {
3493     my ($arch) = @_;
3494     return get_host_arch() eq $arch;
3495 }
3496
3497 my $default_machines = {
3498     x86_64 => 'pc',
3499     aarch64 => 'virt',
3500 };
3501
3502 sub get_basic_machine_info {
3503     my ($conf, $forcemachine) = @_;
3504
3505     my $arch = $conf->{arch} // get_host_arch();
3506     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3507     return ($arch, $machine);
3508 }
3509
3510 sub get_ovmf_files($) {
3511     my ($arch) = @_;
3512
3513     my $ovmf = $OVMF->{$arch}
3514         or die "no OVMF images known for architecture '$arch'\n";
3515
3516     return @$ovmf;
3517 }
3518
3519 my $Arch2Qemu = {
3520     aarch64 => '/usr/bin/qemu-system-aarch64',
3521     x86_64 => '/usr/bin/qemu-system-x86_64',
3522 };
3523 sub get_command_for_arch($) {
3524     my ($arch) = @_;
3525     return '/usr/bin/kvm' if is_native($arch);
3526
3527     my $cmd = $Arch2Qemu->{$arch}
3528         or die "don't know how to emulate architecture '$arch'\n";
3529     return $cmd;
3530 }
3531
3532 sub get_cpu_options {
3533     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3534
3535     my $cpuFlags = [];
3536     my $ostype = $conf->{ostype};
3537
3538     my $cpu = $kvm ? "kvm64" : "qemu64";
3539     if ($arch eq 'aarch64') {
3540         $cpu = 'cortex-a57';
3541     }
3542     my $hv_vendor_id;
3543     if (my $cputype = $conf->{cpu}) {
3544         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3545             or die "Cannot parse cpu description: $cputype\n";
3546         $cpu = $cpuconf->{cputype};
3547         $kvm_off = 1 if $cpuconf->{hidden};
3548         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3549
3550         if (defined(my $flags = $cpuconf->{flags})) {
3551             push @$cpuFlags, split(";", $flags);
3552         }
3553     }
3554
3555     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3556
3557     push @$cpuFlags , '-x2apic'
3558         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3559
3560     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3561
3562     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3563
3564     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3565
3566         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3567         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3568     }
3569
3570     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3571
3572     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3573
3574     push @$cpuFlags, 'kvm=off' if $kvm_off;
3575
3576     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3577         push @$cpuFlags, "vendor=${cpu_vendor}"
3578             if $cpu_vendor ne 'default';
3579     } elsif ($arch ne 'aarch64') {
3580         die "internal error"; # should not happen
3581     }
3582
3583     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3584
3585     return ('-cpu', $cpu);
3586 }
3587
3588 sub config_to_command {
3589     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3590
3591     my $cmd = [];
3592     my $globalFlags = [];
3593     my $machineFlags = [];
3594     my $rtcFlags = [];
3595     my $devices = [];
3596     my $pciaddr = '';
3597     my $bridges = {};
3598     my $vernum = 0; # unknown
3599     my $ostype = $conf->{ostype};
3600     my $winversion = windows_version($ostype);
3601     my $kvm = $conf->{kvm};
3602
3603     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3604     my $kvm_binary = get_command_for_arch($arch);
3605     my $kvmver = kvm_user_version($kvm_binary);
3606     $kvm //= 1 if is_native($arch);
3607
3608     if ($kvm) {
3609         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3610             if !defined kvm_version();
3611     }
3612
3613     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3614         $vernum = $1*1000000+$2*1000;
3615     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3616         $vernum = $1*1000000+$2*1000+$3;
3617     }
3618
3619     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3620
3621     my $have_ovz = -f '/proc/vz/vestat';
3622
3623     my $q35 = machine_type_is_q35($conf);
3624     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3625     my $use_old_bios_files = undef;
3626     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3627
3628     my $cpuunits = defined($conf->{cpuunits}) ?
3629             $conf->{cpuunits} : $defaults->{cpuunits};
3630
3631     push @$cmd, $kvm_binary;
3632
3633     push @$cmd, '-id', $vmid;
3634
3635     my $vmname = $conf->{name} || "vm$vmid";
3636
3637     push @$cmd, '-name', $vmname;
3638
3639     my $use_virtio = 0;
3640
3641     my $qmpsocket = qmp_socket($vmid);
3642     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3643     push @$cmd, '-mon', "chardev=qmp,mode=control";
3644
3645     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3646         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3647         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3648     }
3649
3650     push @$cmd, '-pidfile' , pidfile_name($vmid);
3651
3652     push @$cmd, '-daemonize';
3653
3654     if ($conf->{smbios1}) {
3655         my $smbios_conf = parse_smbios1($conf->{smbios1});
3656         if ($smbios_conf->{base64}) {
3657             # Do not pass base64 flag to qemu
3658             delete $smbios_conf->{base64};
3659             my $smbios_string = "";
3660             foreach my $key (keys %$smbios_conf) {
3661                 my $value;
3662                 if ($key eq "uuid") {
3663                     $value = $smbios_conf->{uuid}
3664                 } else {
3665                     $value = decode_base64($smbios_conf->{$key});
3666                 }
3667                 # qemu accepts any binary data, only commas need escaping by double comma
3668                 $value =~ s/,/,,/g;
3669                 $smbios_string .= "," . $key . "=" . $value if $value;
3670             }
3671             push @$cmd, '-smbios', "type=1" . $smbios_string;
3672         } else {
3673             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3674         }
3675     }
3676
3677     if ($conf->{vmgenid}) {
3678         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3679     }
3680
3681     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3682     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3683         die "uefi base image not found\n" if ! -f $ovmf_code;
3684
3685         my $path;
3686         my $format;
3687         if (my $efidisk = $conf->{efidisk0}) {
3688             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3689             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3690             $format = $d->{format};
3691             if ($storeid) {
3692                 $path = PVE::Storage::path($storecfg, $d->{file});
3693                 if (!defined($format)) {
3694                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3695                     $format = qemu_img_format($scfg, $volname);
3696                 }
3697             } else {
3698                 $path = $d->{file};
3699                 die "efidisk format must be specified\n"
3700                     if !defined($format);
3701             }
3702         } else {
3703             warn "no efidisk configured! Using temporary efivars disk.\n";
3704             $path = "/tmp/$vmid-ovmf.fd";
3705             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3706             $format = 'raw';
3707         }
3708
3709         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3710         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3711     }
3712
3713     # load q35 config
3714     if ($q35) {
3715         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3716         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3717             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3718         } else {
3719             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3720         }
3721     }
3722
3723     # add usb controllers
3724     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3725     push @$devices, @usbcontrollers if @usbcontrollers;
3726     my $vga = parse_vga($conf->{vga});
3727
3728     my $qxlnum = vga_conf_has_spice($conf->{vga});
3729     $vga->{type} = 'qxl' if $qxlnum;
3730
3731     if (!$vga->{type}) {
3732         if ($arch eq 'aarch64') {
3733             $vga->{type} = 'virtio';
3734         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3735             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3736         } else {
3737             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3738         }
3739     }
3740
3741     # enable absolute mouse coordinates (needed by vnc)
3742     my $tablet;
3743     if (defined($conf->{tablet})) {
3744         $tablet = $conf->{tablet};
3745     } else {
3746         $tablet = $defaults->{tablet};
3747         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3748         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3749     }
3750
3751     if ($tablet) {
3752         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3753         my $kbd = print_keyboarddevice_full($conf, $arch);
3754         push @$devices, '-device', $kbd if defined($kbd);
3755     }
3756
3757     my $kvm_off = 0;
3758     my $gpu_passthrough;
3759
3760     # host pci devices
3761     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3762         my $d = parse_hostpci($conf->{"hostpci$i"});
3763         next if !$d;
3764
3765         my $pcie = $d->{pcie};
3766         if ($pcie) {
3767             die "q35 machine model is not enabled" if !$q35;
3768             # win7 wants to have the pcie devices directly on the pcie bus
3769             # instead of in the root port
3770             if ($winversion == 7) {
3771                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3772             } else {
3773                 $pciaddr = print_pcie_addr("hostpci$i");
3774             }
3775         } else {
3776             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3777         }
3778
3779         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3780         my $romfile = $d->{romfile};
3781
3782         my $xvga = '';
3783         if ($d->{'x-vga'}) {
3784             $xvga = ',x-vga=on';
3785             $kvm_off = 1;
3786             $vga->{type} = 'none' if !defined($conf->{vga});
3787             $gpu_passthrough = 1;
3788
3789             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3790                 $xvga = "";
3791             }
3792         }
3793         my $pcidevices = $d->{pciid};
3794         my $multifunction = 1 if @$pcidevices > 1;
3795         my $sysfspath;
3796         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3797             my $id = $pcidevices->[0]->{id};
3798             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3799             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3800         } elsif ($d->{mdev}) {
3801             warn "ignoring mediated device with multifunction device\n";
3802         }
3803
3804         my $j=0;
3805         foreach my $pcidevice (@$pcidevices) {
3806
3807             my $id = "hostpci$i";
3808             $id .= ".$j" if $multifunction;
3809             my $addr = $pciaddr;
3810             $addr .= ".$j" if $multifunction;
3811             my $devicestr = "vfio-pci";
3812             if ($sysfspath) {
3813                 $devicestr .= ",sysfsdev=$sysfspath";
3814             } else {
3815                 $devicestr .= ",host=$pcidevice->{id}";
3816             }
3817             $devicestr .= ",id=$id$addr";
3818
3819             if($j == 0){
3820                 $devicestr .= "$rombar$xvga";
3821                 $devicestr .= ",multifunction=on" if $multifunction;
3822                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3823             }
3824
3825             push @$devices, '-device', $devicestr;
3826             $j++;
3827         }
3828     }
3829
3830     # usb devices
3831     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3832     push @$devices, @usbdevices if @usbdevices;
3833     # serial devices
3834     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3835         if (my $path = $conf->{"serial$i"}) {
3836             if ($path eq 'socket') {
3837                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3838                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3839                 # On aarch64, serial0 is the UART device. Qemu only allows
3840                 # connecting UART devices via the '-serial' command line, as
3841                 # the device has a fixed slot on the hardware...
3842                 if ($arch eq 'aarch64' && $i == 0) {
3843                     push @$devices, '-serial', "chardev:serial$i";
3844                 } else {
3845                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3846                 }
3847             } else {
3848                 die "no such serial device\n" if ! -c $path;
3849                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3850                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3851             }
3852         }
3853     }
3854
3855     # parallel devices
3856     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3857         if (my $path = $conf->{"parallel$i"}) {
3858             die "no such parallel device\n" if ! -c $path;
3859             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3860             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3861             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3862         }
3863     }
3864
3865     if (my $audio = conf_has_audio($conf)) {
3866
3867         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3868
3869         my $id = $audio->{dev_id};
3870         if ($audio->{dev} eq 'AC97') {
3871             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3872         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3873             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3874             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3875             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3876         } else {
3877             die "unkown audio device '$audio->{dev}', implement me!";
3878         }
3879
3880         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3881     }
3882
3883     my $sockets = 1;
3884     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3885     $sockets = $conf->{sockets} if  $conf->{sockets};
3886
3887     my $cores = $conf->{cores} || 1;
3888
3889     my $maxcpus = $sockets * $cores;
3890
3891     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3892
3893     my $allowed_vcpus = $cpuinfo->{cpus};
3894
3895     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3896         if ($allowed_vcpus < $maxcpus);
3897
3898     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3899
3900         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3901         for (my $i = 2; $i <= $vcpus; $i++)  {
3902             my $cpustr = print_cpu_device($conf,$i);
3903             push @$cmd, '-device', $cpustr;
3904         }
3905
3906     } else {
3907
3908         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3909     }
3910     push @$cmd, '-nodefaults';
3911
3912     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3913
3914     my $bootindex_hash = {};
3915     my $i = 1;
3916     foreach my $o (split(//, $bootorder)) {
3917         $bootindex_hash->{$o} = $i*100;
3918         $i++;
3919     }
3920
3921     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3922
3923     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3924
3925     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3926
3927     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3928         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3929         my $socket = vnc_socket($vmid);
3930         push @$cmd,  '-vnc', "unix:$socket,password";
3931     } else {
3932         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3933         push @$cmd, '-nographic';
3934     }
3935
3936     # time drift fix
3937     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3938
3939     my $useLocaltime = $conf->{localtime};
3940
3941     if ($winversion >= 5) { # windows
3942         $useLocaltime = 1 if !defined($conf->{localtime});
3943
3944         # use time drift fix when acpi is enabled
3945         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3946             $tdf = 1 if !defined($conf->{tdf});
3947         }
3948     }
3949
3950     if ($winversion >= 6) {
3951         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3952         push @$cmd, '-no-hpet';
3953     }
3954
3955     push @$rtcFlags, 'driftfix=slew' if $tdf;
3956
3957     if (!$kvm) {
3958         push @$machineFlags, 'accel=tcg';
3959     }
3960
3961     if ($machine_type) {
3962         push @$machineFlags, "type=${machine_type}";
3963     }
3964
3965     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3966         push @$rtcFlags, "base=$conf->{startdate}";
3967     } elsif ($useLocaltime) {
3968         push @$rtcFlags, 'base=localtime';
3969     }
3970
3971     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3972
3973     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3974
3975     push @$cmd, '-S' if $conf->{freeze};
3976
3977     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3978
3979     if (parse_guest_agent($conf)->{enabled}) {
3980         my $qgasocket = qmp_socket($vmid, 1);
3981         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3982         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3983         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3984         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3985     }
3986
3987     my $spice_port;
3988
3989     if ($qxlnum) {
3990         if ($qxlnum > 1) {
3991             if ($winversion){
3992                 for(my $i = 1; $i < $qxlnum; $i++){
3993                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3994                 }
3995             } else {
3996                 # assume other OS works like Linux
3997                 my ($ram, $vram) = ("134217728", "67108864");
3998                 if ($vga->{memory}) {
3999                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4000                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4001                 }
4002                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4003                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4004             }
4005         }
4006
4007         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4008
4009         my $nodename = PVE::INotify::nodename();
4010         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4011         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4012         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4013         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4014         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4015
4016         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4017         if ($spice_enhancement->{foldersharing}) {
4018             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4019             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4020         }
4021
4022         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4023         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4024         push @$devices, '-spice', "$spice_opts";
4025
4026         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4027         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4028         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4029
4030     }
4031
4032     # enable balloon by default, unless explicitly disabled
4033     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4034         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4035         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4036     }
4037
4038     if ($conf->{watchdog}) {
4039         my $wdopts = parse_watchdog($conf->{watchdog});
4040         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4041         my $watchdog = $wdopts->{model} || 'i6300esb';
4042         push @$devices, '-device', "$watchdog$pciaddr";
4043         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4044     }
4045
4046     my $vollist = [];
4047     my $scsicontroller = {};
4048     my $ahcicontroller = {};
4049     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4050
4051     # Add iscsi initiator name if available
4052     if (my $initiator = get_initiator_name()) {
4053         push @$devices, '-iscsi', "initiator-name=$initiator";
4054     }
4055
4056     foreach_drive($conf, sub {
4057         my ($ds, $drive) = @_;
4058
4059         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4060             push @$vollist, $drive->{file};
4061         }
4062
4063         # ignore efidisk here, already added in bios/fw handling code above
4064         return if $drive->{interface} eq 'efidisk';
4065
4066         $use_virtio = 1 if $ds =~ m/^virtio/;
4067
4068         if (drive_is_cdrom ($drive)) {
4069             if ($bootindex_hash->{d}) {
4070                 $drive->{bootindex} = $bootindex_hash->{d};
4071                 $bootindex_hash->{d} += 1;
4072             }
4073         } else {
4074             if ($bootindex_hash->{c}) {
4075                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4076                 $bootindex_hash->{c} += 1;
4077             }
4078         }
4079
4080         if($drive->{interface} eq 'virtio'){
4081            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4082         }
4083
4084         if ($drive->{interface} eq 'scsi') {
4085
4086             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4087
4088             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4089             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4090
4091             my $iothread = '';
4092             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4093                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4094                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4095             } elsif ($drive->{iothread}) {
4096                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4097             }
4098
4099             my $queues = '';
4100             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4101                 $queues = ",num_queues=$drive->{queues}";
4102             }
4103
4104             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4105             $scsicontroller->{$controller}=1;
4106         }
4107
4108         if ($drive->{interface} eq 'sata') {
4109            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4110            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4111            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4112            $ahcicontroller->{$controller}=1;
4113         }
4114
4115         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4116         push @$devices, '-drive',$drive_cmd;
4117         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4118     });
4119
4120     for (my $i = 0; $i < $MAX_NETS; $i++) {
4121          next if !$conf->{"net$i"};
4122          my $d = parse_net($conf->{"net$i"});
4123          next if !$d;
4124
4125          $use_virtio = 1 if $d->{model} eq 'virtio';
4126
4127          if ($bootindex_hash->{n}) {
4128             $d->{bootindex} = $bootindex_hash->{n};
4129             $bootindex_hash->{n} += 1;
4130          }
4131
4132          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4133          push @$devices, '-netdev', $netdevfull;
4134
4135          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4136          push @$devices, '-device', $netdevicefull;
4137     }
4138
4139     if ($conf->{ivshmem}) {
4140         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4141
4142         my $bus;
4143         if ($q35) {
4144             $bus = print_pcie_addr("ivshmem");
4145         } else {
4146             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4147         }
4148
4149         my $ivshmem_name = $ivshmem->{name} // $vmid;
4150         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4151
4152         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4153         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4154     }
4155
4156     if (!$q35) {
4157         # add pci bridges
4158         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4159            $bridges->{1} = 1;
4160            $bridges->{2} = 1;
4161         }
4162
4163         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4164
4165         while (my ($k, $v) = each %$bridges) {
4166             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4167             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4168         }
4169     }
4170
4171     push @$cmd, @$devices;
4172     push @$cmd, '-rtc', join(',', @$rtcFlags)
4173         if scalar(@$rtcFlags);
4174     push @$cmd, '-machine', join(',', @$machineFlags)
4175         if scalar(@$machineFlags);
4176     push @$cmd, '-global', join(',', @$globalFlags)
4177         if scalar(@$globalFlags);
4178
4179     if (my $vmstate = $conf->{vmstate}) {
4180         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4181         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4182         push @$cmd, '-loadstate', $statepath;
4183     }
4184
4185     # add custom args
4186     if ($conf->{args}) {
4187         my $aa = PVE::Tools::split_args($conf->{args});
4188         push @$cmd, @$aa;
4189     }
4190
4191     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4192 }
4193
4194 sub vnc_socket {
4195     my ($vmid) = @_;
4196     return "${var_run_tmpdir}/$vmid.vnc";
4197 }
4198
4199 sub spice_port {
4200     my ($vmid) = @_;
4201
4202     my $res = vm_mon_cmd($vmid, 'query-spice');
4203
4204     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4205 }
4206
4207 sub qmp_socket {
4208     my ($vmid, $qga, $name) = @_;
4209     my $sockettype = $qga ? 'qga' : 'qmp';
4210     my $ext = $name ? '-'.$name : '';
4211     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4212 }
4213
4214 sub pidfile_name {
4215     my ($vmid) = @_;
4216     return "${var_run_tmpdir}/$vmid.pid";
4217 }
4218
4219 sub vm_devices_list {
4220     my ($vmid) = @_;
4221
4222     my $res = vm_mon_cmd($vmid, 'query-pci');
4223     my $devices_to_check = [];
4224     my $devices = {};
4225     foreach my $pcibus (@$res) {
4226         push @$devices_to_check, @{$pcibus->{devices}},
4227     }
4228
4229     while (@$devices_to_check) {
4230         my $to_check = [];
4231         for my $d (@$devices_to_check) {
4232             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4233             next if !$d->{'pci_bridge'};
4234
4235             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4236             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4237         }
4238         $devices_to_check = $to_check;
4239     }
4240
4241     my $resblock = vm_mon_cmd($vmid, 'query-block');
4242     foreach my $block (@$resblock) {
4243         if($block->{device} =~ m/^drive-(\S+)/){
4244                 $devices->{$1} = 1;
4245         }
4246     }
4247
4248     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4249     foreach my $mice (@$resmice) {
4250         if ($mice->{name} eq 'QEMU HID Tablet') {
4251             $devices->{tablet} = 1;
4252             last;
4253         }
4254     }
4255
4256     # for usb devices there is no query-usb
4257     # but we can iterate over the entries in
4258     # qom-list path=/machine/peripheral
4259     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4260     foreach my $per (@$resperipheral) {
4261         if ($per->{name} =~ m/^usb\d+$/) {
4262             $devices->{$per->{name}} = 1;
4263         }
4264     }
4265
4266     return $devices;
4267 }
4268
4269 sub vm_deviceplug {
4270     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4271
4272     my $q35 = machine_type_is_q35($conf);
4273
4274     my $devices_list = vm_devices_list($vmid);
4275     return 1 if defined($devices_list->{$deviceid});
4276
4277     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4278
4279     if ($deviceid eq 'tablet') {
4280
4281         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4282
4283     } elsif ($deviceid eq 'keyboard') {
4284
4285         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4286
4287     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4288
4289         die "usb hotplug currently not reliable\n";
4290         # since we can't reliably hot unplug all added usb devices
4291         # and usb passthrough disables live migration
4292         # we disable usb hotplugging for now
4293         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4294
4295     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4296
4297         qemu_iothread_add($vmid, $deviceid, $device);
4298
4299         qemu_driveadd($storecfg, $vmid, $device);
4300         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4301
4302         qemu_deviceadd($vmid, $devicefull);
4303         eval { qemu_deviceaddverify($vmid, $deviceid); };
4304         if (my $err = $@) {
4305             eval { qemu_drivedel($vmid, $deviceid); };
4306             warn $@ if $@;
4307             die $err;
4308         }
4309
4310     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4311
4312
4313         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4314         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4315         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4316
4317         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4318
4319         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4320             qemu_iothread_add($vmid, $deviceid, $device);
4321             $devicefull .= ",iothread=iothread-$deviceid";
4322         }
4323
4324         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4325             $devicefull .= ",num_queues=$device->{queues}";
4326         }
4327
4328         qemu_deviceadd($vmid, $devicefull);
4329         qemu_deviceaddverify($vmid, $deviceid);
4330
4331     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4332
4333         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4334         qemu_driveadd($storecfg, $vmid, $device);
4335
4336         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4337         eval { qemu_deviceadd($vmid, $devicefull); };
4338         if (my $err = $@) {
4339             eval { qemu_drivedel($vmid, $deviceid); };
4340             warn $@ if $@;
4341             die $err;
4342         }
4343
4344     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4345
4346         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4347
4348         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4349         my $use_old_bios_files = undef;
4350         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4351
4352         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4353         qemu_deviceadd($vmid, $netdevicefull);
4354         eval {
4355             qemu_deviceaddverify($vmid, $deviceid);
4356             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4357         };
4358         if (my $err = $@) {
4359             eval { qemu_netdevdel($vmid, $deviceid); };
4360             warn $@ if $@;
4361             die $err;
4362         }
4363
4364     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4365
4366         my $bridgeid = $2;
4367         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4368         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4369
4370         qemu_deviceadd($vmid, $devicefull);
4371         qemu_deviceaddverify($vmid, $deviceid);
4372
4373     } else {
4374         die "can't hotplug device '$deviceid'\n";
4375     }
4376
4377     return 1;
4378 }
4379
4380 # fixme: this should raise exceptions on error!
4381 sub vm_deviceunplug {
4382     my ($vmid, $conf, $deviceid) = @_;
4383
4384     my $devices_list = vm_devices_list($vmid);
4385     return 1 if !defined($devices_list->{$deviceid});
4386
4387     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4388
4389     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4390
4391         qemu_devicedel($vmid, $deviceid);
4392
4393     } elsif ($deviceid =~ m/^usb\d+$/) {
4394
4395         die "usb hotplug currently not reliable\n";
4396         # when unplugging usb devices this way,
4397         # there may be remaining usb controllers/hubs
4398         # so we disable it for now
4399         qemu_devicedel($vmid, $deviceid);
4400         qemu_devicedelverify($vmid, $deviceid);
4401
4402     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4403
4404         qemu_devicedel($vmid, $deviceid);
4405         qemu_devicedelverify($vmid, $deviceid);
4406         qemu_drivedel($vmid, $deviceid);
4407         qemu_iothread_del($conf, $vmid, $deviceid);
4408
4409     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4410
4411         qemu_devicedel($vmid, $deviceid);
4412         qemu_devicedelverify($vmid, $deviceid);
4413         qemu_iothread_del($conf, $vmid, $deviceid);
4414
4415     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4416
4417         qemu_devicedel($vmid, $deviceid);
4418         qemu_drivedel($vmid, $deviceid);
4419         qemu_deletescsihw($conf, $vmid, $deviceid);
4420
4421     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4422
4423         qemu_devicedel($vmid, $deviceid);
4424         qemu_devicedelverify($vmid, $deviceid);
4425         qemu_netdevdel($vmid, $deviceid);
4426
4427     } else {
4428         die "can't unplug device '$deviceid'\n";
4429     }
4430
4431     return 1;
4432 }
4433
4434 sub qemu_deviceadd {
4435     my ($vmid, $devicefull) = @_;
4436
4437     $devicefull = "driver=".$devicefull;
4438     my %options =  split(/[=,]/, $devicefull);
4439
4440     vm_mon_cmd($vmid, "device_add" , %options);
4441 }
4442
4443 sub qemu_devicedel {
4444     my ($vmid, $deviceid) = @_;
4445
4446     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4447 }
4448
4449 sub qemu_iothread_add {
4450     my($vmid, $deviceid, $device) = @_;
4451
4452     if ($device->{iothread}) {
4453         my $iothreads = vm_iothreads_list($vmid);
4454         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4455     }
4456 }
4457
4458 sub qemu_iothread_del {
4459     my($conf, $vmid, $deviceid) = @_;
4460
4461     my $confid = $deviceid;
4462     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4463         $confid = 'scsi' . $1;
4464     }
4465     my $device = parse_drive($confid, $conf->{$confid});
4466     if ($device->{iothread}) {
4467         my $iothreads = vm_iothreads_list($vmid);
4468         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4469     }
4470 }
4471
4472 sub qemu_objectadd {
4473     my($vmid, $objectid, $qomtype) = @_;
4474
4475     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4476
4477     return 1;
4478 }
4479
4480 sub qemu_objectdel {
4481     my($vmid, $objectid) = @_;
4482
4483     vm_mon_cmd($vmid, "object-del", id => $objectid);
4484
4485     return 1;
4486 }
4487
4488 sub qemu_driveadd {
4489     my ($storecfg, $vmid, $device) = @_;
4490
4491     my $drive = print_drive_full($storecfg, $vmid, $device);
4492     $drive =~ s/\\/\\\\/g;
4493     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4494
4495     # If the command succeeds qemu prints: "OK"
4496     return 1 if $ret =~ m/OK/s;
4497
4498     die "adding drive failed: $ret\n";
4499 }
4500
4501 sub qemu_drivedel {
4502     my($vmid, $deviceid) = @_;
4503
4504     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4505     $ret =~ s/^\s+//;
4506
4507     return 1 if $ret eq "";
4508
4509     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4510     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4511
4512     die "deleting drive $deviceid failed : $ret\n";
4513 }
4514
4515 sub qemu_deviceaddverify {
4516     my ($vmid, $deviceid) = @_;
4517
4518     for (my $i = 0; $i <= 5; $i++) {
4519          my $devices_list = vm_devices_list($vmid);
4520          return 1 if defined($devices_list->{$deviceid});
4521          sleep 1;
4522     }
4523
4524     die "error on hotplug device '$deviceid'\n";
4525 }
4526
4527
4528 sub qemu_devicedelverify {
4529     my ($vmid, $deviceid) = @_;
4530
4531     # need to verify that the device is correctly removed as device_del
4532     # is async and empty return is not reliable
4533
4534     for (my $i = 0; $i <= 5; $i++) {
4535          my $devices_list = vm_devices_list($vmid);
4536          return 1 if !defined($devices_list->{$deviceid});
4537          sleep 1;
4538     }
4539
4540     die "error on hot-unplugging device '$deviceid'\n";
4541 }
4542
4543 sub qemu_findorcreatescsihw {
4544     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4545
4546     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4547
4548     my $scsihwid="$controller_prefix$controller";
4549     my $devices_list = vm_devices_list($vmid);
4550
4551     if(!defined($devices_list->{$scsihwid})) {
4552         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4553     }
4554
4555     return 1;
4556 }
4557
4558 sub qemu_deletescsihw {
4559     my ($conf, $vmid, $opt) = @_;
4560
4561     my $device = parse_drive($opt, $conf->{$opt});
4562
4563     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4564         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4565         return 1;
4566     }
4567
4568     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4569
4570     my $devices_list = vm_devices_list($vmid);
4571     foreach my $opt (keys %{$devices_list}) {
4572         if (PVE::QemuServer::is_valid_drivename($opt)) {
4573             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4574             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4575                 return 1;
4576             }
4577         }
4578     }
4579
4580     my $scsihwid="scsihw$controller";
4581
4582     vm_deviceunplug($vmid, $conf, $scsihwid);
4583
4584     return 1;
4585 }
4586
4587 sub qemu_add_pci_bridge {
4588     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4589
4590     my $bridges = {};
4591
4592     my $bridgeid;
4593
4594     print_pci_addr($device, $bridges, $arch, $machine_type);
4595
4596     while (my ($k, $v) = each %$bridges) {
4597         $bridgeid = $k;
4598     }
4599     return 1 if !defined($bridgeid) || $bridgeid < 1;
4600
4601     my $bridge = "pci.$bridgeid";
4602     my $devices_list = vm_devices_list($vmid);
4603
4604     if (!defined($devices_list->{$bridge})) {
4605         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4606     }
4607
4608     return 1;
4609 }
4610
4611 sub qemu_set_link_status {
4612     my ($vmid, $device, $up) = @_;
4613
4614     vm_mon_cmd($vmid, "set_link", name => $device,
4615                up => $up ? JSON::true : JSON::false);
4616 }
4617
4618 sub qemu_netdevadd {
4619     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4620
4621     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4622     my %options =  split(/[=,]/, $netdev);
4623
4624     vm_mon_cmd($vmid, "netdev_add",  %options);
4625     return 1;
4626 }
4627
4628 sub qemu_netdevdel {
4629     my ($vmid, $deviceid) = @_;
4630
4631     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4632 }
4633
4634 sub qemu_usb_hotplug {
4635     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4636
4637     return if !$device;
4638
4639     # remove the old one first
4640     vm_deviceunplug($vmid, $conf, $deviceid);
4641
4642     # check if xhci controller is necessary and available
4643     if ($device->{usb3}) {
4644
4645         my $devicelist = vm_devices_list($vmid);
4646
4647         if (!$devicelist->{xhci}) {
4648             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4649             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4650         }
4651     }
4652     my $d = parse_usb_device($device->{host});
4653     $d->{usb3} = $device->{usb3};
4654
4655     # add the new one
4656     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4657 }
4658
4659 sub qemu_cpu_hotplug {
4660     my ($vmid, $conf, $vcpus) = @_;
4661
4662     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4663
4664     my $sockets = 1;
4665     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4666     $sockets = $conf->{sockets} if  $conf->{sockets};
4667     my $cores = $conf->{cores} || 1;
4668     my $maxcpus = $sockets * $cores;
4669
4670     $vcpus = $maxcpus if !$vcpus;
4671
4672     die "you can't add more vcpus than maxcpus\n"
4673         if $vcpus > $maxcpus;
4674
4675     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4676
4677     if ($vcpus < $currentvcpus) {
4678
4679         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4680
4681             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4682                 qemu_devicedel($vmid, "cpu$i");
4683                 my $retry = 0;
4684                 my $currentrunningvcpus = undef;
4685                 while (1) {
4686                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4687                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4688                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4689                     $retry++;
4690                     sleep 1;
4691                 }
4692                 #update conf after each succesfull cpu unplug
4693                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4694                 PVE::QemuConfig->write_config($vmid, $conf);
4695             }
4696         } else {
4697             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4698         }
4699
4700         return;
4701     }
4702
4703     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4704     die "vcpus in running vm does not match its configuration\n"
4705         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4706
4707     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4708
4709         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4710             my $cpustr = print_cpu_device($conf, $i);
4711             qemu_deviceadd($vmid, $cpustr);
4712
4713             my $retry = 0;
4714             my $currentrunningvcpus = undef;
4715             while (1) {
4716                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4717                 last if scalar(@{$currentrunningvcpus}) == $i;
4718                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4719                 sleep 1;
4720                 $retry++;
4721             }
4722             #update conf after each succesfull cpu hotplug
4723             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4724             PVE::QemuConfig->write_config($vmid, $conf);
4725         }
4726     } else {
4727
4728         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4729             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4730         }
4731     }
4732 }
4733
4734 sub qemu_block_set_io_throttle {
4735     my ($vmid, $deviceid,
4736         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4737         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4738         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4739         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4740
4741     return if !check_running($vmid) ;
4742
4743     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4744         bps => int($bps),
4745         bps_rd => int($bps_rd),
4746         bps_wr => int($bps_wr),
4747         iops => int($iops),
4748         iops_rd => int($iops_rd),
4749         iops_wr => int($iops_wr),
4750         bps_max => int($bps_max),
4751         bps_rd_max => int($bps_rd_max),
4752         bps_wr_max => int($bps_wr_max),
4753         iops_max => int($iops_max),
4754         iops_rd_max => int($iops_rd_max),
4755         iops_wr_max => int($iops_wr_max),
4756         bps_max_length => int($bps_max_length),
4757         bps_rd_max_length => int($bps_rd_max_length),
4758         bps_wr_max_length => int($bps_wr_max_length),
4759         iops_max_length => int($iops_max_length),
4760         iops_rd_max_length => int($iops_rd_max_length),
4761         iops_wr_max_length => int($iops_wr_max_length),
4762     );
4763
4764 }
4765
4766 # old code, only used to shutdown old VM after update
4767 sub __read_avail {
4768     my ($fh, $timeout) = @_;
4769
4770     my $sel = new IO::Select;
4771     $sel->add($fh);
4772
4773     my $res = '';
4774     my $buf;
4775
4776     my @ready;
4777     while (scalar (@ready = $sel->can_read($timeout))) {
4778         my $count;
4779         if ($count = $fh->sysread($buf, 8192)) {
4780             if ($buf =~ /^(.*)\(qemu\) $/s) {
4781                 $res .= $1;
4782                 last;
4783             } else {
4784                 $res .= $buf;
4785             }
4786         } else {
4787             if (!defined($count)) {
4788                 die "$!\n";
4789             }
4790             last;
4791         }
4792     }
4793
4794     die "monitor read timeout\n" if !scalar(@ready);
4795
4796     return $res;
4797 }
4798
4799 sub qemu_block_resize {
4800     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4801
4802     my $running = check_running($vmid);
4803
4804     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4805
4806     return if !$running;
4807
4808     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4809
4810 }
4811
4812 sub qemu_volume_snapshot {
4813     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4814
4815     my $running = check_running($vmid);
4816
4817     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4818         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4819     } else {
4820         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4821     }
4822 }
4823
4824 sub qemu_volume_snapshot_delete {
4825     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4826
4827     my $running = check_running($vmid);
4828
4829     if($running) {
4830
4831         $running = undef;
4832         my $conf = PVE::QemuConfig->load_config($vmid);
4833         foreach_drive($conf, sub {
4834             my ($ds, $drive) = @_;
4835             $running = 1 if $drive->{file} eq $volid;
4836         });
4837     }
4838
4839     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4840         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4841     } else {
4842         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4843     }
4844 }
4845
4846 sub set_migration_caps {
4847     my ($vmid) = @_;
4848
4849     my $cap_ref = [];
4850
4851     my $enabled_cap = {
4852         "auto-converge" => 1,
4853         "xbzrle" => 1,
4854         "x-rdma-pin-all" => 0,
4855         "zero-blocks" => 0,
4856         "compress" => 0
4857     };
4858
4859     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4860
4861     for my $supported_capability (@$supported_capabilities) {
4862         push @$cap_ref, {
4863             capability => $supported_capability->{capability},
4864             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4865         };
4866     }
4867
4868     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4869 }
4870
4871 my $fast_plug_option = {
4872     'lock' => 1,
4873     'name' => 1,
4874     'onboot' => 1,
4875     'shares' => 1,
4876     'startup' => 1,
4877     'description' => 1,
4878     'protection' => 1,
4879     'vmstatestorage' => 1,
4880     'hookscript' => 1,
4881 };
4882
4883 # hotplug changes in [PENDING]
4884 # $selection hash can be used to only apply specified options, for
4885 # example: { cores => 1 } (only apply changed 'cores')
4886 # $errors ref is used to return error messages
4887 sub vmconfig_hotplug_pending {
4888     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4889
4890     my $defaults = load_defaults();
4891     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4892
4893     # commit values which do not have any impact on running VM first
4894     # Note: those option cannot raise errors, we we do not care about
4895     # $selection and always apply them.
4896
4897     my $add_error = sub {
4898         my ($opt, $msg) = @_;
4899         $errors->{$opt} = "hotplug problem - $msg";
4900     };
4901
4902     my $changes = 0;
4903     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4904         if ($fast_plug_option->{$opt}) {
4905             $conf->{$opt} = $conf->{pending}->{$opt};
4906             delete $conf->{pending}->{$opt};
4907             $changes = 1;
4908         }
4909     }
4910
4911     if ($changes) {
4912         PVE::QemuConfig->write_config($vmid, $conf);
4913         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4914     }
4915
4916     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4917
4918     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4919     while (my ($opt, $force) = each %$pending_delete_hash) {
4920         next if $selection && !$selection->{$opt};
4921         eval {
4922             if ($opt eq 'hotplug') {
4923                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4924             } elsif ($opt eq 'tablet') {
4925                 die "skip\n" if !$hotplug_features->{usb};
4926                 if ($defaults->{tablet}) {
4927                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4928                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4929                         if $arch eq 'aarch64';
4930                 } else {
4931                     vm_deviceunplug($vmid, $conf, 'tablet');
4932                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4933                 }
4934             } elsif ($opt =~ m/^usb\d+/) {
4935                 die "skip\n";
4936                 # since we cannot reliably hot unplug usb devices
4937                 # we are disabling it
4938                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4939                 vm_deviceunplug($vmid, $conf, $opt);
4940             } elsif ($opt eq 'vcpus') {
4941                 die "skip\n" if !$hotplug_features->{cpu};
4942                 qemu_cpu_hotplug($vmid, $conf, undef);
4943             } elsif ($opt eq 'balloon') {
4944                 # enable balloon device is not hotpluggable
4945                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4946                 # here we reset the ballooning value to memory
4947                 my $balloon = $conf->{memory} || $defaults->{memory};
4948                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4949             } elsif ($fast_plug_option->{$opt}) {
4950                 # do nothing
4951             } elsif ($opt =~ m/^net(\d+)$/) {
4952                 die "skip\n" if !$hotplug_features->{network};
4953                 vm_deviceunplug($vmid, $conf, $opt);
4954             } elsif (is_valid_drivename($opt)) {
4955                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4956                 vm_deviceunplug($vmid, $conf, $opt);
4957                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4958             } elsif ($opt =~ m/^memory$/) {
4959                 die "skip\n" if !$hotplug_features->{memory};
4960                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4961             } elsif ($opt eq 'cpuunits') {
4962                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4963             } elsif ($opt eq 'cpulimit') {
4964                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4965             } else {
4966                 die "skip\n";
4967             }
4968         };
4969         if (my $err = $@) {
4970             &$add_error($opt, $err) if $err ne "skip\n";
4971         } else {
4972             # save new config if hotplug was successful
4973             delete $conf->{$opt};
4974             vmconfig_undelete_pending_option($conf, $opt);
4975             PVE::QemuConfig->write_config($vmid, $conf);
4976             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4977         }
4978     }
4979
4980     my $apply_pending_cloudinit;
4981     $apply_pending_cloudinit = sub {
4982         my ($key, $value) = @_;
4983         $apply_pending_cloudinit = sub {}; # once is enough
4984
4985         my @cloudinit_opts = keys %$confdesc_cloudinit;
4986         foreach my $opt (keys %{$conf->{pending}}) {
4987             next if !grep { $_ eq $opt } @cloudinit_opts;
4988             $conf->{$opt} = delete $conf->{pending}->{$opt};
4989         }
4990
4991         my $new_conf = { %$conf };
4992         $new_conf->{$key} = $value;
4993         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4994     };
4995
4996     foreach my $opt (keys %{$conf->{pending}}) {
4997         next if $selection && !$selection->{$opt};
4998         my $value = $conf->{pending}->{$opt};
4999         eval {
5000             if ($opt eq 'hotplug') {
5001                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5002             } elsif ($opt eq 'tablet') {
5003                 die "skip\n" if !$hotplug_features->{usb};
5004                 if ($value == 1) {
5005                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5006                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5007                         if $arch eq 'aarch64';
5008                 } elsif ($value == 0) {
5009                     vm_deviceunplug($vmid, $conf, 'tablet');
5010                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5011                 }
5012             } elsif ($opt =~ m/^usb\d+$/) {
5013                 die "skip\n";
5014                 # since we cannot reliably hot unplug usb devices
5015                 # we are disabling it
5016                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5017                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5018                 die "skip\n" if !$d;
5019                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5020             } elsif ($opt eq 'vcpus') {
5021                 die "skip\n" if !$hotplug_features->{cpu};
5022                 qemu_cpu_hotplug($vmid, $conf, $value);
5023             } elsif ($opt eq 'balloon') {
5024                 # enable/disable balloning device is not hotpluggable
5025                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5026                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5027                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5028
5029                 # allow manual ballooning if shares is set to zero
5030                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5031                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5032                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5033                 }
5034             } elsif ($opt =~ m/^net(\d+)$/) {
5035                 # some changes can be done without hotplug
5036                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5037                                     $vmid, $opt, $value, $arch, $machine_type);
5038             } elsif (is_valid_drivename($opt)) {
5039                 # some changes can be done without hotplug
5040                 my $drive = parse_drive($opt, $value);
5041                 if (drive_is_cloudinit($drive)) {
5042                     &$apply_pending_cloudinit($opt, $value);
5043                 }
5044                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5045                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5046             } elsif ($opt =~ m/^memory$/) { #dimms
5047                 die "skip\n" if !$hotplug_features->{memory};
5048                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5049             } elsif ($opt eq 'cpuunits') {
5050                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5051             } elsif ($opt eq 'cpulimit') {
5052                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5053                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5054             } else {
5055                 die "skip\n";  # skip non-hot-pluggable options
5056             }
5057         };
5058         if (my $err = $@) {
5059             &$add_error($opt, $err) if $err ne "skip\n";
5060         } else {
5061             # save new config if hotplug was successful
5062             $conf->{$opt} = $value;
5063             delete $conf->{pending}->{$opt};
5064             PVE::QemuConfig->write_config($vmid, $conf);
5065             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5066         }
5067     }
5068 }
5069
5070 sub try_deallocate_drive {
5071     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5072
5073     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5074         my $volid = $drive->{file};
5075         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5076             my $sid = PVE::Storage::parse_volume_id($volid);
5077             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5078
5079             # check if the disk is really unused
5080             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5081                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5082             PVE::Storage::vdisk_free($storecfg, $volid);
5083             return 1;
5084         } else {
5085             # If vm is not owner of this disk remove from config
5086             return 1;
5087         }
5088     }
5089
5090     return undef;
5091 }
5092
5093 sub vmconfig_delete_or_detach_drive {
5094     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5095
5096     my $drive = parse_drive($opt, $conf->{$opt});
5097
5098     my $rpcenv = PVE::RPCEnvironment::get();
5099     my $authuser = $rpcenv->get_user();
5100
5101     if ($force) {
5102         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5103         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5104     } else {
5105         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5106     }
5107 }
5108
5109 sub vmconfig_apply_pending {
5110     my ($vmid, $conf, $storecfg) = @_;
5111
5112     # cold plug
5113
5114     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5115     while (my ($opt, $force) = each %$pending_delete_hash) {
5116         die "internal error" if $opt =~ m/^unused/;
5117         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5118         if (!defined($conf->{$opt})) {
5119             vmconfig_undelete_pending_option($conf, $opt);
5120             PVE::QemuConfig->write_config($vmid, $conf);
5121         } elsif (is_valid_drivename($opt)) {
5122             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5123             vmconfig_undelete_pending_option($conf, $opt);
5124             delete $conf->{$opt};
5125             PVE::QemuConfig->write_config($vmid, $conf);
5126         } else {
5127             vmconfig_undelete_pending_option($conf, $opt);
5128             delete $conf->{$opt};
5129             PVE::QemuConfig->write_config($vmid, $conf);
5130         }
5131     }
5132
5133     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5134
5135     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5136         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5137
5138         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5139             # skip if nothing changed
5140         } elsif (is_valid_drivename($opt)) {
5141             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5142                 if defined($conf->{$opt});
5143             $conf->{$opt} = $conf->{pending}->{$opt};
5144         } else {
5145             $conf->{$opt} = $conf->{pending}->{$opt};
5146         }
5147
5148         delete $conf->{pending}->{$opt};
5149         PVE::QemuConfig->write_config($vmid, $conf);
5150     }
5151 }
5152
5153 my $safe_num_ne = sub {
5154     my ($a, $b) = @_;
5155
5156     return 0 if !defined($a) && !defined($b);
5157     return 1 if !defined($a);
5158     return 1 if !defined($b);
5159
5160     return $a != $b;
5161 };
5162
5163 my $safe_string_ne = sub {
5164     my ($a, $b) = @_;
5165
5166     return 0 if !defined($a) && !defined($b);
5167     return 1 if !defined($a);
5168     return 1 if !defined($b);
5169
5170     return $a ne $b;
5171 };
5172
5173 sub vmconfig_update_net {
5174     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5175
5176     my $newnet = parse_net($value);
5177
5178     if ($conf->{$opt}) {
5179         my $oldnet = parse_net($conf->{$opt});
5180
5181         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5182             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5183             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5184             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5185
5186             # for non online change, we try to hot-unplug
5187             die "skip\n" if !$hotplug;
5188             vm_deviceunplug($vmid, $conf, $opt);
5189         } else {
5190
5191             die "internal error" if $opt !~ m/net(\d+)/;
5192             my $iface = "tap${vmid}i$1";
5193
5194             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5195                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5196                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5197                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5198                 PVE::Network::tap_unplug($iface);
5199                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5200             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5201                 # Rate can be applied on its own but any change above needs to
5202                 # include the rate in tap_plug since OVS resets everything.
5203                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5204             }
5205
5206             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5207                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5208             }
5209
5210             return 1;
5211         }
5212     }
5213
5214     if ($hotplug) {
5215         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5216     } else {
5217         die "skip\n";
5218     }
5219 }
5220
5221 sub vmconfig_update_disk {
5222     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5223
5224     # fixme: do we need force?
5225
5226     my $drive = parse_drive($opt, $value);
5227
5228     if ($conf->{$opt}) {
5229
5230         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5231
5232             my $media = $drive->{media} || 'disk';
5233             my $oldmedia = $old_drive->{media} || 'disk';
5234             die "unable to change media type\n" if $media ne $oldmedia;
5235
5236             if (!drive_is_cdrom($old_drive)) {
5237
5238                 if ($drive->{file} ne $old_drive->{file}) {
5239
5240                     die "skip\n" if !$hotplug;
5241
5242                     # unplug and register as unused
5243                     vm_deviceunplug($vmid, $conf, $opt);
5244                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5245
5246                 } else {
5247                     # update existing disk
5248
5249                     # skip non hotpluggable value
5250                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5251                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5252                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5253                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5254                         die "skip\n";
5255                     }
5256
5257                     # apply throttle
5258                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5259                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5260                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5261                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5262                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5263                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5264                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5265                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5266                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5267                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5268                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5269                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5270                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5271                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5272                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5273                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5274                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5275                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5276
5277                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5278                                                    ($drive->{mbps} || 0)*1024*1024,
5279                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5280                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5281                                                    $drive->{iops} || 0,
5282                                                    $drive->{iops_rd} || 0,
5283                                                    $drive->{iops_wr} || 0,
5284                                                    ($drive->{mbps_max} || 0)*1024*1024,
5285                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5286                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5287                                                    $drive->{iops_max} || 0,
5288                                                    $drive->{iops_rd_max} || 0,
5289                                                    $drive->{iops_wr_max} || 0,
5290                                                    $drive->{bps_max_length} || 1,
5291                                                    $drive->{bps_rd_max_length} || 1,
5292                                                    $drive->{bps_wr_max_length} || 1,
5293                                                    $drive->{iops_max_length} || 1,
5294                                                    $drive->{iops_rd_max_length} || 1,
5295                                                    $drive->{iops_wr_max_length} || 1);
5296
5297                     }
5298
5299                     return 1;
5300                 }
5301
5302             } else { # cdrom
5303
5304                 if ($drive->{file} eq 'none') {
5305                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5306                     if (drive_is_cloudinit($old_drive)) {
5307                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5308                     }
5309                 } else {
5310                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5311                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5312                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5313                 }
5314
5315                 return 1;
5316             }
5317         }
5318     }
5319
5320     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5321     # hotplug new disks
5322     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5323     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5324 }
5325
5326 sub vm_start {
5327     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5328         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5329
5330     PVE::QemuConfig->lock_config($vmid, sub {
5331         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5332
5333         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5334
5335         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5336
5337         PVE::QemuConfig->check_lock($conf)
5338             if !($skiplock || $is_suspended);
5339
5340         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5341
5342         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5343             vmconfig_apply_pending($vmid, $conf, $storecfg);
5344             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5345         }
5346
5347         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5348
5349         my $defaults = load_defaults();
5350
5351         # set environment variable useful inside network script
5352         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5353
5354         my $local_volumes = {};
5355
5356         if ($targetstorage) {
5357             foreach_drive($conf, sub {
5358                 my ($ds, $drive) = @_;
5359
5360                 return if drive_is_cdrom($drive);
5361
5362                 my $volid = $drive->{file};
5363
5364                 return if !$volid;
5365
5366                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5367
5368                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5369                 return if $scfg->{shared};
5370                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5371             });
5372
5373             my $format = undef;
5374
5375             foreach my $opt (sort keys %$local_volumes) {
5376
5377                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5378                 my $drive = parse_drive($opt, $conf->{$opt});
5379
5380                 #if remote storage is specified, use default format
5381                 if ($targetstorage && $targetstorage ne "1") {
5382                     $storeid = $targetstorage;
5383                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5384                     $format = $defFormat;
5385                 } else {
5386                     #else we use same format than original
5387                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5388                     $format = qemu_img_format($scfg, $volid);
5389                 }
5390
5391                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5392                 my $newdrive = $drive;
5393                 $newdrive->{format} = $format;
5394                 $newdrive->{file} = $newvolid;
5395                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5396                 $local_volumes->{$opt} = $drivestr;
5397                 #pass drive to conf for command line
5398                 $conf->{$opt} = $drivestr;
5399             }
5400         }
5401
5402         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5403
5404         if ($is_suspended) {
5405             # enforce machine type on suspended vm to ensure HW compatibility
5406             $forcemachine = $conf->{runningmachine};
5407             print "Resuming suspended VM\n";
5408         }
5409
5410         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5411
5412         my $migrate_port = 0;
5413         my $migrate_uri;
5414         if ($statefile) {
5415             if ($statefile eq 'tcp') {
5416                 my $localip = "localhost";
5417                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5418                 my $nodename = PVE::INotify::nodename();
5419
5420                 if (!defined($migration_type)) {
5421                     if (defined($datacenterconf->{migration}->{type})) {
5422                         $migration_type = $datacenterconf->{migration}->{type};
5423                     } else {
5424                         $migration_type = 'secure';
5425                     }
5426                 }
5427
5428                 if ($migration_type eq 'insecure') {
5429                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5430                     if ($migrate_network_addr) {
5431                         $localip = $migrate_network_addr;
5432                     } else {
5433                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5434                     }
5435
5436                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5437                 }
5438
5439                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5440                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5441                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5442                 push @$cmd, '-incoming', $migrate_uri;
5443                 push @$cmd, '-S';
5444
5445             } elsif ($statefile eq 'unix') {
5446                 # should be default for secure migrations as a ssh TCP forward
5447                 # tunnel is not deterministic reliable ready and fails regurarly
5448                 # to set up in time, so use UNIX socket forwards
5449                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5450                 unlink $socket_addr;
5451
5452                 $migrate_uri = "unix:$socket_addr";
5453
5454                 push @$cmd, '-incoming', $migrate_uri;
5455                 push @$cmd, '-S';
5456
5457             } else {
5458                 push @$cmd, '-loadstate', $statefile;
5459             }
5460         } elsif ($paused) {
5461             push @$cmd, '-S';
5462         }
5463
5464         # host pci devices
5465         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5466           my $d = parse_hostpci($conf->{"hostpci$i"});
5467           next if !$d;
5468           my $pcidevices = $d->{pciid};
5469           foreach my $pcidevice (@$pcidevices) {
5470                 my $pciid = $pcidevice->{id};
5471
5472                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5473                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5474                 die "no pci device info for device '$pciid'\n" if !$info;
5475
5476                 if ($d->{mdev}) {
5477                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5478                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5479                 } else {
5480                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5481                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5482                     die "can't reset pci device '$pciid'\n"
5483                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5484                 }
5485           }
5486         }
5487
5488         PVE::Storage::activate_volumes($storecfg, $vollist);
5489
5490         eval {
5491             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5492                 outfunc => sub {}, errfunc => sub {});
5493         };
5494         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5495         # timeout should be more than enough here...
5496         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5497
5498         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5499                                                   : $defaults->{cpuunits};
5500
5501         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5502         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5503
5504         my %properties = (
5505             Slice => 'qemu.slice',
5506             KillMode => 'none',
5507             CPUShares => $cpuunits
5508         );
5509
5510         if (my $cpulimit = $conf->{cpulimit}) {
5511             $properties{CPUQuota} = int($cpulimit * 100);
5512         }
5513         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5514
5515         my $run_qemu = sub {
5516             PVE::Tools::run_fork sub {
5517                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5518                 run_command($cmd, %run_params);
5519             };
5520         };
5521
5522         if ($conf->{hugepages}) {
5523
5524             my $code = sub {
5525                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5526                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5527
5528                 PVE::QemuServer::Memory::hugepages_mount();
5529                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5530
5531                 eval { $run_qemu->() };
5532                 if (my $err = $@) {
5533                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5534                     die $err;
5535                 }
5536
5537                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5538             };
5539             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5540
5541         } else {
5542             eval { $run_qemu->() };
5543         }
5544
5545         if (my $err = $@) {
5546             # deactivate volumes if start fails
5547             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5548             die "start failed: $err";
5549         }
5550
5551         print "migration listens on $migrate_uri\n" if $migrate_uri;
5552
5553         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5554             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5555             warn $@ if $@;
5556         }
5557
5558         #start nbd server for storage migration
5559         if ($targetstorage) {
5560             my $nodename = PVE::INotify::nodename();
5561             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5562             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5563             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5564             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5565
5566             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5567
5568             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5569
5570             foreach my $opt (sort keys %$local_volumes) {
5571                 my $volid = $local_volumes->{$opt};
5572                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5573                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5574                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5575             }
5576         }
5577
5578         if ($migratedfrom) {
5579             eval {
5580                 set_migration_caps($vmid);
5581             };
5582             warn $@ if $@;
5583
5584             if ($spice_port) {
5585                 print "spice listens on port $spice_port\n";
5586                 if ($spice_ticket) {
5587                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5588                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5589                 }
5590             }
5591
5592         } else {
5593             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5594                 if !$statefile && $conf->{balloon};
5595
5596             foreach my $opt (keys %$conf) {
5597                 next if $opt !~  m/^net\d+$/;
5598                 my $nicconf = parse_net($conf->{$opt});
5599                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5600             }
5601         }
5602
5603         vm_mon_cmd_nocheck($vmid, 'qom-set',
5604                     path => "machine/peripheral/balloon0",
5605                     property => "guest-stats-polling-interval",
5606                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5607
5608         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5609             print "Resumed VM, removing state\n";
5610             delete $conf->@{qw(lock vmstate runningmachine)};
5611             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5612             PVE::Storage::vdisk_free($storecfg, $vmstate);
5613             PVE::QemuConfig->write_config($vmid, $conf);
5614         }
5615
5616         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5617     });
5618 }
5619
5620 sub vm_mon_cmd {
5621     my ($vmid, $execute, %params) = @_;
5622
5623     my $cmd = { execute => $execute, arguments => \%params };
5624     vm_qmp_command($vmid, $cmd);
5625 }
5626
5627 sub vm_mon_cmd_nocheck {
5628     my ($vmid, $execute, %params) = @_;
5629
5630     my $cmd = { execute => $execute, arguments => \%params };
5631     vm_qmp_command($vmid, $cmd, 1);
5632 }
5633
5634 sub vm_qmp_command {
5635     my ($vmid, $cmd, $nocheck) = @_;
5636
5637     my $res;
5638
5639     my $timeout;
5640     if ($cmd->{arguments}) {
5641         $timeout = delete $cmd->{arguments}->{timeout};
5642     }
5643
5644     eval {
5645         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5646         my $sname = qmp_socket($vmid);
5647         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5648             my $qmpclient = PVE::QMPClient->new();
5649
5650             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5651         } else {
5652             die "unable to open monitor socket\n";
5653         }
5654     };
5655     if (my $err = $@) {
5656         syslog("err", "VM $vmid qmp command failed - $err");
5657         die $err;
5658     }
5659
5660     return $res;
5661 }
5662
5663 sub vm_human_monitor_command {
5664     my ($vmid, $cmdline) = @_;
5665
5666     my $cmd = {
5667         execute => 'human-monitor-command',
5668         arguments => { 'command-line' => $cmdline},
5669     };
5670
5671     return vm_qmp_command($vmid, $cmd);
5672 }
5673
5674 sub vm_commandline {
5675     my ($storecfg, $vmid, $snapname) = @_;
5676
5677     my $conf = PVE::QemuConfig->load_config($vmid);
5678
5679     if ($snapname) {
5680         my $snapshot = $conf->{snapshots}->{$snapname};
5681         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5682
5683         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5684
5685         $conf = $snapshot;
5686     }
5687
5688     my $defaults = load_defaults();
5689
5690     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5691
5692     return PVE::Tools::cmd2string($cmd);
5693 }
5694
5695 sub vm_reset {
5696     my ($vmid, $skiplock) = @_;
5697
5698     PVE::QemuConfig->lock_config($vmid, sub {
5699
5700         my $conf = PVE::QemuConfig->load_config($vmid);
5701
5702         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5703
5704         vm_mon_cmd($vmid, "system_reset");
5705     });
5706 }
5707
5708 sub get_vm_volumes {
5709     my ($conf) = @_;
5710
5711     my $vollist = [];
5712     foreach_volid($conf, sub {
5713         my ($volid, $attr) = @_;
5714
5715         return if $volid =~ m|^/|;
5716
5717         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5718         return if !$sid;
5719
5720         push @$vollist, $volid;
5721     });
5722
5723     return $vollist;
5724 }
5725
5726 sub vm_stop_cleanup {
5727     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5728
5729     eval {
5730
5731         if (!$keepActive) {
5732             my $vollist = get_vm_volumes($conf);
5733             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5734         }
5735
5736         foreach my $ext (qw(mon qmp pid vnc qga)) {
5737             unlink "/var/run/qemu-server/${vmid}.$ext";
5738         }
5739
5740         if ($conf->{ivshmem}) {
5741             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5742             # just delete it for now, VMs which have this already open do not
5743             # are affected, but new VMs will get a separated one. If this
5744             # becomes an issue we either add some sort of ref-counting or just
5745             # add a "don't delete on stop" flag to the ivshmem format.
5746             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5747         }
5748
5749         foreach my $key (keys %$conf) {
5750             next if $key !~ m/^hostpci(\d+)$/;
5751             my $hostpciindex = $1;
5752             my $d = parse_hostpci($conf->{$key});
5753             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5754
5755             foreach my $pci (@{$d->{pciid}}) {
5756                 my $pciid = $pci->{id};
5757                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5758             }
5759         }
5760
5761         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5762     };
5763     warn $@ if $@; # avoid errors - just warn
5764 }
5765
5766 # Note: use $nockeck to skip tests if VM configuration file exists.
5767 # We need that when migration VMs to other nodes (files already moved)
5768 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5769 sub vm_stop {
5770     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5771
5772     $force = 1 if !defined($force) && !$shutdown;
5773
5774     if ($migratedfrom){
5775         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5776         kill 15, $pid if $pid;
5777         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5778         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5779         return;
5780     }
5781
5782     PVE::QemuConfig->lock_config($vmid, sub {
5783
5784         my $pid = check_running($vmid, $nocheck);
5785         return if !$pid;
5786
5787         my $conf;
5788         if (!$nocheck) {
5789             $conf = PVE::QemuConfig->load_config($vmid);
5790             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5791             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5792                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5793                 $timeout = $opts->{down} if $opts->{down};
5794             }
5795             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5796         }
5797
5798         eval {
5799             if ($shutdown) {
5800                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5801                     vm_qmp_command($vmid, {
5802                         execute => "guest-shutdown",
5803                         arguments => { timeout => $timeout }
5804                     }, $nocheck);
5805                 } else {
5806                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5807                 }
5808             } else {
5809                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5810             }
5811         };
5812         my $err = $@;
5813
5814         if (!$err) {
5815             $timeout = 60 if !defined($timeout);
5816
5817             my $count = 0;
5818             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5819                 $count++;
5820                 sleep 1;
5821             }
5822
5823             if ($count >= $timeout) {
5824                 if ($force) {
5825                     warn "VM still running - terminating now with SIGTERM\n";
5826                     kill 15, $pid;
5827                 } else {
5828                     die "VM quit/powerdown failed - got timeout\n";
5829                 }
5830             } else {
5831                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5832                 return;
5833             }
5834         } else {
5835             if ($force) {
5836                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5837                 kill 15, $pid;
5838             } else {
5839                 die "VM quit/powerdown failed\n";
5840             }
5841         }
5842
5843         # wait again
5844         $timeout = 10;
5845
5846         my $count = 0;
5847         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5848             $count++;
5849             sleep 1;
5850         }
5851
5852         if ($count >= $timeout) {
5853             warn "VM still running - terminating now with SIGKILL\n";
5854             kill 9, $pid;
5855             sleep 1;
5856         }
5857
5858         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5859    });
5860 }
5861
5862 sub vm_suspend {
5863     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5864
5865     my $conf;
5866     my $path;
5867     my $storecfg;
5868     my $vmstate;
5869
5870     PVE::QemuConfig->lock_config($vmid, sub {
5871
5872         $conf = PVE::QemuConfig->load_config($vmid);
5873
5874         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5875         PVE::QemuConfig->check_lock($conf)
5876             if !($skiplock || $is_backing_up);
5877
5878         die "cannot suspend to disk during backup\n"
5879             if $is_backing_up && $includestate;
5880
5881         if ($includestate) {
5882             $conf->{lock} = 'suspending';
5883             my $date = strftime("%Y-%m-%d", localtime(time()));
5884             $storecfg = PVE::Storage::config();
5885             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5886             $path = PVE::Storage::path($storecfg, $vmstate);
5887             PVE::QemuConfig->write_config($vmid, $conf);
5888         } else {
5889             vm_mon_cmd($vmid, "stop");
5890         }
5891     });
5892
5893     if ($includestate) {
5894         # save vm state
5895         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5896
5897         eval {
5898             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5899             for(;;) {
5900                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5901                 if (!$state->{status}) {
5902                     die "savevm not active\n";
5903                 } elsif ($state->{status} eq 'active') {
5904                     sleep(1);
5905                     next;
5906                 } elsif ($state->{status} eq 'completed') {
5907                     print "State saved, quitting\n";
5908                     last;
5909                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5910                     die "query-savevm failed with error '$state->{error}'\n"
5911                 } else {
5912                     die "query-savevm returned status '$state->{status}'\n";
5913                 }
5914             }
5915         };
5916         my $err = $@;
5917
5918         PVE::QemuConfig->lock_config($vmid, sub {
5919             $conf = PVE::QemuConfig->load_config($vmid);
5920             if ($err) {
5921                 # cleanup, but leave suspending lock, to indicate something went wrong
5922                 eval {
5923                     vm_mon_cmd($vmid, "savevm-end");
5924                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5925                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5926                     delete $conf->@{qw(vmstate runningmachine)};
5927                     PVE::QemuConfig->write_config($vmid, $conf);
5928                 };
5929                 warn $@ if $@;
5930                 die $err;
5931             }
5932
5933             die "lock changed unexpectedly\n"
5934                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5935
5936             vm_qmp_command($vmid, { execute => "quit" });
5937             $conf->{lock} = 'suspended';
5938             PVE::QemuConfig->write_config($vmid, $conf);
5939         });
5940     }
5941 }
5942
5943 sub vm_resume {
5944     my ($vmid, $skiplock, $nocheck) = @_;
5945
5946     PVE::QemuConfig->lock_config($vmid, sub {
5947         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5948         my $res = $vm_mon_cmd->($vmid, 'query-status');
5949         my $resume_cmd = 'cont';
5950
5951         if ($res->{status} && $res->{status} eq 'suspended') {
5952             $resume_cmd = 'system_wakeup';
5953         }
5954
5955         if (!$nocheck) {
5956
5957             my $conf = PVE::QemuConfig->load_config($vmid);
5958
5959             PVE::QemuConfig->check_lock($conf)
5960                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5961         }
5962
5963         $vm_mon_cmd->($vmid, $resume_cmd);
5964     });
5965 }
5966
5967 sub vm_sendkey {
5968     my ($vmid, $skiplock, $key) = @_;
5969
5970     PVE::QemuConfig->lock_config($vmid, sub {
5971
5972         my $conf = PVE::QemuConfig->load_config($vmid);
5973
5974         # there is no qmp command, so we use the human monitor command
5975         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5976         die $res if $res ne '';
5977     });
5978 }
5979
5980 sub vm_destroy {
5981     my ($storecfg, $vmid, $skiplock) = @_;
5982
5983     PVE::QemuConfig->lock_config($vmid, sub {
5984
5985         my $conf = PVE::QemuConfig->load_config($vmid);
5986
5987         if (!check_running($vmid)) {
5988             destroy_vm($storecfg, $vmid, undef, $skiplock);
5989         } else {
5990             die "VM $vmid is running - destroy failed\n";
5991         }
5992     });
5993 }
5994
5995 # vzdump restore implementaion
5996
5997 sub tar_archive_read_firstfile {
5998     my $archive = shift;
5999
6000     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6001
6002     # try to detect archive type first
6003     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6004         die "unable to open file '$archive'\n";
6005     my $firstfile = <$fh>;
6006     kill 15, $pid;
6007     close $fh;
6008
6009     die "ERROR: archive contaions no data\n" if !$firstfile;
6010     chomp $firstfile;
6011
6012     return $firstfile;
6013 }
6014
6015 sub tar_restore_cleanup {
6016     my ($storecfg, $statfile) = @_;
6017
6018     print STDERR "starting cleanup\n";
6019
6020     if (my $fd = IO::File->new($statfile, "r")) {
6021         while (defined(my $line = <$fd>)) {
6022             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6023                 my $volid = $2;
6024                 eval {
6025                     if ($volid =~ m|^/|) {
6026                         unlink $volid || die 'unlink failed\n';
6027                     } else {
6028                         PVE::Storage::vdisk_free($storecfg, $volid);
6029                     }
6030                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6031                 };
6032                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6033             } else {
6034                 print STDERR "unable to parse line in statfile - $line";
6035             }
6036         }
6037         $fd->close();
6038     }
6039 }
6040
6041 sub restore_archive {
6042     my ($archive, $vmid, $user, $opts) = @_;
6043
6044     my $format = $opts->{format};
6045     my $comp;
6046
6047     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6048         $format = 'tar' if !$format;
6049         $comp = 'gzip';
6050     } elsif ($archive =~ m/\.tar$/) {
6051         $format = 'tar' if !$format;
6052     } elsif ($archive =~ m/.tar.lzo$/) {
6053         $format = 'tar' if !$format;
6054         $comp = 'lzop';
6055     } elsif ($archive =~ m/\.vma$/) {
6056         $format = 'vma' if !$format;
6057     } elsif ($archive =~ m/\.vma\.gz$/) {
6058         $format = 'vma' if !$format;
6059         $comp = 'gzip';
6060     } elsif ($archive =~ m/\.vma\.lzo$/) {
6061         $format = 'vma' if !$format;
6062         $comp = 'lzop';
6063     } else {
6064         $format = 'vma' if !$format; # default
6065     }
6066
6067     # try to detect archive format
6068     if ($format eq 'tar') {
6069         return restore_tar_archive($archive, $vmid, $user, $opts);
6070     } else {
6071         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6072     }
6073 }
6074
6075 sub restore_update_config_line {
6076     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6077
6078     return if $line =~ m/^\#qmdump\#/;
6079     return if $line =~ m/^\#vzdump\#/;
6080     return if $line =~ m/^lock:/;
6081     return if $line =~ m/^unused\d+:/;
6082     return if $line =~ m/^parent:/;
6083
6084     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6085     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6086         # try to convert old 1.X settings
6087         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6088         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6089             my ($model, $macaddr) = split(/\=/, $devconfig);
6090             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6091             my $net = {
6092                 model => $model,
6093                 bridge => "vmbr$ind",
6094                 macaddr => $macaddr,
6095             };
6096             my $netstr = print_net($net);
6097
6098             print $outfd "net$cookie->{netcount}: $netstr\n";
6099             $cookie->{netcount}++;
6100         }
6101     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6102         my ($id, $netstr) = ($1, $2);
6103         my $net = parse_net($netstr);
6104         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6105         $netstr = print_net($net);
6106         print $outfd "$id: $netstr\n";
6107     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6108         my $virtdev = $1;
6109         my $value = $3;
6110         my $di = parse_drive($virtdev, $value);
6111         if (defined($di->{backup}) && !$di->{backup}) {
6112             print $outfd "#$line";
6113         } elsif ($map->{$virtdev}) {
6114             delete $di->{format}; # format can change on restore
6115             $di->{file} = $map->{$virtdev};
6116             $value = print_drive($vmid, $di);
6117             print $outfd "$virtdev: $value\n";
6118         } else {
6119             print $outfd $line;
6120         }
6121     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6122         my $vmgenid = $1;
6123         if ($vmgenid ne '0') {
6124             # always generate a new vmgenid if there was a valid one setup
6125             $vmgenid = generate_uuid();
6126         }
6127         print $outfd "vmgenid: $vmgenid\n";
6128     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6129         my ($uuid, $uuid_str);
6130         UUID::generate($uuid);
6131         UUID::unparse($uuid, $uuid_str);
6132         my $smbios1 = parse_smbios1($2);
6133         $smbios1->{uuid} = $uuid_str;
6134         print $outfd $1.print_smbios1($smbios1)."\n";
6135     } else {
6136         print $outfd $line;
6137     }
6138 }
6139
6140 sub scan_volids {
6141     my ($cfg, $vmid) = @_;
6142
6143     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6144
6145     my $volid_hash = {};
6146     foreach my $storeid (keys %$info) {
6147         foreach my $item (@{$info->{$storeid}}) {
6148             next if !($item->{volid} && $item->{size});
6149             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6150             $volid_hash->{$item->{volid}} = $item;
6151         }
6152     }
6153
6154     return $volid_hash;
6155 }
6156
6157 sub is_volume_in_use {
6158     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6159
6160     my $path = PVE::Storage::path($storecfg, $volid);
6161
6162     my $scan_config = sub {
6163         my ($cref, $snapname) = @_;
6164
6165         foreach my $key (keys %$cref) {
6166             my $value = $cref->{$key};
6167             if (is_valid_drivename($key)) {
6168                 next if $skip_drive && $key eq $skip_drive;
6169                 my $drive = parse_drive($key, $value);
6170                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6171                 return 1 if $volid eq $drive->{file};
6172                 if ($drive->{file} =~ m!^/!) {
6173                     return 1 if $drive->{file} eq $path;
6174                 } else {
6175                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6176                     next if !$storeid;
6177                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6178                     next if !$scfg;
6179                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6180                 }
6181             }
6182         }
6183
6184         return 0;
6185     };
6186
6187     return 1 if &$scan_config($conf);
6188
6189     undef $skip_drive;
6190
6191     foreach my $snapname (keys %{$conf->{snapshots}}) {
6192         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6193     }
6194
6195     return 0;
6196 }
6197
6198 sub update_disksize {
6199     my ($vmid, $conf, $volid_hash) = @_;
6200
6201     my $changes;
6202     my $prefix = "VM $vmid:";
6203
6204     # used and unused disks
6205     my $referenced = {};
6206
6207     # Note: it is allowed to define multiple storages with same path (alias), so
6208     # we need to check both 'volid' and real 'path' (two different volid can point
6209     # to the same path).
6210
6211     my $referencedpath = {};
6212
6213     # update size info
6214     foreach my $opt (keys %$conf) {
6215         if (is_valid_drivename($opt)) {
6216             my $drive = parse_drive($opt, $conf->{$opt});
6217             my $volid = $drive->{file};
6218             next if !$volid;
6219
6220             $referenced->{$volid} = 1;
6221             if ($volid_hash->{$volid} &&
6222                 (my $path = $volid_hash->{$volid}->{path})) {
6223                 $referencedpath->{$path} = 1;
6224             }
6225
6226             next if drive_is_cdrom($drive);
6227             next if !$volid_hash->{$volid};
6228
6229             $drive->{size} = $volid_hash->{$volid}->{size};
6230             my $new = print_drive($vmid, $drive);
6231             if ($new ne $conf->{$opt}) {
6232                 $changes = 1;
6233                 $conf->{$opt} = $new;
6234                 print "$prefix update disk '$opt' information.\n";
6235             }
6236         }
6237     }
6238
6239     # remove 'unusedX' entry if volume is used
6240     foreach my $opt (keys %$conf) {
6241         next if $opt !~ m/^unused\d+$/;
6242         my $volid = $conf->{$opt};
6243         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6244         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6245             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6246             $changes = 1;
6247             delete $conf->{$opt};
6248         }
6249
6250         $referenced->{$volid} = 1;
6251         $referencedpath->{$path} = 1 if $path;
6252     }
6253
6254     foreach my $volid (sort keys %$volid_hash) {
6255         next if $volid =~ m/vm-$vmid-state-/;
6256         next if $referenced->{$volid};
6257         my $path = $volid_hash->{$volid}->{path};
6258         next if !$path; # just to be sure
6259         next if $referencedpath->{$path};
6260         $changes = 1;
6261         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6262         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6263         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6264     }
6265
6266     return $changes;
6267 }
6268
6269 sub rescan {
6270     my ($vmid, $nolock, $dryrun) = @_;
6271
6272     my $cfg = PVE::Storage::config();
6273
6274     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6275     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6276     foreach my $stor (keys %{$cfg->{ids}}) {
6277         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6278     }
6279
6280     print "rescan volumes...\n";
6281     my $volid_hash = scan_volids($cfg, $vmid);
6282
6283     my $updatefn =  sub {
6284         my ($vmid) = @_;
6285
6286         my $conf = PVE::QemuConfig->load_config($vmid);
6287
6288         PVE::QemuConfig->check_lock($conf);
6289
6290         my $vm_volids = {};
6291         foreach my $volid (keys %$volid_hash) {
6292             my $info = $volid_hash->{$volid};
6293             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6294         }
6295
6296         my $changes = update_disksize($vmid, $conf, $vm_volids);
6297
6298         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6299     };
6300
6301     if (defined($vmid)) {
6302         if ($nolock) {
6303             &$updatefn($vmid);
6304         } else {
6305             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6306         }
6307     } else {
6308         my $vmlist = config_list();
6309         foreach my $vmid (keys %$vmlist) {
6310             if ($nolock) {
6311                 &$updatefn($vmid);
6312             } else {
6313                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6314             }
6315         }
6316     }
6317 }
6318
6319 sub restore_vma_archive {
6320     my ($archive, $vmid, $user, $opts, $comp) = @_;
6321
6322     my $readfrom = $archive;
6323
6324     my $cfg = PVE::Storage::config();
6325     my $commands = [];
6326     my $bwlimit = $opts->{bwlimit};
6327
6328     my $dbg_cmdstring = '';
6329     my $add_pipe = sub {
6330         my ($cmd) = @_;
6331         push @$commands, $cmd;
6332         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6333         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6334         $readfrom = '-';
6335     };
6336
6337     my $input = undef;
6338     if ($archive eq '-') {
6339         $input = '<&STDIN';
6340     } else {
6341         # If we use a backup from a PVE defined storage we also consider that
6342         # storage's rate limit:
6343         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6344         if (defined($volid)) {
6345             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6346             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6347             if ($readlimit) {
6348                 print STDERR "applying read rate limit: $readlimit\n";
6349                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6350                 $add_pipe->($cstream);
6351             }
6352         }
6353     }
6354
6355     if ($comp) {
6356         my $cmd;
6357         if ($comp eq 'gzip') {
6358             $cmd = ['zcat', $readfrom];
6359         } elsif ($comp eq 'lzop') {
6360             $cmd = ['lzop', '-d', '-c', $readfrom];
6361         } else {
6362             die "unknown compression method '$comp'\n";
6363         }
6364         $add_pipe->($cmd);
6365     }
6366
6367     my $tmpdir = "/var/tmp/vzdumptmp$$";
6368     rmtree $tmpdir;
6369
6370     # disable interrupts (always do cleanups)
6371     local $SIG{INT} =
6372         local $SIG{TERM} =
6373         local $SIG{QUIT} =
6374         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6375
6376     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6377     POSIX::mkfifo($mapfifo, 0600);
6378     my $fifofh;
6379
6380     my $openfifo = sub {
6381         open($fifofh, '>', $mapfifo) || die $!;
6382     };
6383
6384     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6385
6386     my $oldtimeout;
6387     my $timeout = 5;
6388
6389     my $devinfo = {};
6390
6391     my $rpcenv = PVE::RPCEnvironment::get();
6392
6393     my $conffile = PVE::QemuConfig->config_file($vmid);
6394     my $tmpfn = "$conffile.$$.tmp";
6395
6396     # Note: $oldconf is undef if VM does not exists
6397     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6398     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6399
6400     my %storage_limits;
6401
6402     my $print_devmap = sub {
6403         my $virtdev_hash = {};
6404
6405         my $cfgfn = "$tmpdir/qemu-server.conf";
6406
6407         # we can read the config - that is already extracted
6408         my $fh = IO::File->new($cfgfn, "r") ||
6409             "unable to read qemu-server.conf - $!\n";
6410
6411         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6412         if (-f $fwcfgfn) {
6413             my $pve_firewall_dir = '/etc/pve/firewall';
6414             mkdir $pve_firewall_dir; # make sure the dir exists
6415             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6416         }
6417
6418         while (defined(my $line = <$fh>)) {
6419             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6420                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6421                 die "archive does not contain data for drive '$virtdev'\n"
6422                     if !$devinfo->{$devname};
6423                 if (defined($opts->{storage})) {
6424                     $storeid = $opts->{storage} || 'local';
6425                 } elsif (!$storeid) {
6426                     $storeid = 'local';
6427                 }
6428                 $format = 'raw' if !$format;
6429                 $devinfo->{$devname}->{devname} = $devname;
6430                 $devinfo->{$devname}->{virtdev} = $virtdev;
6431                 $devinfo->{$devname}->{format} = $format;
6432                 $devinfo->{$devname}->{storeid} = $storeid;
6433
6434                 # check permission on storage
6435                 my $pool = $opts->{pool}; # todo: do we need that?
6436                 if ($user ne 'root@pam') {
6437                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6438                 }
6439
6440                 $storage_limits{$storeid} = $bwlimit;
6441
6442                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6443             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6444                 my $virtdev = $1;
6445                 my $drive = parse_drive($virtdev, $2);
6446                 if (drive_is_cloudinit($drive)) {
6447                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6448                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6449                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6450
6451                     my $d = {
6452                         format => $format,
6453                         storeid => $opts->{storage} // $storeid,
6454                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6455                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6456                         name => "vm-$vmid-cloudinit",
6457                         is_cloudinit => 1,
6458                     };
6459                     $virtdev_hash->{$virtdev} = $d;
6460                 }
6461             }
6462         }
6463
6464         foreach my $key (keys %storage_limits) {
6465             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6466             next if !$limit;
6467             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6468             $storage_limits{$key} = $limit * 1024;
6469         }
6470
6471         foreach my $devname (keys %$devinfo) {
6472             die "found no device mapping information for device '$devname'\n"
6473                 if !$devinfo->{$devname}->{virtdev};
6474         }
6475
6476         # create empty/temp config
6477         if ($oldconf) {
6478             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6479             foreach_drive($oldconf, sub {
6480                 my ($ds, $drive) = @_;
6481
6482                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6483
6484                 my $volid = $drive->{file};
6485                 return if !$volid || $volid =~ m|^/|;
6486
6487                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6488                 return if !$path || !$owner || ($owner != $vmid);
6489
6490                 # Note: only delete disk we want to restore
6491                 # other volumes will become unused
6492                 if ($virtdev_hash->{$ds}) {
6493                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6494                     if (my $err = $@) {
6495                         warn $err;
6496                     }
6497                 }
6498             });
6499
6500             # delete vmstate files, after the restore we have no snapshots anymore
6501             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6502                 my $snap = $oldconf->{snapshots}->{$snapname};
6503                 if ($snap->{vmstate}) {
6504                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6505                     if (my $err = $@) {
6506                         warn $err;
6507                     }
6508                 }
6509             }
6510         }
6511
6512         my $map = {};
6513         foreach my $virtdev (sort keys %$virtdev_hash) {
6514             my $d = $virtdev_hash->{$virtdev};
6515             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6516             my $storeid = $d->{storeid};
6517             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6518
6519             my $map_opts = '';
6520             if (my $limit = $storage_limits{$storeid}) {
6521                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6522             }
6523
6524             # test if requested format is supported
6525             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6526             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6527             $d->{format} = $defFormat if !$supported;
6528
6529             my $name;
6530             if ($d->{is_cloudinit}) {
6531                 $name = $d->{name};
6532                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6533             }
6534
6535             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6536             print STDERR "new volume ID is '$volid'\n";
6537             $d->{volid} = $volid;
6538
6539             PVE::Storage::activate_volumes($cfg, [$volid]);
6540
6541             my $write_zeros = 1;
6542             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6543                 $write_zeros = 0;
6544             }
6545
6546             if (!$d->{is_cloudinit}) {
6547                 my $path = PVE::Storage::path($cfg, $volid);
6548
6549                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6550
6551                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6552             }
6553             $map->{$virtdev} = $volid;
6554         }
6555
6556         $fh->seek(0, 0) || die "seek failed - $!\n";
6557
6558         my $outfd = new IO::File ($tmpfn, "w") ||
6559             die "unable to write config for VM $vmid\n";
6560
6561         my $cookie = { netcount => 0 };
6562         while (defined(my $line = <$fh>)) {
6563             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6564         }
6565
6566         $fh->close();
6567         $outfd->close();
6568     };
6569
6570     eval {
6571         # enable interrupts
6572         local $SIG{INT} =
6573             local $SIG{TERM} =
6574             local $SIG{QUIT} =
6575             local $SIG{HUP} =
6576             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6577         local $SIG{ALRM} = sub { die "got timeout\n"; };
6578
6579         $oldtimeout = alarm($timeout);
6580
6581         my $parser = sub {
6582             my $line = shift;
6583
6584             print "$line\n";
6585
6586             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6587                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6588                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6589             } elsif ($line =~ m/^CTIME: /) {
6590                 # we correctly received the vma config, so we can disable
6591                 # the timeout now for disk allocation (set to 10 minutes, so
6592                 # that we always timeout if something goes wrong)
6593                 alarm(600);
6594                 &$print_devmap();
6595                 print $fifofh "done\n";
6596                 my $tmp = $oldtimeout || 0;
6597                 $oldtimeout = undef;
6598                 alarm($tmp);
6599                 close($fifofh);
6600             }
6601         };
6602
6603         print "restore vma archive: $dbg_cmdstring\n";
6604         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6605     };
6606     my $err = $@;
6607
6608     alarm($oldtimeout) if $oldtimeout;
6609
6610     my $vollist = [];
6611     foreach my $devname (keys %$devinfo) {
6612         my $volid = $devinfo->{$devname}->{volid};
6613         push @$vollist, $volid if $volid;
6614     }
6615
6616     PVE::Storage::deactivate_volumes($cfg, $vollist);
6617
6618     unlink $mapfifo;
6619
6620     if ($err) {
6621         rmtree $tmpdir;
6622         unlink $tmpfn;
6623
6624         foreach my $devname (keys %$devinfo) {
6625             my $volid = $devinfo->{$devname}->{volid};
6626             next if !$volid;
6627             eval {
6628                 if ($volid =~ m|^/|) {
6629                     unlink $volid || die 'unlink failed\n';
6630                 } else {
6631                     PVE::Storage::vdisk_free($cfg, $volid);
6632                 }
6633                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6634             };
6635             print STDERR "unable to cleanup '$volid' - $@" if $@;
6636         }
6637         die $err;
6638     }
6639
6640     rmtree $tmpdir;
6641
6642     rename($tmpfn, $conffile) ||
6643         die "unable to commit configuration file '$conffile'\n";
6644
6645     PVE::Cluster::cfs_update(); # make sure we read new file
6646
6647     eval { rescan($vmid, 1); };
6648     warn $@ if $@;
6649 }
6650
6651 sub restore_tar_archive {
6652     my ($archive, $vmid, $user, $opts) = @_;
6653
6654     if ($archive ne '-') {
6655         my $firstfile = tar_archive_read_firstfile($archive);
6656         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6657             if $firstfile ne 'qemu-server.conf';
6658     }
6659
6660     my $storecfg = PVE::Storage::config();
6661
6662     # destroy existing data - keep empty config
6663     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6664     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6665
6666     my $tocmd = "/usr/lib/qemu-server/qmextract";
6667
6668     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6669     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6670     $tocmd .= ' --prealloc' if $opts->{prealloc};
6671     $tocmd .= ' --info' if $opts->{info};
6672
6673     # tar option "xf" does not autodetect compression when read from STDIN,
6674     # so we pipe to zcat
6675     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6676         PVE::Tools::shellquote("--to-command=$tocmd");
6677
6678     my $tmpdir = "/var/tmp/vzdumptmp$$";
6679     mkpath $tmpdir;
6680
6681     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6682     local $ENV{VZDUMP_VMID} = $vmid;
6683     local $ENV{VZDUMP_USER} = $user;
6684
6685     my $conffile = PVE::QemuConfig->config_file($vmid);
6686     my $tmpfn = "$conffile.$$.tmp";
6687
6688     # disable interrupts (always do cleanups)
6689     local $SIG{INT} =
6690         local $SIG{TERM} =
6691         local $SIG{QUIT} =
6692         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6693
6694     eval {
6695         # enable interrupts
6696         local $SIG{INT} =
6697             local $SIG{TERM} =
6698             local $SIG{QUIT} =
6699             local $SIG{HUP} =
6700             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6701
6702         if ($archive eq '-') {
6703             print "extracting archive from STDIN\n";
6704             run_command($cmd, input => "<&STDIN");
6705         } else {
6706             print "extracting archive '$archive'\n";
6707             run_command($cmd);
6708         }
6709
6710         return if $opts->{info};
6711
6712         # read new mapping
6713         my $map = {};
6714         my $statfile = "$tmpdir/qmrestore.stat";
6715         if (my $fd = IO::File->new($statfile, "r")) {
6716             while (defined (my $line = <$fd>)) {
6717                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6718                     $map->{$1} = $2 if $1;
6719                 } else {
6720                     print STDERR "unable to parse line in statfile - $line\n";
6721                 }
6722             }
6723             $fd->close();
6724         }
6725
6726         my $confsrc = "$tmpdir/qemu-server.conf";
6727
6728         my $srcfd = new IO::File($confsrc, "r") ||
6729             die "unable to open file '$confsrc'\n";
6730
6731         my $outfd = new IO::File ($tmpfn, "w") ||
6732             die "unable to write config for VM $vmid\n";
6733
6734         my $cookie = { netcount => 0 };
6735         while (defined (my $line = <$srcfd>)) {
6736             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6737         }
6738
6739         $srcfd->close();
6740         $outfd->close();
6741     };
6742     my $err = $@;
6743
6744     if ($err) {
6745
6746         unlink $tmpfn;
6747
6748         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6749
6750         die $err;
6751     }
6752
6753     rmtree $tmpdir;
6754
6755     rename $tmpfn, $conffile ||
6756         die "unable to commit configuration file '$conffile'\n";
6757
6758     PVE::Cluster::cfs_update(); # make sure we read new file
6759
6760     eval { rescan($vmid, 1); };
6761     warn $@ if $@;
6762 };
6763
6764 sub foreach_storage_used_by_vm {
6765     my ($conf, $func) = @_;
6766
6767     my $sidhash = {};
6768
6769     foreach_drive($conf, sub {
6770         my ($ds, $drive) = @_;
6771         return if drive_is_cdrom($drive);
6772
6773         my $volid = $drive->{file};
6774
6775         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6776         $sidhash->{$sid} = $sid if $sid;
6777     });
6778
6779     foreach my $sid (sort keys %$sidhash) {
6780         &$func($sid);
6781     }
6782 }
6783
6784 sub do_snapshots_with_qemu {
6785     my ($storecfg, $volid) = @_;
6786
6787     my $storage_name = PVE::Storage::parse_volume_id($volid);
6788     my $scfg = $storecfg->{ids}->{$storage_name};
6789
6790     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6791         return 1;
6792     }
6793
6794     if ($volid =~ m/\.(qcow2|qed)$/){
6795         return 1;
6796     }
6797
6798     return undef;
6799 }
6800
6801 sub qga_check_running {
6802     my ($vmid, $nowarn) = @_;
6803
6804     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6805     if ($@) {
6806         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6807         return 0;
6808     }
6809     return 1;
6810 }
6811
6812 sub template_create {
6813     my ($vmid, $conf, $disk) = @_;
6814
6815     my $storecfg = PVE::Storage::config();
6816
6817     foreach_drive($conf, sub {
6818         my ($ds, $drive) = @_;
6819
6820         return if drive_is_cdrom($drive);
6821         return if $disk && $ds ne $disk;
6822
6823         my $volid = $drive->{file};
6824         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6825
6826         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6827         $drive->{file} = $voliddst;
6828         $conf->{$ds} = print_drive($vmid, $drive);
6829         PVE::QemuConfig->write_config($vmid, $conf);
6830     });
6831 }
6832
6833 sub convert_iscsi_path {
6834     my ($path) = @_;
6835
6836     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6837         my $portal = $1;
6838         my $target = $2;
6839         my $lun = $3;
6840
6841         my $initiator_name = get_initiator_name();
6842
6843         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6844                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6845     }
6846
6847     die "cannot convert iscsi path '$path', unkown format\n";
6848 }
6849
6850 sub qemu_img_convert {
6851     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6852
6853     my $storecfg = PVE::Storage::config();
6854     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6855     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6856
6857     if ($src_storeid && $dst_storeid) {
6858
6859         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6860
6861         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6862         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6863
6864         my $src_format = qemu_img_format($src_scfg, $src_volname);
6865         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6866
6867         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6868         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6869
6870         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6871         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6872
6873         my $cmd = [];
6874         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6875         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6876         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6877         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6878
6879         if ($src_is_iscsi) {
6880             push @$cmd, '--image-opts';
6881             $src_path = convert_iscsi_path($src_path);
6882         } else {
6883             push @$cmd, '-f', $src_format;
6884         }
6885
6886         if ($dst_is_iscsi) {
6887             push @$cmd, '--target-image-opts';
6888             $dst_path = convert_iscsi_path($dst_path);
6889         } else {
6890             push @$cmd, '-O', $dst_format;
6891         }
6892
6893         push @$cmd, $src_path;
6894
6895         if (!$dst_is_iscsi && $is_zero_initialized) {
6896             push @$cmd, "zeroinit:$dst_path";
6897         } else {
6898             push @$cmd, $dst_path;
6899         }
6900
6901         my $parser = sub {
6902             my $line = shift;
6903             if($line =~ m/\((\S+)\/100\%\)/){
6904                 my $percent = $1;
6905                 my $transferred = int($size * $percent / 100);
6906                 my $remaining = $size - $transferred;
6907
6908                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6909             }
6910
6911         };
6912
6913         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6914         my $err = $@;
6915         die "copy failed: $err" if $err;
6916     }
6917 }
6918
6919 sub qemu_img_format {
6920     my ($scfg, $volname) = @_;
6921
6922     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6923         return $1;
6924     } else {
6925         return "raw";
6926     }
6927 }
6928
6929 sub qemu_drive_mirror {
6930     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6931
6932     $jobs = {} if !$jobs;
6933
6934     my $qemu_target;
6935     my $format;
6936     $jobs->{"drive-$drive"} = {};
6937
6938     if ($dst_volid =~ /^nbd:/) {
6939         $qemu_target = $dst_volid;
6940         $format = "nbd";
6941     } else {
6942         my $storecfg = PVE::Storage::config();
6943         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6944
6945         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6946
6947         $format = qemu_img_format($dst_scfg, $dst_volname);
6948
6949         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6950
6951         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6952     }
6953
6954     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6955     $opts->{format} = $format if $format;
6956
6957     if (defined($bwlimit)) {
6958         $opts->{speed} = $bwlimit * 1024;
6959         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6960     } else {
6961         print "drive mirror is starting for drive-$drive\n";
6962     }
6963
6964     # if a job already runs for this device we get an error, catch it for cleanup
6965     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6966     if (my $err = $@) {
6967         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6968         warn "$@\n" if $@;
6969         die "mirroring error: $err\n";
6970     }
6971
6972     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6973 }
6974
6975 sub qemu_drive_mirror_monitor {
6976     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6977
6978     eval {
6979         my $err_complete = 0;
6980
6981         while (1) {
6982             die "storage migration timed out\n" if $err_complete > 300;
6983
6984             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6985
6986             my $running_mirror_jobs = {};
6987             foreach my $stat (@$stats) {
6988                 next if $stat->{type} ne 'mirror';
6989                 $running_mirror_jobs->{$stat->{device}} = $stat;
6990             }
6991
6992             my $readycounter = 0;
6993
6994             foreach my $job (keys %$jobs) {
6995
6996                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6997                     print "$job : finished\n";
6998                     delete $jobs->{$job};
6999                     next;
7000                 }
7001
7002                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7003
7004                 my $busy = $running_mirror_jobs->{$job}->{busy};
7005                 my $ready = $running_mirror_jobs->{$job}->{ready};
7006                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7007                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7008                     my $remaining = $total - $transferred;
7009                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7010
7011                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7012                 }
7013
7014                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7015             }
7016
7017             last if scalar(keys %$jobs) == 0;
7018
7019             if ($readycounter == scalar(keys %$jobs)) {
7020                 print "all mirroring jobs are ready \n";
7021                 last if $skipcomplete; #do the complete later
7022
7023                 if ($vmiddst && $vmiddst != $vmid) {
7024                     my $agent_running = $qga && qga_check_running($vmid);
7025                     if ($agent_running) {
7026                         print "freeze filesystem\n";
7027                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7028                     } else {
7029                         print "suspend vm\n";
7030                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7031                     }
7032
7033                     # if we clone a disk for a new target vm, we don't switch the disk
7034                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7035
7036                     if ($agent_running) {
7037                         print "unfreeze filesystem\n";
7038                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7039                     } else {
7040                         print "resume vm\n";
7041                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7042                     }
7043
7044                     last;
7045                 } else {
7046
7047                     foreach my $job (keys %$jobs) {
7048                         # try to switch the disk if source and destination are on the same guest
7049                         print "$job: Completing block job...\n";
7050
7051                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7052                         if ($@ =~ m/cannot be completed/) {
7053                             print "$job: Block job cannot be completed, try again.\n";
7054                             $err_complete++;
7055                         }else {
7056                             print "$job: Completed successfully.\n";
7057                             $jobs->{$job}->{complete} = 1;
7058                         }
7059                     }
7060                 }
7061             }
7062             sleep 1;
7063         }
7064     };
7065     my $err = $@;
7066
7067     if ($err) {
7068         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7069         die "mirroring error: $err";
7070     }
7071
7072 }
7073
7074 sub qemu_blockjobs_cancel {
7075     my ($vmid, $jobs) = @_;
7076
7077     foreach my $job (keys %$jobs) {
7078         print "$job: Cancelling block job\n";
7079         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7080         $jobs->{$job}->{cancel} = 1;
7081     }
7082
7083     while (1) {
7084         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7085
7086         my $running_jobs = {};
7087         foreach my $stat (@$stats) {
7088             $running_jobs->{$stat->{device}} = $stat;
7089         }
7090
7091         foreach my $job (keys %$jobs) {
7092
7093             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7094                 print "$job: Done.\n";
7095                 delete $jobs->{$job};
7096             }
7097         }
7098
7099         last if scalar(keys %$jobs) == 0;
7100
7101         sleep 1;
7102     }
7103 }
7104
7105 sub clone_disk {
7106     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7107         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7108
7109     my $newvolid;
7110
7111     if (!$full) {
7112         print "create linked clone of drive $drivename ($drive->{file})\n";
7113         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7114         push @$newvollist, $newvolid;
7115     } else {
7116
7117         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7118         $storeid = $storage if $storage;
7119
7120         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7121         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7122
7123         print "create full clone of drive $drivename ($drive->{file})\n";
7124         my $name = undef;
7125         if (drive_is_cloudinit($drive)) {
7126             $name = "vm-$newvmid-cloudinit";
7127             $snapname = undef;
7128             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7129             if ($dst_format ne 'raw') {
7130                 $name .= ".$dst_format";
7131             }
7132         }
7133         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7134         push @$newvollist, $newvolid;
7135
7136         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7137
7138         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7139         if (!$running || $snapname) {
7140             # TODO: handle bwlimits
7141             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7142         } else {
7143
7144             my $kvmver = get_running_qemu_version ($vmid);
7145             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7146                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7147                     if $drive->{iothread};
7148             }
7149
7150             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7151         }
7152     }
7153
7154     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7155
7156     my $disk = $drive;
7157     $disk->{format} = undef;
7158     $disk->{file} = $newvolid;
7159     $disk->{size} = $size;
7160
7161     return $disk;
7162 }
7163
7164 # this only works if VM is running
7165 sub get_current_qemu_machine {
7166     my ($vmid) = @_;
7167
7168     my $cmd = { execute => 'query-machines', arguments => {} };
7169     my $res = vm_qmp_command($vmid, $cmd);
7170
7171     my ($current, $default);
7172     foreach my $e (@$res) {
7173         $default = $e->{name} if $e->{'is-default'};
7174         $current = $e->{name} if $e->{'is-current'};
7175     }
7176
7177     # fallback to the default machine if current is not supported by qemu
7178     return $current || $default || 'pc';
7179 }
7180
7181 sub get_running_qemu_version {
7182     my ($vmid) = @_;
7183     my $cmd = { execute => 'query-version', arguments => {} };
7184     my $res = vm_qmp_command($vmid, $cmd);
7185     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7186 }
7187
7188 sub qemu_machine_feature_enabled {
7189     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7190
7191     my $current_major;
7192     my $current_minor;
7193
7194     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7195
7196         $current_major = $3;
7197         $current_minor = $4;
7198
7199     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7200
7201         $current_major = $1;
7202         $current_minor = $2;
7203     }
7204
7205     return 1 if $current_major > $version_major ||
7206                 ($current_major == $version_major &&
7207                  $current_minor >= $version_minor);
7208 }
7209
7210 sub qemu_machine_pxe {
7211     my ($vmid, $conf) = @_;
7212
7213     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7214
7215     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7216         $machine .= '.pxe';
7217     }
7218
7219     return $machine;
7220 }
7221
7222 sub qemu_use_old_bios_files {
7223     my ($machine_type) = @_;
7224
7225     return if !$machine_type;
7226
7227     my $use_old_bios_files = undef;
7228
7229     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7230         $machine_type = $1;
7231         $use_old_bios_files = 1;
7232     } else {
7233         my $kvmver = kvm_user_version();
7234         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7235         # load new efi bios files on migration. So this hack is required to allow
7236         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7237         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7238         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7239     }
7240
7241     return ($use_old_bios_files, $machine_type);
7242 }
7243
7244 sub create_efidisk($$$$$) {
7245     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7246
7247     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7248     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7249
7250     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7251     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7252     PVE::Storage::activate_volumes($storecfg, [$volid]);
7253
7254     my $path = PVE::Storage::path($storecfg, $volid);
7255     eval {
7256         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7257     };
7258     die "Copying EFI vars image failed: $@" if $@;
7259
7260     return ($volid, $vars_size);
7261 }
7262
7263 sub vm_iothreads_list {
7264     my ($vmid) = @_;
7265
7266     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7267
7268     my $iothreads = {};
7269     foreach my $iothread (@$res) {
7270         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7271     }
7272
7273     return $iothreads;
7274 }
7275
7276 sub scsihw_infos {
7277     my ($conf, $drive) = @_;
7278
7279     my $maxdev = 0;
7280
7281     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7282         $maxdev = 7;
7283     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7284         $maxdev = 1;
7285     } else {
7286         $maxdev = 256;
7287     }
7288
7289     my $controller = int($drive->{index} / $maxdev);
7290     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7291
7292     return ($maxdev, $controller, $controller_prefix);
7293 }
7294
7295 sub add_hyperv_enlightenments {
7296     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7297
7298     return if $winversion < 6;
7299     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7300
7301     if ($gpu_passthrough || defined($hv_vendor_id)) {
7302         $hv_vendor_id //= 'proxmox';
7303         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7304     }
7305
7306     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7307         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7308         push @$cpuFlags , 'hv_vapic';
7309         push @$cpuFlags , 'hv_time';
7310     } else {
7311         push @$cpuFlags , 'hv_spinlocks=0xffff';
7312     }
7313
7314     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7315         push @$cpuFlags , 'hv_reset';
7316         push @$cpuFlags , 'hv_vpindex';
7317         push @$cpuFlags , 'hv_runtime';
7318     }
7319
7320     if ($winversion >= 7) {
7321         push @$cpuFlags , 'hv_relaxed';
7322
7323         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7324             push @$cpuFlags , 'hv_synic';
7325             push @$cpuFlags , 'hv_stimer';
7326         }
7327
7328         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7329             push @$cpuFlags , 'hv_ipi';
7330         }
7331     }
7332 }
7333
7334 sub windows_version {
7335     my ($ostype) = @_;
7336
7337     return 0 if !$ostype;
7338
7339     my $winversion = 0;
7340
7341     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7342         $winversion = 5;
7343     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7344         $winversion = 6;
7345     } elsif ($ostype =~ m/^win(\d+)$/) {
7346         $winversion = $1;
7347     }
7348
7349     return $winversion;
7350 }
7351
7352 sub resolve_dst_disk_format {
7353         my ($storecfg, $storeid, $src_volname, $format) = @_;
7354         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7355
7356         if (!$format) {
7357             # if no target format is specified, use the source disk format as hint
7358             if ($src_volname) {
7359                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7360                 $format = qemu_img_format($scfg, $src_volname);
7361             } else {
7362                 return $defFormat;
7363             }
7364         }
7365
7366         # test if requested format is supported - else use default
7367         my $supported = grep { $_ eq $format } @$validFormats;
7368         $format = $defFormat if !$supported;
7369         return $format;
7370 }
7371
7372 sub resolve_first_disk {
7373     my $conf = shift;
7374     my @disks = PVE::QemuServer::valid_drive_names();
7375     my $firstdisk;
7376     foreach my $ds (reverse @disks) {
7377         next if !$conf->{$ds};
7378         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7379         next if PVE::QemuServer::drive_is_cdrom($disk);
7380         $firstdisk = $ds;
7381     }
7382     return $firstdisk;
7383 }
7384
7385 sub generate_uuid {
7386     my ($uuid, $uuid_str);
7387     UUID::generate($uuid);
7388     UUID::unparse($uuid, $uuid_str);
7389     return $uuid_str;
7390 }
7391
7392 sub generate_smbios1_uuid {
7393     return "uuid=".generate_uuid();
7394 }
7395
7396 sub nbd_stop {
7397     my ($vmid) = @_;
7398
7399     vm_mon_cmd($vmid, 'nbd-server-stop');
7400 }
7401
7402 # bash completion helper
7403
7404 sub complete_backup_archives {
7405     my ($cmdname, $pname, $cvalue) = @_;
7406
7407     my $cfg = PVE::Storage::config();
7408
7409     my $storeid;
7410
7411     if ($cvalue =~ m/^([^:]+):/) {
7412         $storeid = $1;
7413     }
7414
7415     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7416
7417     my $res = [];
7418     foreach my $id (keys %$data) {
7419         foreach my $item (@{$data->{$id}}) {
7420             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7421             push @$res, $item->{volid} if defined($item->{volid});
7422         }
7423     }
7424
7425     return $res;
7426 }
7427
7428 my $complete_vmid_full = sub {
7429     my ($running) = @_;
7430
7431     my $idlist = vmstatus();
7432
7433     my $res = [];
7434
7435     foreach my $id (keys %$idlist) {
7436         my $d = $idlist->{$id};
7437         if (defined($running)) {
7438             next if $d->{template};
7439             next if $running && $d->{status} ne 'running';
7440             next if !$running && $d->{status} eq 'running';
7441         }
7442         push @$res, $id;
7443
7444     }
7445     return $res;
7446 };
7447
7448 sub complete_vmid {
7449     return &$complete_vmid_full();
7450 }
7451
7452 sub complete_vmid_stopped {
7453     return &$complete_vmid_full(0);
7454 }
7455
7456 sub complete_vmid_running {
7457     return &$complete_vmid_full(1);
7458 }
7459
7460 sub complete_storage {
7461
7462     my $cfg = PVE::Storage::config();
7463     my $ids = $cfg->{ids};
7464
7465     my $res = [];
7466     foreach my $sid (keys %$ids) {
7467         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7468         next if !$ids->{$sid}->{content}->{images};
7469         push @$res, $sid;
7470     }
7471
7472     return $res;
7473 }
7474
7475 1;