PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my @supported_cpu_flags = (
 171     'pcid',
 172     'spec-ctrl',
 173     'ibpb',
 174     'ssbd',
 175     'virt-ssbd',
 176     'amd-ssbd',
 177     'amd-no-ssb',
 178     'pdpe1gb',
 179     'md-clear',
 180     'hv-tlbflush',
 181     'hv-evmcs',
 182     'aes'
 183 );
 184 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 185
 186 my $cpu_fmt = {
 187     cputype => {
 188         description => "Emulated CPU type.",
 189         type => 'string',
 190         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 191         default => 'kvm64',
 192         default_key => 1,
 193     },
 194     hidden => {
 195         description => "Do not identify as a KVM virtual machine.",
 196         type => 'boolean',
 197         optional => 1,
 198         default => 0
 199     },
 200     'hv-vendor-id' => {
 201         type => 'string',
 202         pattern => qr/[a-zA-Z0-9]{1,12}/,
 203         format_description => 'vendor-id',
 204         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 205         optional => 1,
 206     },
 207     flags => {
 208         description => "List of additional CPU flags separated by ';'."
 209                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 210                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 211         format_description => '+FLAG[;-FLAG...]',
 212         type => 'string',
 213         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 214         optional => 1,
 215     },
 216 };
 217
 218 my $watchdog_fmt = {
 219     model => {
 220         default_key => 1,
 221         type => 'string',
 222         enum => [qw(i6300esb ib700)],
 223         description => "Watchdog type to emulate.",
 224         default => 'i6300esb',
 225         optional => 1,
 226     },
 227     action => {
 228         type => 'string',
 229         enum => [qw(reset shutdown poweroff pause debug none)],
 230         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 231         optional => 1,
 232     },
 233 };
 234 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 235
 236 my $agent_fmt = {
 237     enabled => {
 238         description => "Enable/disable Qemu GuestAgent.",
 239         type => 'boolean',
 240         default => 0,
 241         default_key => 1,
 242     },
 243     fstrim_cloned_disks => {
 244         description => "Run fstrim after cloning/moving a disk.",
 245         type => 'boolean',
 246         optional => 1,
 247         default => 0
 248     },
 249 };
 250
 251 my $vga_fmt = {
 252     type => {
 253         description => "Select the VGA type.",
 254         type => 'string',
 255         default => 'std',
 256         optional => 1,
 257         default_key => 1,
 258         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 259     },
 260     memory => {
 261         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 262         type => 'integer',
 263         optional => 1,
 264         minimum => 4,
 265         maximum => 512,
 266     },
 267 };
 268
 269 my $ivshmem_fmt = {
 270     size => {
 271         type => 'integer',
 272         minimum => 1,
 273         description => "The size of the file in MB.",
 274     },
 275     name => {
 276         type => 'string',
 277         pattern => '[a-zA-Z0-9\-]+',
 278         optional => 1,
 279         format_description => 'string',
 280         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 281     },
 282 };
 283
 284 my $audio_fmt = {
 285     device => {
 286         type => 'string',
 287         enum => [qw(ich9-intel-hda intel-hda AC97)],
 288         description =>  "Configure an audio device."
 289     },
 290     driver =>  {
 291         type => 'string',
 292         enum => ['spice'],
 293         default => 'spice',
 294         optional => 1,
 295         description => "Driver backend for the audio device."
 296     },
 297 };
 298
 299 my $spice_enhancements_fmt = {
 300     foldersharing => {
 301         type => 'boolean',
 302         optional => 1,
 303         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 304     },
 305     videostreaming =>  {
 306         type => 'string',
 307         enum => ['off', 'all', 'filter'],
 308         optional => 1,
 309         description => "Enable video streaming. Uses compression for detected video streams."
 310     },
 311 };
 312
 313 my $confdesc = {
 314     onboot => {
 315         optional => 1,
 316         type => 'boolean',
 317         description => "Specifies whether a VM will be started during system bootup.",
 318         default => 0,
 319     },
 320     autostart => {
 321         optional => 1,
 322         type => 'boolean',
 323         description => "Automatic restart after crash (currently ignored).",
 324         default => 0,
 325     },
 326     hotplug => {
 327         optional => 1,
 328         type => 'string', format => 'pve-hotplug-features',
 329         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 330         default => 'network,disk,usb',
 331     },
 332     reboot => {
 333         optional => 1,
 334         type => 'boolean',
 335         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 336         default => 1,
 337     },
 338     lock => {
 339         optional => 1,
 340         type => 'string',
 341         description => "Lock/unlock the VM.",
 342         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 343     },
 344     cpulimit => {
 345         optional => 1,
 346         type => 'number',
 347         description => "Limit of CPU usage.",
 348         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 349         minimum => 0,
 350         maximum => 128,
 351         default => 0,
 352     },
 353     cpuunits => {
 354         optional => 1,
 355         type => 'integer',
 356         description => "CPU weight for a VM.",
 357         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 358         minimum => 2,
 359         maximum => 262144,
 360         default => 1024,
 361     },
 362     memory => {
 363         optional => 1,
 364         type => 'integer',
 365         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 366         minimum => 16,
 367         default => 512,
 368     },
 369     balloon => {
 370         optional => 1,
 371         type => 'integer',
 372         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 373         minimum => 0,
 374     },
 375     shares => {
 376         optional => 1,
 377         type => 'integer',
 378         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 379         minimum => 0,
 380         maximum => 50000,
 381         default => 1000,
 382     },
 383     keyboard => {
 384         optional => 1,
 385         type => 'string',
 386         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 387                        "It should not be necessary to set it.",
 388         enum => PVE::Tools::kvmkeymaplist(),
 389         default => undef,
 390     },
 391     name => {
 392         optional => 1,
 393         type => 'string', format => 'dns-name',
 394         description => "Set a name for the VM. Only used on the configuration web interface.",
 395     },
 396     scsihw => {
 397         optional => 1,
 398         type => 'string',
 399         description => "SCSI controller model",
 400         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 401         default => 'lsi',
 402     },
 403     description => {
 404         optional => 1,
 405         type => 'string',
 406         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 407     },
 408     ostype => {
 409         optional => 1,
 410         type => 'string',
 411         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 412         description => "Specify guest operating system.",
 413         verbose_description => <<EODESC,
 414 Specify guest operating system. This is used to enable special
 415 optimization/features for specific operating systems:
 416
 417 [horizontal]
 418 other;; unspecified OS
 419 wxp;; Microsoft Windows XP
 420 w2k;; Microsoft Windows 2000
 421 w2k3;; Microsoft Windows 2003
 422 w2k8;; Microsoft Windows 2008
 423 wvista;; Microsoft Windows Vista
 424 win7;; Microsoft Windows 7
 425 win8;; Microsoft Windows 8/2012/2012r2
 426 win10;; Microsoft Windows 10/2016
 427 l24;; Linux 2.4 Kernel
 428 l26;; Linux 2.6/3.X Kernel
 429 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 430 EODESC
 431     },
 432     boot => {
 433         optional => 1,
 434         type => 'string',
 435         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 436         pattern => '[acdn]{1,4}',
 437         default => 'cdn',
 438     },
 439     bootdisk => {
 440         optional => 1,
 441         type => 'string', format => 'pve-qm-bootdisk',
 442         description => "Enable booting from specified disk.",
 443         pattern => '(ide|sata|scsi|virtio)\d+',
 444     },
 445     smp => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "The number of CPUs. Please use option -sockets instead.",
 449         minimum => 1,
 450         default => 1,
 451     },
 452     sockets => {
 453         optional => 1,
 454         type => 'integer',
 455         description => "The number of CPU sockets.",
 456         minimum => 1,
 457         default => 1,
 458     },
 459     cores => {
 460         optional => 1,
 461         type => 'integer',
 462         description => "The number of cores per socket.",
 463         minimum => 1,
 464         default => 1,
 465     },
 466     numa => {
 467         optional => 1,
 468         type => 'boolean',
 469         description => "Enable/disable NUMA.",
 470         default => 0,
 471     },
 472     hugepages => {
 473         optional => 1,
 474         type => 'string',
 475         description => "Enable/disable hugepages memory.",
 476         enum => [qw(any 2 1024)],
 477     },
 478     vcpus => {
 479         optional => 1,
 480         type => 'integer',
 481         description => "Number of hotplugged vcpus.",
 482         minimum => 1,
 483         default => 0,
 484     },
 485     acpi => {
 486         optional => 1,
 487         type => 'boolean',
 488         description => "Enable/disable ACPI.",
 489         default => 1,
 490     },
 491     agent => {
 492         optional => 1,
 493         description => "Enable/disable Qemu GuestAgent and its properties.",
 494         type => 'string',
 495         format => $agent_fmt,
 496     },
 497     kvm => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Enable/disable KVM hardware virtualization.",
 501         default => 1,
 502     },
 503     tdf => {
 504         optional => 1,
 505         type => 'boolean',
 506         description => "Enable/disable time drift fix.",
 507         default => 0,
 508     },
 509     localtime => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 513     },
 514     freeze => {
 515         optional => 1,
 516         type => 'boolean',
 517         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 518     },
 519     vga => {
 520         optional => 1,
 521         type => 'string', format => $vga_fmt,
 522         description => "Configure the VGA hardware.",
 523         verbose_description => "Configure the VGA Hardware. If you want to use ".
 524             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 525             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 526             "is 'std' for all OS types besides some Windows versions (XP and " .
 527             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 528             "display server. For win* OS you can select how many independent " .
 529             "displays you want, Linux guests can add displays them self.\n".
 530             "You can also run without any graphic card, using a serial device as terminal.",
 531     },
 532     watchdog => {
 533         optional => 1,
 534         type => 'string', format => 'pve-qm-watchdog',
 535         description => "Create a virtual hardware watchdog device.",
 536         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 537             " (by a guest action), the watchdog must be periodically polled " .
 538             "by an agent inside the guest or else the watchdog will reset " .
 539             "the guest (or execute the respective action specified)",
 540     },
 541     startdate => {
 542         optional => 1,
 543         type => 'string',
 544         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 545         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 546         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 547         default => 'now',
 548     },
 549     startup =>  get_standard_option('pve-startup-order'),
 550     template => {
 551         optional => 1,
 552         type => 'boolean',
 553         description => "Enable/disable Template.",
 554         default => 0,
 555     },
 556     args => {
 557         optional => 1,
 558         type => 'string',
 559         description => "Arbitrary arguments passed to kvm.",
 560         verbose_description => <<EODESCR,
 561 Arbitrary arguments passed to kvm, for example:
 562
 563 args: -no-reboot -no-hpet
 564
 565 NOTE: this option is for experts only.
 566 EODESCR
 567     },
 568     tablet => {
 569         optional => 1,
 570         type => 'boolean',
 571         default => 1,
 572         description => "Enable/disable the USB tablet device.",
 573         verbose_description => "Enable/disable the USB tablet device. This device is " .
 574             "usually needed to allow absolute mouse positioning with VNC. " .
 575             "Else the mouse runs out of sync with normal VNC clients. " .
 576             "If you're running lots of console-only guests on one host, " .
 577             "you may consider disabling this to save some context switches. " .
 578             "This is turned off by default if you use spice (-vga=qxl).",
 579     },
 580     migrate_speed => {
 581         optional => 1,
 582         type => 'integer',
 583         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 584         minimum => 0,
 585         default => 0,
 586     },
 587     migrate_downtime => {
 588         optional => 1,
 589         type => 'number',
 590         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 591         minimum => 0,
 592         default => 0.1,
 593     },
 594     cdrom => {
 595         optional => 1,
 596         type => 'string', format => 'pve-qm-ide',
 597         typetext => '<volume>',
 598         description => "This is an alias for option -ide2",
 599     },
 600     cpu => {
 601         optional => 1,
 602         description => "Emulated CPU type.",
 603         type => 'string',
 604         format => $cpu_fmt,
 605     },
 606     parent => get_standard_option('pve-snapshot-name', {
 607         optional => 1,
 608         description => "Parent snapshot name. This is used internally, and should not be modified.",
 609     }),
 610     snaptime => {
 611         optional => 1,
 612         description => "Timestamp for snapshots.",
 613         type => 'integer',
 614         minimum => 0,
 615     },
 616     vmstate => {
 617         optional => 1,
 618         type => 'string', format => 'pve-volume-id',
 619         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 620     },
 621     vmstatestorage => get_standard_option('pve-storage-id', {
 622         description => "Default storage for VM state volumes/files.",
 623         optional => 1,
 624     }),
 625     runningmachine => get_standard_option('pve-qemu-machine', {
 626         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 627     }),
 628     machine => get_standard_option('pve-qemu-machine'),
 629     arch => {
 630         description => "Virtual processor architecture. Defaults to the host.",
 631         optional => 1,
 632         type => 'string',
 633         enum => [qw(x86_64 aarch64)],
 634     },
 635     smbios1 => {
 636         description => "Specify SMBIOS type 1 fields.",
 637         type => 'string', format => 'pve-qm-smbios1',
 638         maxLength => 512,
 639         optional => 1,
 640     },
 641     protection => {
 642         optional => 1,
 643         type => 'boolean',
 644         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 645         default => 0,
 646     },
 647     bios => {
 648         optional => 1,
 649         type => 'string',
 650         enum => [ qw(seabios ovmf) ],
 651         description => "Select BIOS implementation.",
 652         default => 'seabios',
 653     },
 654     vmgenid => {
 655         type => 'string',
 656         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 657         format_description => 'UUID',
 658         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 659         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 660             " 128-bit integer value identifier to the guest OS. This allows to".
 661             " notify the guest operating system when the virtual machine is".
 662             " executed with a different configuration (e.g. snapshot execution".
 663             " or creation from a template). The guest operating system notices".
 664             " the change, and is then able to react as appropriate by marking".
 665             " its copies of distributed databases as dirty, re-initializing its".
 666             " random number generator, etc.\n".
 667             "Note that auto-creation only works when done throug API/CLI create".
 668             " or update methods, but not when manually editing the config file.",
 669         default => "1 (autogenerated)",
 670         optional => 1,
 671     },
 672     hookscript => {
 673         type => 'string',
 674         format => 'pve-volume-id',
 675         optional => 1,
 676         description => "Script that will be executed during various steps in the vms lifetime.",
 677     },
 678     ivshmem => {
 679         type => 'string',
 680         format => $ivshmem_fmt,
 681         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 682         optional => 1,
 683     },
 684     audio0 => {
 685         type => 'string',
 686         format => $audio_fmt,
 687         description => "Configure a audio device, useful in combination with QXL/Spice.",
 688         optional => 1
 689     },
 690     spice_enhancements => {
 691         type => 'string',
 692         format => $spice_enhancements_fmt,
 693         description => "Configure additional enhancements for SPICE.",
 694         optional => 1
 695     },
 696 };
 697
 698 my $cicustom_fmt = {
 699     meta => {
 700         type => 'string',
 701         optional => 1,
 702         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 703         format => 'pve-volume-id',
 704         format_description => 'volume',
 705     },
 706     network => {
 707         type => 'string',
 708         optional => 1,
 709         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 710         format => 'pve-volume-id',
 711         format_description => 'volume',
 712     },
 713     user => {
 714         type => 'string',
 715         optional => 1,
 716         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 717         format => 'pve-volume-id',
 718         format_description => 'volume',
 719     },
 720 };
 721 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 722
 723 my $confdesc_cloudinit = {
 724     citype => {
 725         optional => 1,
 726         type => 'string',
 727         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 728         enum => ['configdrive2', 'nocloud'],
 729     },
 730     ciuser => {
 731         optional => 1,
 732         type => 'string',
 733         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 734     },
 735     cipassword => {
 736         optional => 1,
 737         type => 'string',
 738         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 739     },
 740     cicustom => {
 741         optional => 1,
 742         type => 'string',
 743         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 744         format => 'pve-qm-cicustom',
 745     },
 746     searchdomain => {
 747         optional => 1,
 748         type => 'string',
 749         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 750     },
 751     nameserver => {
 752         optional => 1,
 753         type => 'string', format => 'address-list',
 754         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 755     },
 756     sshkeys => {
 757         optional => 1,
 758         type => 'string',
 759         format => 'urlencoded',
 760         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 761     },
 762 };
 763
 764 # what about other qemu settings ?
 765 #cpu => 'string',
 766 #machine => 'string',
 767 #fda => 'file',
 768 #fdb => 'file',
 769 #mtdblock => 'file',
 770 #sd => 'file',
 771 #pflash => 'file',
 772 #snapshot => 'bool',
 773 #bootp => 'file',
 774 ##tftp => 'dir',
 775 ##smb => 'dir',
 776 #kernel => 'file',
 777 #append => 'string',
 778 #initrd => 'file',
 779 ##soundhw => 'string',
 780
 781 while (my ($k, $v) = each %$confdesc) {
 782     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 783 }
 784
 785 my $MAX_IDE_DISKS = 4;
 786 my $MAX_SCSI_DISKS = 14;
 787 my $MAX_VIRTIO_DISKS = 16;
 788 my $MAX_SATA_DISKS = 6;
 789 my $MAX_USB_DEVICES = 5;
 790 my $MAX_NETS = 32;
 791 my $MAX_UNUSED_DISKS = 256;
 792 my $MAX_HOSTPCI_DEVICES = 4;
 793 my $MAX_SERIAL_PORTS = 4;
 794 my $MAX_PARALLEL_PORTS = 3;
 795 my $MAX_NUMA = 8;
 796
 797 my $numa_fmt = {
 798     cpus => {
 799         type => "string",
 800         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 801         description => "CPUs accessing this NUMA node.",
 802         format_description => "id[-id];...",
 803     },
 804     memory => {
 805         type => "number",
 806         description => "Amount of memory this NUMA node provides.",
 807         optional => 1,
 808     },
 809     hostnodes => {
 810         type => "string",
 811         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 812         description => "Host NUMA nodes to use.",
 813         format_description => "id[-id];...",
 814         optional => 1,
 815     },
 816     policy => {
 817         type => 'string',
 818         enum => [qw(preferred bind interleave)],
 819         description => "NUMA allocation policy.",
 820         optional => 1,
 821     },
 822 };
 823 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 824 my $numadesc = {
 825     optional => 1,
 826     type => 'string', format => $numa_fmt,
 827     description => "NUMA topology.",
 828 };
 829 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 830
 831 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 832     $confdesc->{"numa$i"} = $numadesc;
 833 }
 834
 835 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 836                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 837                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 838 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 839
 840 my $net_fmt_bridge_descr = <<__EOD__;
 841 Bridge to attach the network device to. The Proxmox VE standard bridge
 842 is called 'vmbr0'.
 843
 844 If you do not specify a bridge, we create a kvm user (NATed) network
 845 device, which provides DHCP and DNS services. The following addresses
 846 are used:
 847
 848  10.0.2.2   Gateway
 849  10.0.2.3   DNS Server
 850  10.0.2.4   SMB Server
 851
 852 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 853 __EOD__
 854
 855 my $net_fmt = {
 856     macaddr  => get_standard_option('mac-addr', {
 857         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 858     }),
 859     model => {
 860         type => 'string',
 861         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 862         enum => $nic_model_list,
 863         default_key => 1,
 864     },
 865     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 866     bridge => {
 867         type => 'string',
 868         description => $net_fmt_bridge_descr,
 869         format_description => 'bridge',
 870         optional => 1,
 871     },
 872     queues => {
 873         type => 'integer',
 874         minimum => 0, maximum => 16,
 875         description => 'Number of packet queues to be used on the device.',
 876         optional => 1,
 877     },
 878     rate => {
 879         type => 'number',
 880         minimum => 0,
 881         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 882         optional => 1,
 883     },
 884     tag => {
 885         type => 'integer',
 886         minimum => 1, maximum => 4094,
 887         description => 'VLAN tag to apply to packets on this interface.',
 888         optional => 1,
 889     },
 890     trunks => {
 891         type => 'string',
 892         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 893         description => 'VLAN trunks to pass through this interface.',
 894         format_description => 'vlanid[;vlanid...]',
 895         optional => 1,
 896     },
 897     firewall => {
 898         type => 'boolean',
 899         description => 'Whether this interface should be protected by the firewall.',
 900         optional => 1,
 901     },
 902     link_down => {
 903         type => 'boolean',
 904         description => 'Whether this interface should be disconnected (like pulling the plug).',
 905         optional => 1,
 906     },
 907 };
 908
 909 my $netdesc = {
 910     optional => 1,
 911     type => 'string', format => $net_fmt,
 912     description => "Specify network devices.",
 913 };
 914
 915 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 916
 917 my $ipconfig_fmt = {
 918     ip => {
 919         type => 'string',
 920         format => 'pve-ipv4-config',
 921         format_description => 'IPv4Format/CIDR',
 922         description => 'IPv4 address in CIDR format.',
 923         optional => 1,
 924         default => 'dhcp',
 925     },
 926     gw => {
 927         type => 'string',
 928         format => 'ipv4',
 929         format_description => 'GatewayIPv4',
 930         description => 'Default gateway for IPv4 traffic.',
 931         optional => 1,
 932         requires => 'ip',
 933     },
 934     ip6 => {
 935         type => 'string',
 936         format => 'pve-ipv6-config',
 937         format_description => 'IPv6Format/CIDR',
 938         description => 'IPv6 address in CIDR format.',
 939         optional => 1,
 940         default => 'dhcp',
 941     },
 942     gw6 => {
 943         type => 'string',
 944         format => 'ipv6',
 945         format_description => 'GatewayIPv6',
 946         description => 'Default gateway for IPv6 traffic.',
 947         optional => 1,
 948         requires => 'ip6',
 949     },
 950 };
 951 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 952 my $ipconfigdesc = {
 953     optional => 1,
 954     type => 'string', format => 'pve-qm-ipconfig',
 955     description => <<'EODESCR',
 956 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 957
 958 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 959
 960 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 961 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 962
 963 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 964 EODESCR
 965 };
 966 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 967
 968 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 969     $confdesc->{"net$i"} = $netdesc;
 970     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 971 }
 972
 973 foreach my $key (keys %$confdesc_cloudinit) {
 974     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 975 }
 976
 977 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 978 sub verify_volume_id_or_qm_path {
 979     my ($volid, $noerr) = @_;
 980
 981     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 982         return $volid;
 983     }
 984
 985     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 986     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 987     if ($@) {
 988         return undef if $noerr;
 989         die $@;
 990     }
 991     return $volid;
 992 }
 993
 994 my $drivename_hash;
 995
 996 my %drivedesc_base = (
 997     volume => { alias => 'file' },
 998     file => {
 999         type => 'string',
1000         format => 'pve-volume-id-or-qm-path',
1001         default_key => 1,
1002         format_description => 'volume',
1003         description => "The drive's backing volume.",
1004     },
1005     media => {
1006         type => 'string',
1007         enum => [qw(cdrom disk)],
1008         description => "The drive's media type.",
1009         default => 'disk',
1010         optional => 1
1011     },
1012     cyls => {
1013         type => 'integer',
1014         description => "Force the drive's physical geometry to have a specific cylinder count.",
1015         optional => 1
1016     },
1017     heads => {
1018         type => 'integer',
1019         description => "Force the drive's physical geometry to have a specific head count.",
1020         optional => 1
1021     },
1022     secs => {
1023         type => 'integer',
1024         description => "Force the drive's physical geometry to have a specific sector count.",
1025         optional => 1
1026     },
1027     trans => {
1028         type => 'string',
1029         enum => [qw(none lba auto)],
1030         description => "Force disk geometry bios translation mode.",
1031         optional => 1,
1032     },
1033     snapshot => {
1034         type => 'boolean',
1035         description => "Controls qemu's snapshot mode feature."
1036             . " If activated, changes made to the disk are temporary and will"
1037             . " be discarded when the VM is shutdown.",
1038         optional => 1,
1039     },
1040     cache => {
1041         type => 'string',
1042         enum => [qw(none writethrough writeback unsafe directsync)],
1043         description => "The drive's cache mode",
1044         optional => 1,
1045     },
1046     format => get_standard_option('pve-qm-image-format'),
1047     size => {
1048         type => 'string',
1049         format => 'disk-size',
1050         format_description => 'DiskSize',
1051         description => "Disk size. This is purely informational and has no effect.",
1052         optional => 1,
1053     },
1054     backup => {
1055         type => 'boolean',
1056         description => "Whether the drive should be included when making backups.",
1057         optional => 1,
1058     },
1059     replicate => {
1060         type => 'boolean',
1061         description => 'Whether the drive should considered for replication jobs.',
1062         optional => 1,
1063         default => 1,
1064     },
1065     rerror => {
1066         type => 'string',
1067         enum => [qw(ignore report stop)],
1068         description => 'Read error action.',
1069         optional => 1,
1070     },
1071     werror => {
1072         type => 'string',
1073         enum => [qw(enospc ignore report stop)],
1074         description => 'Write error action.',
1075         optional => 1,
1076     },
1077     aio => {
1078         type => 'string',
1079         enum => [qw(native threads)],
1080         description => 'AIO type to use.',
1081         optional => 1,
1082     },
1083     discard => {
1084         type => 'string',
1085         enum => [qw(ignore on)],
1086         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1087         optional => 1,
1088     },
1089     detect_zeroes => {
1090         type => 'boolean',
1091         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1092         optional => 1,
1093     },
1094     serial => {
1095         type => 'string',
1096         format => 'urlencoded',
1097         format_description => 'serial',
1098         maxLength => 20*3, # *3 since it's %xx url enoded
1099         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1100         optional => 1,
1101     },
1102     shared => {
1103         type => 'boolean',
1104         description => 'Mark this locally-managed volume as available on all nodes',
1105         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1106         optional => 1,
1107         default => 0,
1108     }
1109 );
1110
1111 my %iothread_fmt = ( iothread => {
1112         type => 'boolean',
1113         description => "Whether to use iothreads for this drive",
1114         optional => 1,
1115 });
1116
1117 my %model_fmt = (
1118     model => {
1119         type => 'string',
1120         format => 'urlencoded',
1121         format_description => 'model',
1122         maxLength => 40*3, # *3 since it's %xx url enoded
1123         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1124         optional => 1,
1125     },
1126 );
1127
1128 my %queues_fmt = (
1129     queues => {
1130         type => 'integer',
1131         description => "Number of queues.",
1132         minimum => 2,
1133         optional => 1
1134     }
1135 );
1136
1137 my %scsiblock_fmt = (
1138     scsiblock => {
1139         type => 'boolean',
1140         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1141         optional => 1,
1142         default => 0,
1143     },
1144 );
1145
1146 my %ssd_fmt = (
1147     ssd => {
1148         type => 'boolean',
1149         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1150         optional => 1,
1151     },
1152 );
1153
1154 my %wwn_fmt = (
1155     wwn => {
1156         type => 'string',
1157         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1158         format_description => 'wwn',
1159         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1160         optional => 1,
1161     },
1162 );
1163
1164 my $add_throttle_desc = sub {
1165     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1166     my $d = {
1167         type => $type,
1168         format_description => $unit,
1169         description => "Maximum $what in $longunit.",
1170         optional => 1,
1171     };
1172     $d->{minimum} = $minimum if defined($minimum);
1173     $drivedesc_base{$key} = $d;
1174 };
1175 # throughput: (leaky bucket)
1176 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1177 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1178 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1179 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1180 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1181 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1182 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1183 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1184 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1185
1186 # pools: (pool of IO before throttling starts taking effect)
1187 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1189 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1190 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1191 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1192 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1193
1194 # burst lengths
1195 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1196 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1197 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1198 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1199 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1200 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1201
1202 # legacy support
1203 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1204 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1205 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1206 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1207
1208 my $ide_fmt = {
1209     %drivedesc_base,
1210     %model_fmt,
1211     %ssd_fmt,
1212     %wwn_fmt,
1213 };
1214 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1215
1216 my $idedesc = {
1217     optional => 1,
1218     type => 'string', format => $ide_fmt,
1219     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1220 };
1221 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1222
1223 my $scsi_fmt = {
1224     %drivedesc_base,
1225     %iothread_fmt,
1226     %queues_fmt,
1227     %scsiblock_fmt,
1228     %ssd_fmt,
1229     %wwn_fmt,
1230 };
1231 my $scsidesc = {
1232     optional => 1,
1233     type => 'string', format => $scsi_fmt,
1234     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1235 };
1236 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1237
1238 my $sata_fmt = {
1239     %drivedesc_base,
1240     %ssd_fmt,
1241     %wwn_fmt,
1242 };
1243 my $satadesc = {
1244     optional => 1,
1245     type => 'string', format => $sata_fmt,
1246     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1247 };
1248 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1249
1250 my $virtio_fmt = {
1251     %drivedesc_base,
1252     %iothread_fmt,
1253 };
1254 my $virtiodesc = {
1255     optional => 1,
1256     type => 'string', format => $virtio_fmt,
1257     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1258 };
1259 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1260
1261 my $alldrive_fmt = {
1262     %drivedesc_base,
1263     %iothread_fmt,
1264     %model_fmt,
1265     %queues_fmt,
1266     %scsiblock_fmt,
1267     %ssd_fmt,
1268     %wwn_fmt,
1269 };
1270
1271 my $efidisk_fmt = {
1272     volume => { alias => 'file' },
1273     file => {
1274         type => 'string',
1275         format => 'pve-volume-id-or-qm-path',
1276         default_key => 1,
1277         format_description => 'volume',
1278         description => "The drive's backing volume.",
1279     },
1280     format => get_standard_option('pve-qm-image-format'),
1281     size => {
1282         type => 'string',
1283         format => 'disk-size',
1284         format_description => 'DiskSize',
1285         description => "Disk size. This is purely informational and has no effect.",
1286         optional => 1,
1287     },
1288 };
1289
1290 my $efidisk_desc = {
1291     optional => 1,
1292     type => 'string', format => $efidisk_fmt,
1293     description => "Configure a Disk for storing EFI vars",
1294 };
1295
1296 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1297
1298 my $usb_fmt = {
1299     host => {
1300         default_key => 1,
1301         type => 'string', format => 'pve-qm-usb-device',
1302         format_description => 'HOSTUSBDEVICE|spice',
1303         description => <<EODESCR,
1304 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1305
1306  'bus-port(.port)*' (decimal numbers) or
1307  'vendor_id:product_id' (hexadeciaml numbers) or
1308  'spice'
1309
1310 You can use the 'lsusb -t' command to list existing usb devices.
1311
1312 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1313
1314 The value 'spice' can be used to add a usb redirection devices for spice.
1315 EODESCR
1316     },
1317     usb3 => {
1318         optional => 1,
1319         type => 'boolean',
1320         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1321         default => 0,
1322     },
1323 };
1324
1325 my $usbdesc = {
1326     optional => 1,
1327     type => 'string', format => $usb_fmt,
1328     description => "Configure an USB device (n is 0 to 4).",
1329 };
1330 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1331
1332 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1333 my $hostpci_fmt = {
1334     host => {
1335         default_key => 1,
1336         type => 'string',
1337         pattern => qr/$PCIRE(;$PCIRE)*/,
1338         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1339         description => <<EODESCR,
1340 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1341 of PCI virtual functions of the host. HOSTPCIID syntax is:
1342
1343 'bus:dev.func' (hexadecimal numbers)
1344
1345 You can us the 'lspci' command to list existing PCI devices.
1346 EODESCR
1347     },
1348     rombar => {
1349         type => 'boolean',
1350         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1351         optional => 1,
1352         default => 1,
1353     },
1354     romfile => {
1355         type => 'string',
1356         pattern => '[^,;]+',
1357         format_description => 'string',
1358         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1359         optional => 1,
1360     },
1361     pcie => {
1362         type => 'boolean',
1363         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1364         optional => 1,
1365         default => 0,
1366     },
1367     'x-vga' => {
1368         type => 'boolean',
1369         description =>  "Enable vfio-vga device support.",
1370         optional => 1,
1371         default => 0,
1372     },
1373     'mdev' => {
1374         type => 'string',
1375         format_description => 'string',
1376         pattern => '[^/\.:]+',
1377         optional => 1,
1378         description => <<EODESCR
1379 The type of mediated device to use.
1380 An instance of this type will be created on startup of the VM and
1381 will be cleaned up when the VM stops.
1382 EODESCR
1383     }
1384 };
1385 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1386
1387 my $hostpcidesc = {
1388         optional => 1,
1389         type => 'string', format => 'pve-qm-hostpci',
1390         description => "Map host PCI devices into guest.",
1391         verbose_description =>  <<EODESCR,
1392 Map host PCI devices into guest.
1393
1394 NOTE: This option allows direct access to host hardware. So it is no longer
1395 possible to migrate such machines - use with special care.
1396
1397 CAUTION: Experimental! User reported problems with this option.
1398 EODESCR
1399 };
1400 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1401
1402 my $serialdesc = {
1403         optional => 1,
1404         type => 'string',
1405         pattern => '(/dev/.+|socket)',
1406         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1407         verbose_description =>  <<EODESCR,
1408 Create a serial device inside the VM (n is 0 to 3), and pass through a
1409 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1410 host side (use 'qm terminal' to open a terminal connection).
1411
1412 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1413
1414 CAUTION: Experimental! User reported problems with this option.
1415 EODESCR
1416 };
1417
1418 my $paralleldesc= {
1419         optional => 1,
1420         type => 'string',
1421         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1422         description =>  "Map host parallel devices (n is 0 to 2).",
1423         verbose_description =>  <<EODESCR,
1424 Map host parallel devices (n is 0 to 2).
1425
1426 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1427
1428 CAUTION: Experimental! User reported problems with this option.
1429 EODESCR
1430 };
1431
1432 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1433     $confdesc->{"parallel$i"} = $paralleldesc;
1434 }
1435
1436 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1437     $confdesc->{"serial$i"} = $serialdesc;
1438 }
1439
1440 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1441     $confdesc->{"hostpci$i"} = $hostpcidesc;
1442 }
1443
1444 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1445     $drivename_hash->{"ide$i"} = 1;
1446     $confdesc->{"ide$i"} = $idedesc;
1447 }
1448
1449 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1450     $drivename_hash->{"sata$i"} = 1;
1451     $confdesc->{"sata$i"} = $satadesc;
1452 }
1453
1454 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1455     $drivename_hash->{"scsi$i"} = 1;
1456     $confdesc->{"scsi$i"} = $scsidesc ;
1457 }
1458
1459 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1460     $drivename_hash->{"virtio$i"} = 1;
1461     $confdesc->{"virtio$i"} = $virtiodesc;
1462 }
1463
1464 $drivename_hash->{efidisk0} = 1;
1465 $confdesc->{efidisk0} = $efidisk_desc;
1466
1467 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1468     $confdesc->{"usb$i"} = $usbdesc;
1469 }
1470
1471 my $unuseddesc = {
1472     optional => 1,
1473     type => 'string', format => 'pve-volume-id',
1474     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1475 };
1476
1477 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1478     $confdesc->{"unused$i"} = $unuseddesc;
1479 }
1480
1481 my $kvm_api_version = 0;
1482
1483 sub kvm_version {
1484     return $kvm_api_version if $kvm_api_version;
1485
1486     open my $fh, '<', '/dev/kvm'
1487         or return undef;
1488
1489     # 0xae00 => KVM_GET_API_VERSION
1490     $kvm_api_version = ioctl($fh, 0xae00, 0);
1491
1492     return $kvm_api_version;
1493 }
1494
1495 my $kvm_user_version = {};
1496 my $kvm_mtime = {};
1497
1498 sub kvm_user_version {
1499     my ($binary) = @_;
1500
1501     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1502     my $st = stat($binary);
1503
1504     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1505     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1506         $cachedmtime == $st->mtime;
1507
1508     $kvm_user_version->{$binary} = 'unknown';
1509     $kvm_mtime->{$binary} = $st->mtime;
1510
1511     my $code = sub {
1512         my $line = shift;
1513         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1514             $kvm_user_version->{$binary} = $2;
1515         }
1516     };
1517
1518     eval { run_command([$binary, '--version'], outfunc => $code); };
1519     warn $@ if $@;
1520
1521     return $kvm_user_version->{$binary};
1522
1523 }
1524
1525 sub kernel_has_vhost_net {
1526     return -c '/dev/vhost-net';
1527 }
1528
1529 sub valid_drive_names {
1530     # order is important - used to autoselect boot disk
1531     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1532             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1533             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1534             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1535             'efidisk0');
1536 }
1537
1538 sub is_valid_drivename {
1539     my $dev = shift;
1540
1541     return defined($drivename_hash->{$dev});
1542 }
1543
1544 sub option_exists {
1545     my $key = shift;
1546     return defined($confdesc->{$key});
1547 }
1548
1549 sub nic_models {
1550     return $nic_model_list;
1551 }
1552
1553 sub os_list_description {
1554
1555     return {
1556         other => 'Other',
1557         wxp => 'Windows XP',
1558         w2k => 'Windows 2000',
1559         w2k3 =>, 'Windows 2003',
1560         w2k8 => 'Windows 2008',
1561         wvista => 'Windows Vista',
1562         win7 => 'Windows 7',
1563         win8 => 'Windows 8/2012',
1564         win10 => 'Windows 10/2016',
1565         l24 => 'Linux 2.4',
1566         l26 => 'Linux 2.6',
1567     };
1568 }
1569
1570 my $cdrom_path;
1571
1572 sub get_cdrom_path {
1573
1574     return  $cdrom_path if $cdrom_path;
1575
1576     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1577     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1578     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1579 }
1580
1581 sub get_iso_path {
1582     my ($storecfg, $vmid, $cdrom) = @_;
1583
1584     if ($cdrom eq 'cdrom') {
1585         return get_cdrom_path();
1586     } elsif ($cdrom eq 'none') {
1587         return '';
1588     } elsif ($cdrom =~ m|^/|) {
1589         return $cdrom;
1590     } else {
1591         return PVE::Storage::path($storecfg, $cdrom);
1592     }
1593 }
1594
1595 # try to convert old style file names to volume IDs
1596 sub filename_to_volume_id {
1597     my ($vmid, $file, $media) = @_;
1598
1599      if (!($file eq 'none' || $file eq 'cdrom' ||
1600           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1601
1602         return undef if $file =~ m|/|;
1603
1604         if ($media && $media eq 'cdrom') {
1605             $file = "local:iso/$file";
1606         } else {
1607             $file = "local:$vmid/$file";
1608         }
1609     }
1610
1611     return $file;
1612 }
1613
1614 sub verify_media_type {
1615     my ($opt, $vtype, $media) = @_;
1616
1617     return if !$media;
1618
1619     my $etype;
1620     if ($media eq 'disk') {
1621         $etype = 'images';
1622     } elsif ($media eq 'cdrom') {
1623         $etype = 'iso';
1624     } else {
1625         die "internal error";
1626     }
1627
1628     return if ($vtype eq $etype);
1629
1630     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1631 }
1632
1633 sub cleanup_drive_path {
1634     my ($opt, $storecfg, $drive) = @_;
1635
1636     # try to convert filesystem paths to volume IDs
1637
1638     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1639         ($drive->{file} !~ m|^/dev/.+|) &&
1640         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1641         ($drive->{file} !~ m/^\d+$/)) {
1642         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1643         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1644         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1645         verify_media_type($opt, $vtype, $drive->{media});
1646         $drive->{file} = $volid;
1647     }
1648
1649     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1650 }
1651
1652 sub parse_hotplug_features {
1653     my ($data) = @_;
1654
1655     my $res = {};
1656
1657     return $res if $data eq '0';
1658
1659     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1660
1661     foreach my $feature (PVE::Tools::split_list($data)) {
1662         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1663             $res->{$1} = 1;
1664         } else {
1665             die "invalid hotplug feature '$feature'\n";
1666         }
1667     }
1668     return $res;
1669 }
1670
1671 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1672 sub pve_verify_hotplug_features {
1673     my ($value, $noerr) = @_;
1674
1675     return $value if parse_hotplug_features($value);
1676
1677     return undef if $noerr;
1678
1679     die "unable to parse hotplug option\n";
1680 }
1681
1682 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1683 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1684 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1685 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1686 #        [,iothread=on][,serial=serial][,model=model]
1687
1688 sub parse_drive {
1689     my ($key, $data) = @_;
1690
1691     my ($interface, $index);
1692
1693     if ($key =~ m/^([^\d]+)(\d+)$/) {
1694         $interface = $1;
1695         $index = $2;
1696     } else {
1697         return undef;
1698     }
1699
1700     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1701                                      : $confdesc->{$key}->{format};
1702     if (!$desc) {
1703         warn "invalid drive key: $key\n";
1704         return undef;
1705     }
1706     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1707     return undef if !$res;
1708     $res->{interface} = $interface;
1709     $res->{index} = $index;
1710
1711     my $error = 0;
1712     foreach my $opt (qw(bps bps_rd bps_wr)) {
1713         if (my $bps = defined(delete $res->{$opt})) {
1714             if (defined($res->{"m$opt"})) {
1715                 warn "both $opt and m$opt specified\n";
1716                 ++$error;
1717                 next;
1718             }
1719             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1720         }
1721     }
1722
1723     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1724     for my $requirement (
1725         [mbps_max => 'mbps'],
1726         [mbps_rd_max => 'mbps_rd'],
1727         [mbps_wr_max => 'mbps_wr'],
1728         [miops_max => 'miops'],
1729         [miops_rd_max => 'miops_rd'],
1730         [miops_wr_max => 'miops_wr'],
1731         [bps_max_length => 'mbps_max'],
1732         [bps_rd_max_length => 'mbps_rd_max'],
1733         [bps_wr_max_length => 'mbps_wr_max'],
1734         [iops_max_length => 'iops_max'],
1735         [iops_rd_max_length => 'iops_rd_max'],
1736         [iops_wr_max_length => 'iops_wr_max']) {
1737         my ($option, $requires) = @$requirement;
1738         if ($res->{$option} && !$res->{$requires}) {
1739             warn "$option requires $requires\n";
1740             ++$error;
1741         }
1742     }
1743
1744     return undef if $error;
1745
1746     return undef if $res->{mbps_rd} && $res->{mbps};
1747     return undef if $res->{mbps_wr} && $res->{mbps};
1748     return undef if $res->{iops_rd} && $res->{iops};
1749     return undef if $res->{iops_wr} && $res->{iops};
1750
1751     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1752         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1753         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1754         return undef if $res->{interface} eq 'virtio';
1755     }
1756
1757     if (my $size = $res->{size}) {
1758         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1759     }
1760
1761     return $res;
1762 }
1763
1764 sub print_drive {
1765     my ($vmid, $drive) = @_;
1766     my $data = { %$drive };
1767     delete $data->{$_} for qw(index interface);
1768     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1769 }
1770
1771 sub scsi_inquiry {
1772     my($fh, $noerr) = @_;
1773
1774     my $SG_IO = 0x2285;
1775     my $SG_GET_VERSION_NUM = 0x2282;
1776
1777     my $versionbuf = "\x00" x 8;
1778     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1779     if (!$ret) {
1780         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1781         return undef;
1782     }
1783     my $version = unpack("I", $versionbuf);
1784     if ($version < 30000) {
1785         die "scsi generic interface too old\n"  if !$noerr;
1786         return undef;
1787     }
1788
1789     my $buf = "\x00" x 36;
1790     my $sensebuf = "\x00" x 8;
1791     my $cmd = pack("C x3 C x1", 0x12, 36);
1792
1793     # see /usr/include/scsi/sg.h
1794     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1795
1796     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1797                       length($sensebuf), 0, length($buf), $buf,
1798                       $cmd, $sensebuf, 6000);
1799
1800     $ret = ioctl($fh, $SG_IO, $packet);
1801     if (!$ret) {
1802         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1803         return undef;
1804     }
1805
1806     my @res = unpack($sg_io_hdr_t, $packet);
1807     if ($res[17] || $res[18]) {
1808         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1809         return undef;
1810     }
1811
1812     my $res = {};
1813     (my $byte0, my $byte1, $res->{vendor},
1814      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1815
1816     $res->{removable} = $byte1 & 128 ? 1 : 0;
1817     $res->{type} = $byte0 & 31;
1818
1819     return $res;
1820 }
1821
1822 sub path_is_scsi {
1823     my ($path) = @_;
1824
1825     my $fh = IO::File->new("+<$path") || return undef;
1826     my $res = scsi_inquiry($fh, 1);
1827     close($fh);
1828
1829     return $res;
1830 }
1831
1832 sub machine_type_is_q35 {
1833     my ($conf) = @_;
1834
1835     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1836 }
1837
1838 sub print_tabletdevice_full {
1839     my ($conf, $arch) = @_;
1840
1841     my $q35 = machine_type_is_q35($conf);
1842
1843     # we use uhci for old VMs because tablet driver was buggy in older qemu
1844     my $usbbus;
1845     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1846         $usbbus = 'ehci';
1847     } else {
1848         $usbbus = 'uhci';
1849     }
1850
1851     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1852 }
1853
1854 sub print_keyboarddevice_full {
1855     my ($conf, $arch, $machine) = @_;
1856
1857     return undef if $arch ne 'aarch64';
1858
1859     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1860 }
1861
1862 sub print_drivedevice_full {
1863     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1864
1865     my $device = '';
1866     my $maxdev = 0;
1867
1868     if ($drive->{interface} eq 'virtio') {
1869         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1870         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1871         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1872     } elsif ($drive->{interface} eq 'scsi') {
1873
1874         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1875         my $unit = $drive->{index} % $maxdev;
1876         my $devicetype = 'hd';
1877         my $path = '';
1878         if (drive_is_cdrom($drive)) {
1879             $devicetype = 'cd';
1880         } else {
1881             if ($drive->{file} =~ m|^/|) {
1882                 $path = $drive->{file};
1883                 if (my $info = path_is_scsi($path)) {
1884                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1885                         $devicetype = 'block';
1886                     } elsif ($info->{type} == 1) { # tape
1887                         $devicetype = 'generic';
1888                     }
1889                 }
1890             } else {
1891                  $path = PVE::Storage::path($storecfg, $drive->{file});
1892             }
1893
1894             if($path =~ m/^iscsi\:\/\//){
1895                 $devicetype = 'generic';
1896             }
1897         }
1898
1899         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1900             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1901         } else {
1902             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1903         }
1904
1905         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1906             $device .= ",rotation_rate=1";
1907         }
1908         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1909
1910     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1911         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1912         my $controller = int($drive->{index} / $maxdev);
1913         my $unit = $drive->{index} % $maxdev;
1914         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1915
1916         $device = "ide-$devicetype";
1917         if ($drive->{interface} eq 'ide') {
1918             $device .= ",bus=ide.$controller,unit=$unit";
1919         } else {
1920             $device .= ",bus=ahci$controller.$unit";
1921         }
1922         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1923
1924         if ($devicetype eq 'hd') {
1925             if (my $model = $drive->{model}) {
1926                 $model = URI::Escape::uri_unescape($model);
1927                 $device .= ",model=$model";
1928             }
1929             if ($drive->{ssd}) {
1930                 $device .= ",rotation_rate=1";
1931             }
1932         }
1933         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1934     } elsif ($drive->{interface} eq 'usb') {
1935         die "implement me";
1936         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1937     } else {
1938         die "unsupported interface type";
1939     }
1940
1941     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1942
1943     if (my $serial = $drive->{serial}) {
1944         $serial = URI::Escape::uri_unescape($serial);
1945         $device .= ",serial=$serial";
1946     }
1947
1948
1949     return $device;
1950 }
1951
1952 sub get_initiator_name {
1953     my $initiator;
1954
1955     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1956     while (defined(my $line = <$fh>)) {
1957         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1958         $initiator = $1;
1959         last;
1960     }
1961     $fh->close();
1962
1963     return $initiator;
1964 }
1965
1966 sub print_drive_full {
1967     my ($storecfg, $vmid, $drive) = @_;
1968
1969     my $path;
1970     my $volid = $drive->{file};
1971     my $format;
1972
1973     if (drive_is_cdrom($drive)) {
1974         $path = get_iso_path($storecfg, $vmid, $volid);
1975     } else {
1976         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1977         if ($storeid) {
1978             $path = PVE::Storage::path($storecfg, $volid);
1979             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1980             $format = qemu_img_format($scfg, $volname);
1981         } else {
1982             $path = $volid;
1983             $format = "raw";
1984         }
1985    }
1986
1987     my $opts = '';
1988     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1989     foreach my $o (@qemu_drive_options) {
1990         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1991     }
1992
1993     # snapshot only accepts on|off
1994     if (defined($drive->{snapshot})) {
1995         my $v = $drive->{snapshot} ? 'on' : 'off';
1996         $opts .= ",snapshot=$v";
1997     }
1998
1999     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2000         my ($dir, $qmpname) = @$type;
2001         if (my $v = $drive->{"mbps$dir"}) {
2002             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2003         }
2004         if (my $v = $drive->{"mbps${dir}_max"}) {
2005             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2006         }
2007         if (my $v = $drive->{"bps${dir}_max_length"}) {
2008             $opts .= ",throttling.bps$qmpname-max-length=$v";
2009         }
2010         if (my $v = $drive->{"iops${dir}"}) {
2011             $opts .= ",throttling.iops$qmpname=$v";
2012         }
2013         if (my $v = $drive->{"iops${dir}_max"}) {
2014             $opts .= ",throttling.iops$qmpname-max=$v";
2015         }
2016         if (my $v = $drive->{"iops${dir}_max_length"}) {
2017             $opts .= ",throttling.iops$qmpname-max-length=$v";
2018         }
2019     }
2020
2021     $opts .= ",format=$format" if $format && !$drive->{format};
2022
2023     my $cache_direct = 0;
2024
2025     if (my $cache = $drive->{cache}) {
2026         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2027     } elsif (!drive_is_cdrom($drive)) {
2028         $opts .= ",cache=none";
2029         $cache_direct = 1;
2030     }
2031
2032     # aio native works only with O_DIRECT
2033     if (!$drive->{aio}) {
2034         if($cache_direct) {
2035             $opts .= ",aio=native";
2036         } else {
2037             $opts .= ",aio=threads";
2038         }
2039     }
2040
2041     if (!drive_is_cdrom($drive)) {
2042         my $detectzeroes;
2043         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2044             $detectzeroes = 'off';
2045         } elsif ($drive->{discard}) {
2046             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2047         } else {
2048             # This used to be our default with discard not being specified:
2049             $detectzeroes = 'on';
2050         }
2051         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2052     }
2053
2054     my $pathinfo = $path ? "file=$path," : '';
2055
2056     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2057 }
2058
2059 sub print_netdevice_full {
2060     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2061
2062     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2063
2064     my $device = $net->{model};
2065     if ($net->{model} eq 'virtio') {
2066          $device = 'virtio-net-pci';
2067      };
2068
2069     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2070     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2071     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2072         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2073         my $vectors = $net->{queues} * 2 + 2;
2074         $tmpstr .= ",vectors=$vectors,mq=on";
2075     }
2076     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2077
2078     if ($use_old_bios_files) {
2079         my $romfile;
2080         if ($device eq 'virtio-net-pci') {
2081             $romfile = 'pxe-virtio.rom';
2082         } elsif ($device eq 'e1000') {
2083             $romfile = 'pxe-e1000.rom';
2084         } elsif ($device eq 'ne2k') {
2085             $romfile = 'pxe-ne2k_pci.rom';
2086         } elsif ($device eq 'pcnet') {
2087             $romfile = 'pxe-pcnet.rom';
2088         } elsif ($device eq 'rtl8139') {
2089             $romfile = 'pxe-rtl8139.rom';
2090         }
2091         $tmpstr .= ",romfile=$romfile" if $romfile;
2092     }
2093
2094     return $tmpstr;
2095 }
2096
2097 sub print_netdev_full {
2098     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2099
2100     my $i = '';
2101     if ($netid =~ m/^net(\d+)$/) {
2102         $i = int($1);
2103     }
2104
2105     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2106
2107     my $ifname = "tap${vmid}i$i";
2108
2109     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2110     die "interface name '$ifname' is too long (max 15 character)\n"
2111         if length($ifname) >= 16;
2112
2113     my $vhostparam = '';
2114     if (is_native($arch)) {
2115         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2116     }
2117
2118     my $vmname = $conf->{name} || "vm$vmid";
2119
2120     my $netdev = "";
2121     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2122
2123     if ($net->{bridge}) {
2124         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2125     } else {
2126         $netdev = "type=user,id=$netid,hostname=$vmname";
2127     }
2128
2129     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2130
2131     return $netdev;
2132 }
2133
2134
2135 sub print_cpu_device {
2136     my ($conf, $id) = @_;
2137
2138     my $kvm = $conf->{kvm} // 1;
2139     my $cpu = $kvm ? "kvm64" : "qemu64";
2140     if (my $cputype = $conf->{cpu}) {
2141         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2142             or die "Cannot parse cpu description: $cputype\n";
2143         $cpu = $cpuconf->{cputype};
2144     }
2145
2146     my $cores = $conf->{cores} || 1;
2147
2148     my $current_core = ($id - 1) % $cores;
2149     my $current_socket = int(($id - 1 - $current_core)/$cores);
2150
2151     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2152 }
2153
2154 my $vga_map = {
2155     'cirrus' => 'cirrus-vga',
2156     'std' => 'VGA',
2157     'vmware' => 'vmware-svga',
2158     'virtio' => 'virtio-vga',
2159 };
2160
2161 sub print_vga_device {
2162     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2163
2164     my $type = $vga_map->{$vga->{type}};
2165     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2166         $type = 'virtio-gpu';
2167     }
2168     my $vgamem_mb = $vga->{memory};
2169     if ($qxlnum) {
2170         $type = $id ? 'qxl' : 'qxl-vga';
2171     }
2172     die "no devicetype for $vga->{type}\n" if !$type;
2173
2174     my $memory = "";
2175     if ($vgamem_mb) {
2176         if ($vga->{type} eq 'virtio') {
2177             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2178             $memory = ",max_hostmem=$bytes";
2179         } elsif ($qxlnum) {
2180             # from https://www.spice-space.org/multiple-monitors.html
2181             $memory = ",vgamem_mb=$vga->{memory}";
2182             my $ram = $vgamem_mb * 4;
2183             my $vram = $vgamem_mb * 2;
2184             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2185         } else {
2186             $memory = ",vgamem_mb=$vga->{memory}";
2187         }
2188     } elsif ($qxlnum && $id) {
2189         $memory = ",ram_size=67108864,vram_size=33554432";
2190     }
2191
2192     my $q35 = machine_type_is_q35($conf);
2193     my $vgaid = "vga" . ($id // '');
2194     my $pciaddr;
2195
2196     if ($q35 && $vgaid eq 'vga') {
2197         # the first display uses pcie.0 bus on q35 machines
2198         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2199     } else {
2200         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2201     }
2202
2203     return "$type,id=${vgaid}${memory}${pciaddr}";
2204 }
2205
2206 sub drive_is_cloudinit {
2207     my ($drive) = @_;
2208     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2209 }
2210
2211 sub drive_is_cdrom {
2212     my ($drive, $exclude_cloudinit) = @_;
2213
2214     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2215
2216     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2217
2218 }
2219
2220 sub parse_number_sets {
2221     my ($set) = @_;
2222     my $res = [];
2223     foreach my $part (split(/;/, $set)) {
2224         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2225             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2226             push @$res, [ $1, $2 ];
2227         } else {
2228             die "invalid range: $part\n";
2229         }
2230     }
2231     return $res;
2232 }
2233
2234 sub parse_numa {
2235     my ($data) = @_;
2236
2237     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2238     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2239     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2240     return $res;
2241 }
2242
2243 sub parse_hostpci {
2244     my ($value) = @_;
2245
2246     return undef if !$value;
2247
2248     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2249
2250     my @idlist = split(/;/, $res->{host});
2251     delete $res->{host};
2252     foreach my $id (@idlist) {
2253         if ($id =~ m/\./) { # full id 00:00.1
2254             push @{$res->{pciid}}, {
2255                 id => $id,
2256             };
2257         } else { # partial id 00:00
2258             $res->{pciid} = PVE::SysFSTools::lspci($id);
2259         }
2260     }
2261     return $res;
2262 }
2263
2264 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2265 sub parse_net {
2266     my ($data) = @_;
2267
2268     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2269     if ($@) {
2270         warn $@;
2271         return undef;
2272     }
2273     if (!defined($res->{macaddr})) {
2274         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2275         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2276     }
2277     return $res;
2278 }
2279
2280 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2281 sub parse_ipconfig {
2282     my ($data) = @_;
2283
2284     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2285     if ($@) {
2286         warn $@;
2287         return undef;
2288     }
2289
2290     if ($res->{gw} && !$res->{ip}) {
2291         warn 'gateway specified without specifying an IP address';
2292         return undef;
2293     }
2294     if ($res->{gw6} && !$res->{ip6}) {
2295         warn 'IPv6 gateway specified without specifying an IPv6 address';
2296         return undef;
2297     }
2298     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2299         warn 'gateway specified together with DHCP';
2300         return undef;
2301     }
2302     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2303         # gw6 + auto/dhcp
2304         warn "IPv6 gateway specified together with $res->{ip6} address";
2305         return undef;
2306     }
2307
2308     if (!$res->{ip} && !$res->{ip6}) {
2309         return { ip => 'dhcp', ip6 => 'dhcp' };
2310     }
2311
2312     return $res;
2313 }
2314
2315 sub print_net {
2316     my $net = shift;
2317
2318     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2319 }
2320
2321 sub add_random_macs {
2322     my ($settings) = @_;
2323
2324     foreach my $opt (keys %$settings) {
2325         next if $opt !~ m/^net(\d+)$/;
2326         my $net = parse_net($settings->{$opt});
2327         next if !$net;
2328         $settings->{$opt} = print_net($net);
2329     }
2330 }
2331
2332 sub vm_is_volid_owner {
2333     my ($storecfg, $vmid, $volid) = @_;
2334
2335     if ($volid !~  m|^/|) {
2336         my ($path, $owner);
2337         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2338         if ($owner && ($owner == $vmid)) {
2339             return 1;
2340         }
2341     }
2342
2343     return undef;
2344 }
2345
2346 sub split_flagged_list {
2347     my $text = shift || '';
2348     $text =~ s/[,;]/ /g;
2349     $text =~ s/^\s+//;
2350     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2351 }
2352
2353 sub join_flagged_list {
2354     my ($how, $lst) = @_;
2355     join $how, map { $lst->{$_} . $_ } keys %$lst;
2356 }
2357
2358 sub vmconfig_delete_pending_option {
2359     my ($conf, $key, $force) = @_;
2360
2361     delete $conf->{pending}->{$key};
2362     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2363     $pending_delete_hash->{$key} = $force ? '!' : '';
2364     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2365 }
2366
2367 sub vmconfig_undelete_pending_option {
2368     my ($conf, $key) = @_;
2369
2370     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2371     delete $pending_delete_hash->{$key};
2372
2373     if (%$pending_delete_hash) {
2374         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2375     } else {
2376         delete $conf->{pending}->{delete};
2377     }
2378 }
2379
2380 sub vmconfig_register_unused_drive {
2381     my ($storecfg, $vmid, $conf, $drive) = @_;
2382
2383     if (drive_is_cloudinit($drive)) {
2384         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2385         warn $@ if $@;
2386     } elsif (!drive_is_cdrom($drive)) {
2387         my $volid = $drive->{file};
2388         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2389             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2390         }
2391     }
2392 }
2393
2394 sub vmconfig_cleanup_pending {
2395     my ($conf) = @_;
2396
2397     # remove pending changes when nothing changed
2398     my $changes;
2399     foreach my $opt (keys %{$conf->{pending}}) {
2400         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2401             $changes = 1;
2402             delete $conf->{pending}->{$opt};
2403         }
2404     }
2405
2406     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2407     my $pending_delete_hash = {};
2408     while (my ($opt, $force) = each %$current_delete_hash) {
2409         if (defined($conf->{$opt})) {
2410             $pending_delete_hash->{$opt} = $force;
2411         } else {
2412             $changes = 1;
2413         }
2414     }
2415
2416     if (%$pending_delete_hash) {
2417         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2418     } else {
2419         delete $conf->{pending}->{delete};
2420     }
2421
2422     return $changes;
2423 }
2424
2425 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2426 my $smbios1_fmt = {
2427     uuid => {
2428         type => 'string',
2429         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2430         format_description => 'UUID',
2431         description => "Set SMBIOS1 UUID.",
2432         optional => 1,
2433     },
2434     version => {
2435         type => 'string',
2436         pattern => '[A-Za-z0-9+\/]+={0,2}',
2437         format_description => 'Base64 encoded string',
2438         description => "Set SMBIOS1 version.",
2439         optional => 1,
2440     },
2441     serial => {
2442         type => 'string',
2443         pattern => '[A-Za-z0-9+\/]+={0,2}',
2444         format_description => 'Base64 encoded string',
2445         description => "Set SMBIOS1 serial number.",
2446         optional => 1,
2447     },
2448     manufacturer => {
2449         type => 'string',
2450         pattern => '[A-Za-z0-9+\/]+={0,2}',
2451         format_description => 'Base64 encoded string',
2452         description => "Set SMBIOS1 manufacturer.",
2453         optional => 1,
2454     },
2455     product => {
2456         type => 'string',
2457         pattern => '[A-Za-z0-9+\/]+={0,2}',
2458         format_description => 'Base64 encoded string',
2459         description => "Set SMBIOS1 product ID.",
2460         optional => 1,
2461     },
2462     sku => {
2463         type => 'string',
2464         pattern => '[A-Za-z0-9+\/]+={0,2}',
2465         format_description => 'Base64 encoded string',
2466         description => "Set SMBIOS1 SKU string.",
2467         optional => 1,
2468     },
2469     family => {
2470         type => 'string',
2471         pattern => '[A-Za-z0-9+\/]+={0,2}',
2472         format_description => 'Base64 encoded string',
2473         description => "Set SMBIOS1 family string.",
2474         optional => 1,
2475     },
2476     base64 => {
2477         type => 'boolean',
2478         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2479         optional => 1,
2480     },
2481 };
2482
2483 sub parse_smbios1 {
2484     my ($data) = @_;
2485
2486     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2487     warn $@ if $@;
2488     return $res;
2489 }
2490
2491 sub print_smbios1 {
2492     my ($smbios1) = @_;
2493     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2494 }
2495
2496 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2497
2498 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2499 sub verify_bootdisk {
2500     my ($value, $noerr) = @_;
2501
2502     return $value if is_valid_drivename($value);
2503
2504     return undef if $noerr;
2505
2506     die "invalid boot disk '$value'\n";
2507 }
2508
2509 sub parse_watchdog {
2510     my ($value) = @_;
2511
2512     return undef if !$value;
2513
2514     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2515     warn $@ if $@;
2516     return $res;
2517 }
2518
2519 sub parse_guest_agent {
2520     my ($value) = @_;
2521
2522     return {} if !defined($value->{agent});
2523
2524     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2525     warn $@ if $@;
2526
2527     # if the agent is disabled ignore the other potentially set properties
2528     return {} if !$res->{enabled};
2529     return $res;
2530 }
2531
2532 sub parse_vga {
2533     my ($value) = @_;
2534
2535     return {} if !$value;
2536     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2537     warn $@ if $@;
2538     return $res;
2539 }
2540
2541 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2542 sub verify_usb_device {
2543     my ($value, $noerr) = @_;
2544
2545     return $value if parse_usb_device($value);
2546
2547     return undef if $noerr;
2548
2549     die "unable to parse usb device\n";
2550 }
2551
2552 # add JSON properties for create and set function
2553 sub json_config_properties {
2554     my $prop = shift;
2555
2556     foreach my $opt (keys %$confdesc) {
2557         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2558         $prop->{$opt} = $confdesc->{$opt};
2559     }
2560
2561     return $prop;
2562 }
2563
2564 # return copy of $confdesc_cloudinit to generate documentation
2565 sub cloudinit_config_properties {
2566
2567     return dclone($confdesc_cloudinit);
2568 }
2569
2570 sub check_type {
2571     my ($key, $value) = @_;
2572
2573     die "unknown setting '$key'\n" if !$confdesc->{$key};
2574
2575     my $type = $confdesc->{$key}->{type};
2576
2577     if (!defined($value)) {
2578         die "got undefined value\n";
2579     }
2580
2581     if ($value =~ m/[\n\r]/) {
2582         die "property contains a line feed\n";
2583     }
2584
2585     if ($type eq 'boolean') {
2586         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2587         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2588         die "type check ('boolean') failed - got '$value'\n";
2589     } elsif ($type eq 'integer') {
2590         return int($1) if $value =~ m/^(\d+)$/;
2591         die "type check ('integer') failed - got '$value'\n";
2592     } elsif ($type eq 'number') {
2593         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2594         die "type check ('number') failed - got '$value'\n";
2595     } elsif ($type eq 'string') {
2596         if (my $fmt = $confdesc->{$key}->{format}) {
2597             PVE::JSONSchema::check_format($fmt, $value);
2598             return $value;
2599         }
2600         $value =~ s/^\"(.*)\"$/$1/;
2601         return $value;
2602     } else {
2603         die "internal error"
2604     }
2605 }
2606
2607 sub touch_config {
2608     my ($vmid) = @_;
2609
2610     my $conf = PVE::QemuConfig->config_file($vmid);
2611     utime undef, undef, $conf;
2612 }
2613
2614 sub destroy_vm {
2615     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2616
2617     my $conffile = PVE::QemuConfig->config_file($vmid);
2618
2619     my $conf = PVE::QemuConfig->load_config($vmid);
2620
2621     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2622
2623     if ($conf->{template}) {
2624         # check if any base image is still used by a linked clone
2625         foreach_drive($conf, sub {
2626                 my ($ds, $drive) = @_;
2627
2628                 return if drive_is_cdrom($drive);
2629
2630                 my $volid = $drive->{file};
2631
2632                 return if !$volid || $volid =~ m|^/|;
2633
2634                 die "base volume '$volid' is still in use by linked cloned\n"
2635                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2636
2637         });
2638     }
2639
2640     # only remove disks owned by this VM
2641     foreach_drive($conf, sub {
2642         my ($ds, $drive) = @_;
2643
2644         return if drive_is_cdrom($drive, 1);
2645
2646         my $volid = $drive->{file};
2647
2648         return if !$volid || $volid =~ m|^/|;
2649
2650         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2651         return if !$path || !$owner || ($owner != $vmid);
2652
2653         eval {
2654             PVE::Storage::vdisk_free($storecfg, $volid);
2655         };
2656         warn "Could not remove disk '$volid', check manually: $@" if $@;
2657
2658     });
2659
2660     if ($keep_empty_config) {
2661         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2662     } else {
2663         unlink $conffile;
2664     }
2665
2666     # also remove unused disk
2667     eval {
2668         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2669
2670         eval {
2671             PVE::Storage::foreach_volid($dl, sub {
2672                 my ($volid, $sid, $volname, $d) = @_;
2673                 PVE::Storage::vdisk_free($storecfg, $volid);
2674             });
2675         };
2676         warn $@ if $@;
2677
2678     };
2679     warn $@ if $@;
2680 }
2681
2682 sub parse_vm_config {
2683     my ($filename, $raw) = @_;
2684
2685     return undef if !defined($raw);
2686
2687     my $res = {
2688         digest => Digest::SHA::sha1_hex($raw),
2689         snapshots => {},
2690         pending => {},
2691     };
2692
2693     $filename =~ m|/qemu-server/(\d+)\.conf$|
2694         || die "got strange filename '$filename'";
2695
2696     my $vmid = $1;
2697
2698     my $conf = $res;
2699     my $descr;
2700     my $section = '';
2701
2702     my @lines = split(/\n/, $raw);
2703     foreach my $line (@lines) {
2704         next if $line =~ m/^\s*$/;
2705
2706         if ($line =~ m/^\[PENDING\]\s*$/i) {
2707             $section = 'pending';
2708             if (defined($descr)) {
2709                 $descr =~ s/\s+$//;
2710                 $conf->{description} = $descr;
2711             }
2712             $descr = undef;
2713             $conf = $res->{$section} = {};
2714             next;
2715
2716         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2717             $section = $1;
2718             if (defined($descr)) {
2719                 $descr =~ s/\s+$//;
2720                 $conf->{description} = $descr;
2721             }
2722             $descr = undef;
2723             $conf = $res->{snapshots}->{$section} = {};
2724             next;
2725         }
2726
2727         if ($line =~ m/^\#(.*)\s*$/) {
2728             $descr = '' if !defined($descr);
2729             $descr .= PVE::Tools::decode_text($1) . "\n";
2730             next;
2731         }
2732
2733         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2734             $descr = '' if !defined($descr);
2735             $descr .= PVE::Tools::decode_text($2);
2736         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2737             $conf->{snapstate} = $1;
2738         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2739             my $key = $1;
2740             my $value = $2;
2741             $conf->{$key} = $value;
2742         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2743             my $value = $1;
2744             if ($section eq 'pending') {
2745                 $conf->{delete} = $value; # we parse this later
2746             } else {
2747                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2748             }
2749         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2750             my $key = $1;
2751             my $value = $2;
2752             eval { $value = check_type($key, $value); };
2753             if ($@) {
2754                 warn "vm $vmid - unable to parse value of '$key' - $@";
2755             } else {
2756                 $key = 'ide2' if $key eq 'cdrom';
2757                 my $fmt = $confdesc->{$key}->{format};
2758                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2759                     my $v = parse_drive($key, $value);
2760                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2761                         $v->{file} = $volid;
2762                         $value = print_drive($vmid, $v);
2763                     } else {
2764                         warn "vm $vmid - unable to parse value of '$key'\n";
2765                         next;
2766                     }
2767                 }
2768
2769                 $conf->{$key} = $value;
2770             }
2771         }
2772     }
2773
2774     if (defined($descr)) {
2775         $descr =~ s/\s+$//;
2776         $conf->{description} = $descr;
2777     }
2778     delete $res->{snapstate}; # just to be sure
2779
2780     return $res;
2781 }
2782
2783 sub write_vm_config {
2784     my ($filename, $conf) = @_;
2785
2786     delete $conf->{snapstate}; # just to be sure
2787
2788     if ($conf->{cdrom}) {
2789         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2790         $conf->{ide2} = $conf->{cdrom};
2791         delete $conf->{cdrom};
2792     }
2793
2794     # we do not use 'smp' any longer
2795     if ($conf->{sockets}) {
2796         delete $conf->{smp};
2797     } elsif ($conf->{smp}) {
2798         $conf->{sockets} = $conf->{smp};
2799         delete $conf->{cores};
2800         delete $conf->{smp};
2801     }
2802
2803     my $used_volids = {};
2804
2805     my $cleanup_config = sub {
2806         my ($cref, $pending, $snapname) = @_;
2807
2808         foreach my $key (keys %$cref) {
2809             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2810                 $key eq 'snapstate' || $key eq 'pending';
2811             my $value = $cref->{$key};
2812             if ($key eq 'delete') {
2813                 die "propertry 'delete' is only allowed in [PENDING]\n"
2814                     if !$pending;
2815                 # fixme: check syntax?
2816                 next;
2817             }
2818             eval { $value = check_type($key, $value); };
2819             die "unable to parse value of '$key' - $@" if $@;
2820
2821             $cref->{$key} = $value;
2822
2823             if (!$snapname && is_valid_drivename($key)) {
2824                 my $drive = parse_drive($key, $value);
2825                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2826             }
2827         }
2828     };
2829
2830     &$cleanup_config($conf);
2831
2832     &$cleanup_config($conf->{pending}, 1);
2833
2834     foreach my $snapname (keys %{$conf->{snapshots}}) {
2835         die "internal error" if $snapname eq 'pending';
2836         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2837     }
2838
2839     # remove 'unusedX' settings if we re-add a volume
2840     foreach my $key (keys %$conf) {
2841         my $value = $conf->{$key};
2842         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2843             delete $conf->{$key};
2844         }
2845     }
2846
2847     my $generate_raw_config = sub {
2848         my ($conf, $pending) = @_;
2849
2850         my $raw = '';
2851
2852         # add description as comment to top of file
2853         if (defined(my $descr = $conf->{description})) {
2854             if ($descr) {
2855                 foreach my $cl (split(/\n/, $descr)) {
2856                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2857                 }
2858             } else {
2859                 $raw .= "#\n" if $pending;
2860             }
2861         }
2862
2863         foreach my $key (sort keys %$conf) {
2864             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2865             $raw .= "$key: $conf->{$key}\n";
2866         }
2867         return $raw;
2868     };
2869
2870     my $raw = &$generate_raw_config($conf);
2871
2872     if (scalar(keys %{$conf->{pending}})){
2873         $raw .= "\n[PENDING]\n";
2874         $raw .= &$generate_raw_config($conf->{pending}, 1);
2875     }
2876
2877     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2878         $raw .= "\n[$snapname]\n";
2879         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2880     }
2881
2882     return $raw;
2883 }
2884
2885 sub load_defaults {
2886
2887     my $res = {};
2888
2889     # we use static defaults from our JSON schema configuration
2890     foreach my $key (keys %$confdesc) {
2891         if (defined(my $default = $confdesc->{$key}->{default})) {
2892             $res->{$key} = $default;
2893         }
2894     }
2895
2896     return $res;
2897 }
2898
2899 sub config_list {
2900     my $vmlist = PVE::Cluster::get_vmlist();
2901     my $res = {};
2902     return $res if !$vmlist || !$vmlist->{ids};
2903     my $ids = $vmlist->{ids};
2904
2905     foreach my $vmid (keys %$ids) {
2906         my $d = $ids->{$vmid};
2907         next if !$d->{node} || $d->{node} ne $nodename;
2908         next if !$d->{type} || $d->{type} ne 'qemu';
2909         $res->{$vmid}->{exists} = 1;
2910     }
2911     return $res;
2912 }
2913
2914 # test if VM uses local resources (to prevent migration)
2915 sub check_local_resources {
2916     my ($conf, $noerr) = @_;
2917
2918     my @loc_res = ();
2919
2920     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2921     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2922
2923     push @loc_res, "ivshmem" if $conf->{ivshmem};
2924
2925     foreach my $k (keys %$conf) {
2926         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2927         # sockets are safe: they will recreated be on the target side post-migrate
2928         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2929         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2930     }
2931
2932     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2933
2934     return \@loc_res;
2935 }
2936
2937 # check if used storages are available on all nodes (use by migrate)
2938 sub check_storage_availability {
2939     my ($storecfg, $conf, $node) = @_;
2940
2941     foreach_drive($conf, sub {
2942         my ($ds, $drive) = @_;
2943
2944         my $volid = $drive->{file};
2945         return if !$volid;
2946
2947         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2948         return if !$sid;
2949
2950         # check if storage is available on both nodes
2951         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2952         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2953    });
2954 }
2955
2956 # list nodes where all VM images are available (used by has_feature API)
2957 sub shared_nodes {
2958     my ($conf, $storecfg) = @_;
2959
2960     my $nodelist = PVE::Cluster::get_nodelist();
2961     my $nodehash = { map { $_ => 1 } @$nodelist };
2962     my $nodename = PVE::INotify::nodename();
2963
2964     foreach_drive($conf, sub {
2965         my ($ds, $drive) = @_;
2966
2967         my $volid = $drive->{file};
2968         return if !$volid;
2969
2970         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2971         if ($storeid) {
2972             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2973             if ($scfg->{disable}) {
2974                 $nodehash = {};
2975             } elsif (my $avail = $scfg->{nodes}) {
2976                 foreach my $node (keys %$nodehash) {
2977                     delete $nodehash->{$node} if !$avail->{$node};
2978                 }
2979             } elsif (!$scfg->{shared}) {
2980                 foreach my $node (keys %$nodehash) {
2981                     delete $nodehash->{$node} if $node ne $nodename
2982                 }
2983             }
2984         }
2985     });
2986
2987     return $nodehash
2988 }
2989
2990 sub check_local_storage_availability {
2991     my ($conf, $storecfg) = @_;
2992
2993     my $nodelist = PVE::Cluster::get_nodelist();
2994     my $nodehash = { map { $_ => {} } @$nodelist };
2995
2996     foreach_drive($conf, sub {
2997         my ($ds, $drive) = @_;
2998
2999         my $volid = $drive->{file};
3000         return if !$volid;
3001
3002         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3003         if ($storeid) {
3004             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3005
3006             if ($scfg->{disable}) {
3007                 foreach my $node (keys %$nodehash) {
3008                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3009                 }
3010             } elsif (my $avail = $scfg->{nodes}) {
3011                 foreach my $node (keys %$nodehash) {
3012                     if (!$avail->{$node}) {
3013                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3014                     }
3015                 }
3016             }
3017         }
3018     });
3019
3020     foreach my $node (values %$nodehash) {
3021         if (my $unavail = $node->{unavailable_storages}) {
3022             $node->{unavailable_storages} = [ sort keys %$unavail ];
3023         }
3024     }
3025
3026     return $nodehash
3027 }
3028
3029 sub check_cmdline {
3030     my ($pidfile, $pid) = @_;
3031
3032     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3033     if (defined($fh)) {
3034         my $line = <$fh>;
3035         $fh->close;
3036         return undef if !$line;
3037         my @param = split(/\0/, $line);
3038
3039         my $cmd = $param[0];
3040         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3041
3042         for (my $i = 0; $i < scalar (@param); $i++) {
3043             my $p = $param[$i];
3044             next if !$p;
3045             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3046                 my $p = $param[$i+1];
3047                 return 1 if $p && ($p eq $pidfile);
3048                 return undef;
3049             }
3050         }
3051     }
3052     return undef;
3053 }
3054
3055 sub check_running {
3056     my ($vmid, $nocheck, $node) = @_;
3057
3058     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3059
3060     die "unable to find configuration file for VM $vmid - no such machine\n"
3061         if !$nocheck && ! -f $filename;
3062
3063     my $pidfile = pidfile_name($vmid);
3064
3065     if (my $fd = IO::File->new("<$pidfile")) {
3066         my $st = stat($fd);
3067         my $line = <$fd>;
3068         close($fd);
3069
3070         my $mtime = $st->mtime;
3071         if ($mtime > time()) {
3072             warn "file '$filename' modified in future\n";
3073         }
3074
3075         if ($line =~ m/^(\d+)$/) {
3076             my $pid = $1;
3077             if (check_cmdline($pidfile, $pid)) {
3078                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3079                     return $pid;
3080                 }
3081             }
3082         }
3083     }
3084
3085     return undef;
3086 }
3087
3088 sub vzlist {
3089
3090     my $vzlist = config_list();
3091
3092     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3093
3094     while (defined(my $de = $fd->read)) {
3095         next if $de !~ m/^(\d+)\.pid$/;
3096         my $vmid = $1;
3097         next if !defined($vzlist->{$vmid});
3098         if (my $pid = check_running($vmid)) {
3099             $vzlist->{$vmid}->{pid} = $pid;
3100         }
3101     }
3102
3103     return $vzlist;
3104 }
3105
3106 sub disksize {
3107     my ($storecfg, $conf) = @_;
3108
3109     my $bootdisk = $conf->{bootdisk};
3110     return undef if !$bootdisk;
3111     return undef if !is_valid_drivename($bootdisk);
3112
3113     return undef if !$conf->{$bootdisk};
3114
3115     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3116     return undef if !defined($drive);
3117
3118     return undef if drive_is_cdrom($drive);
3119
3120     my $volid = $drive->{file};
3121     return undef if !$volid;
3122
3123     return $drive->{size};
3124 }
3125
3126 our $vmstatus_return_properties = {
3127     vmid => get_standard_option('pve-vmid'),
3128     status => {
3129         description => "Qemu process status.",
3130         type => 'string',
3131         enum => ['stopped', 'running'],
3132     },
3133     maxmem => {
3134         description => "Maximum memory in bytes.",
3135         type => 'integer',
3136         optional => 1,
3137         renderer => 'bytes',
3138     },
3139     maxdisk => {
3140         description => "Root disk size in bytes.",
3141         type => 'integer',
3142         optional => 1,
3143         renderer => 'bytes',
3144     },
3145     name => {
3146         description => "VM name.",
3147         type => 'string',
3148         optional => 1,
3149     },
3150     qmpstatus => {
3151         description => "Qemu QMP agent status.",
3152         type => 'string',
3153         optional => 1,
3154     },
3155     pid => {
3156         description => "PID of running qemu process.",
3157         type => 'integer',
3158         optional => 1,
3159     },
3160     uptime => {
3161         description => "Uptime.",
3162         type => 'integer',
3163         optional => 1,
3164         renderer => 'duration',
3165     },
3166     cpus => {
3167         description => "Maximum usable CPUs.",
3168         type => 'number',
3169         optional => 1,
3170     },
3171     lock => {
3172         description => "The current config lock, if any.",
3173         type => 'string',
3174         optional => 1,
3175     }
3176 };
3177
3178 my $last_proc_pid_stat;
3179
3180 # get VM status information
3181 # This must be fast and should not block ($full == false)
3182 # We only query KVM using QMP if $full == true (this can be slow)
3183 sub vmstatus {
3184     my ($opt_vmid, $full) = @_;
3185
3186     my $res = {};
3187
3188     my $storecfg = PVE::Storage::config();
3189
3190     my $list = vzlist();
3191     my $defaults = load_defaults();
3192
3193     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3194
3195     my $cpucount = $cpuinfo->{cpus} || 1;
3196
3197     foreach my $vmid (keys %$list) {
3198         next if $opt_vmid && ($vmid ne $opt_vmid);
3199
3200         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3201         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3202
3203         my $d = { vmid => $vmid };
3204         $d->{pid} = $list->{$vmid}->{pid};
3205
3206         # fixme: better status?
3207         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3208
3209         my $size = disksize($storecfg, $conf);
3210         if (defined($size)) {
3211             $d->{disk} = 0; # no info available
3212             $d->{maxdisk} = $size;
3213         } else {
3214             $d->{disk} = 0;
3215             $d->{maxdisk} = 0;
3216         }
3217
3218         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3219             * ($conf->{cores} || $defaults->{cores});
3220         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3221         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3222
3223         $d->{name} = $conf->{name} || "VM $vmid";
3224         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3225             : $defaults->{memory}*(1024*1024);
3226
3227         if ($conf->{balloon}) {
3228             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3229             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3230                 : $defaults->{shares};
3231         }
3232
3233         $d->{uptime} = 0;
3234         $d->{cpu} = 0;
3235         $d->{mem} = 0;
3236
3237         $d->{netout} = 0;
3238         $d->{netin} = 0;
3239
3240         $d->{diskread} = 0;
3241         $d->{diskwrite} = 0;
3242
3243         $d->{template} = PVE::QemuConfig->is_template($conf);
3244
3245         $d->{serial} = 1 if conf_has_serial($conf);
3246         $d->{lock} = $conf->{lock} if $conf->{lock};
3247
3248         $res->{$vmid} = $d;
3249     }
3250
3251     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3252     foreach my $dev (keys %$netdev) {
3253         next if $dev !~ m/^tap([1-9]\d*)i/;
3254         my $vmid = $1;
3255         my $d = $res->{$vmid};
3256         next if !$d;
3257
3258         $d->{netout} += $netdev->{$dev}->{receive};
3259         $d->{netin} += $netdev->{$dev}->{transmit};
3260
3261         if ($full) {
3262             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3263             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3264         }
3265
3266     }
3267
3268     my $ctime = gettimeofday;
3269
3270     foreach my $vmid (keys %$list) {
3271
3272         my $d = $res->{$vmid};
3273         my $pid = $d->{pid};
3274         next if !$pid;
3275
3276         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3277         next if !$pstat; # not running
3278
3279         my $used = $pstat->{utime} + $pstat->{stime};
3280
3281         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3282
3283         if ($pstat->{vsize}) {
3284             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3285         }
3286
3287         my $old = $last_proc_pid_stat->{$pid};
3288         if (!$old) {
3289             $last_proc_pid_stat->{$pid} = {
3290                 time => $ctime,
3291                 used => $used,
3292                 cpu => 0,
3293             };
3294             next;
3295         }
3296
3297         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3298
3299         if ($dtime > 1000) {
3300             my $dutime = $used -  $old->{used};
3301
3302             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3303             $last_proc_pid_stat->{$pid} = {
3304                 time => $ctime,
3305                 used => $used,
3306                 cpu => $d->{cpu},
3307             };
3308         } else {
3309             $d->{cpu} = $old->{cpu};
3310         }
3311     }
3312
3313     return $res if !$full;
3314
3315     my $qmpclient = PVE::QMPClient->new();
3316
3317     my $ballooncb = sub {
3318         my ($vmid, $resp) = @_;
3319
3320         my $info = $resp->{'return'};
3321         return if !$info->{max_mem};
3322
3323         my $d = $res->{$vmid};
3324
3325         # use memory assigned to VM
3326         $d->{maxmem} = $info->{max_mem};
3327         $d->{balloon} = $info->{actual};
3328
3329         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3330             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3331             $d->{freemem} = $info->{free_mem};
3332         }
3333
3334         $d->{ballooninfo} = $info;
3335     };
3336
3337     my $blockstatscb = sub {
3338         my ($vmid, $resp) = @_;
3339         my $data = $resp->{'return'} || [];
3340         my $totalrdbytes = 0;
3341         my $totalwrbytes = 0;
3342
3343         for my $blockstat (@$data) {
3344             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3345             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3346
3347             $blockstat->{device} =~ s/drive-//;
3348             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3349         }
3350         $res->{$vmid}->{diskread} = $totalrdbytes;
3351         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3352     };
3353
3354     my $statuscb = sub {
3355         my ($vmid, $resp) = @_;
3356
3357         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3358         # this fails if ballon driver is not loaded, so this must be
3359         # the last commnand (following command are aborted if this fails).
3360         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3361
3362         my $status = 'unknown';
3363         if (!defined($status = $resp->{'return'}->{status})) {
3364             warn "unable to get VM status\n";
3365             return;
3366         }
3367
3368         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3369     };
3370
3371     foreach my $vmid (keys %$list) {
3372         next if $opt_vmid && ($vmid ne $opt_vmid);
3373         next if !$res->{$vmid}->{pid}; # not running
3374         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3375     }
3376
3377     $qmpclient->queue_execute(undef, 2);
3378
3379     foreach my $vmid (keys %$list) {
3380         next if $opt_vmid && ($vmid ne $opt_vmid);
3381         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3382     }
3383
3384     return $res;
3385 }
3386
3387 sub foreach_drive {
3388     my ($conf, $func, @param) = @_;
3389
3390     foreach my $ds (valid_drive_names()) {
3391         next if !defined($conf->{$ds});
3392
3393         my $drive = parse_drive($ds, $conf->{$ds});
3394         next if !$drive;
3395
3396         &$func($ds, $drive, @param);
3397     }
3398 }
3399
3400 sub foreach_volid {
3401     my ($conf, $func, @param) = @_;
3402
3403     my $volhash = {};
3404
3405     my $test_volid = sub {
3406         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3407
3408         return if !$volid;
3409
3410         $volhash->{$volid}->{cdrom} //= 1;
3411         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3412
3413         $volhash->{$volid}->{replicate} //= 0;
3414         $volhash->{$volid}->{replicate} = 1 if $replicate;
3415
3416         $volhash->{$volid}->{shared} //= 0;
3417         $volhash->{$volid}->{shared} = 1 if $shared;
3418
3419         $volhash->{$volid}->{referenced_in_config} //= 0;
3420         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3421
3422         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3423             if defined($snapname);
3424         $volhash->{$volid}->{size} = $size if $size;
3425     };
3426
3427     foreach_drive($conf, sub {
3428         my ($ds, $drive) = @_;
3429         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3430     });
3431
3432     foreach my $snapname (keys %{$conf->{snapshots}}) {
3433         my $snap = $conf->{snapshots}->{$snapname};
3434         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3435         foreach_drive($snap, sub {
3436             my ($ds, $drive) = @_;
3437             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3438         });
3439     }
3440
3441     foreach my $volid (keys %$volhash) {
3442         &$func($volid, $volhash->{$volid}, @param);
3443     }
3444 }
3445
3446 sub conf_has_serial {
3447     my ($conf) = @_;
3448
3449     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3450         if ($conf->{"serial$i"}) {
3451             return 1;
3452         }
3453     }
3454
3455     return 0;
3456 }
3457
3458 sub conf_has_audio {
3459     my ($conf, $id) = @_;
3460
3461     $id //= 0;
3462     my $audio = $conf->{"audio$id"};
3463     return undef if !defined($audio);
3464
3465     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3466     my $audiodriver = $audioproperties->{driver} // 'spice';
3467
3468     return {
3469         dev => $audioproperties->{device},
3470         dev_id => "audiodev$id",
3471         backend => $audiodriver,
3472         backend_id => "$audiodriver-backend${id}",
3473     };
3474 }
3475
3476 sub vga_conf_has_spice {
3477     my ($vga) = @_;
3478
3479     my $vgaconf = parse_vga($vga);
3480     my $vgatype = $vgaconf->{type};
3481     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3482
3483     return $1 || 1;
3484 }
3485
3486 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3487 sub get_host_arch() {
3488     $host_arch = (POSIX::uname())[4] if !$host_arch;
3489     return $host_arch;
3490 }
3491
3492 sub is_native($) {
3493     my ($arch) = @_;
3494     return get_host_arch() eq $arch;
3495 }
3496
3497 my $default_machines = {
3498     x86_64 => 'pc',
3499     aarch64 => 'virt',
3500 };
3501
3502 sub get_basic_machine_info {
3503     my ($conf, $forcemachine) = @_;
3504
3505     my $arch = $conf->{arch} // get_host_arch();
3506     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3507     return ($arch, $machine);
3508 }
3509
3510 sub get_ovmf_files($) {
3511     my ($arch) = @_;
3512
3513     my $ovmf = $OVMF->{$arch}
3514         or die "no OVMF images known for architecture '$arch'\n";
3515
3516     return @$ovmf;
3517 }
3518
3519 my $Arch2Qemu = {
3520     aarch64 => '/usr/bin/qemu-system-aarch64',
3521     x86_64 => '/usr/bin/qemu-system-x86_64',
3522 };
3523 sub get_command_for_arch($) {
3524     my ($arch) = @_;
3525     return '/usr/bin/kvm' if is_native($arch);
3526
3527     my $cmd = $Arch2Qemu->{$arch}
3528         or die "don't know how to emulate architecture '$arch'\n";
3529     return $cmd;
3530 }
3531
3532 sub get_cpu_options {
3533     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3534
3535     my $cpuFlags = [];
3536     my $ostype = $conf->{ostype};
3537
3538     my $cpu = $kvm ? "kvm64" : "qemu64";
3539     if ($arch eq 'aarch64') {
3540         $cpu = 'cortex-a57';
3541     }
3542     my $hv_vendor_id;
3543     if (my $cputype = $conf->{cpu}) {
3544         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3545             or die "Cannot parse cpu description: $cputype\n";
3546         $cpu = $cpuconf->{cputype};
3547         $kvm_off = 1 if $cpuconf->{hidden};
3548         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3549
3550         if (defined(my $flags = $cpuconf->{flags})) {
3551             push @$cpuFlags, split(";", $flags);
3552         }
3553     }
3554
3555     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3556
3557     push @$cpuFlags , '-x2apic'
3558         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3559
3560     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3561
3562     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3563
3564     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3565
3566         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3567         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3568     }
3569
3570     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3571
3572     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3573
3574     push @$cpuFlags, 'kvm=off' if $kvm_off;
3575
3576     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3577         push @$cpuFlags, "vendor=${cpu_vendor}"
3578             if $cpu_vendor ne 'default';
3579     } elsif ($arch ne 'aarch64') {
3580         die "internal error"; # should not happen
3581     }
3582
3583     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3584
3585     return ('-cpu', $cpu);
3586 }
3587
3588 sub config_to_command {
3589     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3590
3591     my $cmd = [];
3592     my $globalFlags = [];
3593     my $machineFlags = [];
3594     my $rtcFlags = [];
3595     my $devices = [];
3596     my $pciaddr = '';
3597     my $bridges = {};
3598     my $vernum = 0; # unknown
3599     my $ostype = $conf->{ostype};
3600     my $winversion = windows_version($ostype);
3601     my $kvm = $conf->{kvm};
3602
3603     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3604     my $kvm_binary = get_command_for_arch($arch);
3605     my $kvmver = kvm_user_version($kvm_binary);
3606     $kvm //= 1 if is_native($arch);
3607
3608     if ($kvm) {
3609         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3610             if !defined kvm_version();
3611     }
3612
3613     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3614         $vernum = $1*1000000+$2*1000;
3615     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3616         $vernum = $1*1000000+$2*1000+$3;
3617     }
3618
3619     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3620
3621     my $have_ovz = -f '/proc/vz/vestat';
3622
3623     my $q35 = machine_type_is_q35($conf);
3624     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3625     my $use_old_bios_files = undef;
3626     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3627
3628     my $cpuunits = defined($conf->{cpuunits}) ?
3629             $conf->{cpuunits} : $defaults->{cpuunits};
3630
3631     push @$cmd, $kvm_binary;
3632
3633     push @$cmd, '-id', $vmid;
3634
3635     my $vmname = $conf->{name} || "vm$vmid";
3636
3637     push @$cmd, '-name', $vmname;
3638
3639     my $use_virtio = 0;
3640
3641     my $qmpsocket = qmp_socket($vmid);
3642     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3643     push @$cmd, '-mon', "chardev=qmp,mode=control";
3644
3645     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3646         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3647         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3648     }
3649
3650     push @$cmd, '-pidfile' , pidfile_name($vmid);
3651
3652     push @$cmd, '-daemonize';
3653
3654     if ($conf->{smbios1}) {
3655         my $smbios_conf = parse_smbios1($conf->{smbios1});
3656         if ($smbios_conf->{base64}) {
3657             # Do not pass base64 flag to qemu
3658             delete $smbios_conf->{base64};
3659             my $smbios_string = "";
3660             foreach my $key (keys %$smbios_conf) {
3661                 my $value;
3662                 if ($key eq "uuid") {
3663                     $value = $smbios_conf->{uuid}
3664                 } else {
3665                     $value = decode_base64($smbios_conf->{$key});
3666                 }
3667                 # qemu accepts any binary data, only commas need escaping by double comma
3668                 $value =~ s/,/,,/g;
3669                 $smbios_string .= "," . $key . "=" . $value if $value;
3670             }
3671             push @$cmd, '-smbios', "type=1" . $smbios_string;
3672         } else {
3673             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3674         }
3675     }
3676
3677     if ($conf->{vmgenid}) {
3678         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3679     }
3680
3681     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3682     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3683         die "uefi base image not found\n" if ! -f $ovmf_code;
3684
3685         my $path;
3686         my $format;
3687         if (my $efidisk = $conf->{efidisk0}) {
3688             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3689             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3690             $format = $d->{format};
3691             if ($storeid) {
3692                 $path = PVE::Storage::path($storecfg, $d->{file});
3693                 if (!defined($format)) {
3694                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3695                     $format = qemu_img_format($scfg, $volname);
3696                 }
3697             } else {
3698                 $path = $d->{file};
3699                 die "efidisk format must be specified\n"
3700                     if !defined($format);
3701             }
3702         } else {
3703             warn "no efidisk configured! Using temporary efivars disk.\n";
3704             $path = "/tmp/$vmid-ovmf.fd";
3705             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3706             $format = 'raw';
3707         }
3708
3709         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3710         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3711     }
3712
3713     # load q35 config
3714     if ($q35) {
3715         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3716         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3717             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3718         } else {
3719             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3720         }
3721     }
3722
3723     # add usb controllers
3724     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3725     push @$devices, @usbcontrollers if @usbcontrollers;
3726     my $vga = parse_vga($conf->{vga});
3727
3728     my $qxlnum = vga_conf_has_spice($conf->{vga});
3729     $vga->{type} = 'qxl' if $qxlnum;
3730
3731     if (!$vga->{type}) {
3732         if ($arch eq 'aarch64') {
3733             $vga->{type} = 'virtio';
3734         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3735             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3736         } else {
3737             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3738         }
3739     }
3740
3741     # enable absolute mouse coordinates (needed by vnc)
3742     my $tablet;
3743     if (defined($conf->{tablet})) {
3744         $tablet = $conf->{tablet};
3745     } else {
3746         $tablet = $defaults->{tablet};
3747         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3748         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3749     }
3750
3751     if ($tablet) {
3752         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3753         my $kbd = print_keyboarddevice_full($conf, $arch);
3754         push @$devices, '-device', $kbd if defined($kbd);
3755     }
3756
3757     my $kvm_off = 0;
3758     my $gpu_passthrough;
3759
3760     # host pci devices
3761     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3762         my $d = parse_hostpci($conf->{"hostpci$i"});
3763         next if !$d;
3764
3765         my $pcie = $d->{pcie};
3766         if ($pcie) {
3767             die "q35 machine model is not enabled" if !$q35;
3768             # win7 wants to have the pcie devices directly on the pcie bus
3769             # instead of in the root port
3770             if ($winversion == 7) {
3771                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3772             } else {
3773                 $pciaddr = print_pcie_addr("hostpci$i");
3774             }
3775         } else {
3776             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3777         }
3778
3779         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3780         my $romfile = $d->{romfile};
3781
3782         my $xvga = '';
3783         if ($d->{'x-vga'}) {
3784             $xvga = ',x-vga=on';
3785             $kvm_off = 1;
3786             $vga->{type} = 'none' if !defined($conf->{vga});
3787             $gpu_passthrough = 1;
3788
3789             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3790                 $xvga = "";
3791             }
3792         }
3793         my $pcidevices = $d->{pciid};
3794         my $multifunction = 1 if @$pcidevices > 1;
3795         my $sysfspath;
3796         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3797             my $id = $pcidevices->[0]->{id};
3798             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3799             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3800         } elsif ($d->{mdev}) {
3801             warn "ignoring mediated device with multifunction device\n";
3802         }
3803
3804         my $j=0;
3805         foreach my $pcidevice (@$pcidevices) {
3806
3807             my $id = "hostpci$i";
3808             $id .= ".$j" if $multifunction;
3809             my $addr = $pciaddr;
3810             $addr .= ".$j" if $multifunction;
3811             my $devicestr = "vfio-pci";
3812             if ($sysfspath) {
3813                 $devicestr .= ",sysfsdev=$sysfspath";
3814             } else {
3815                 $devicestr .= ",host=$pcidevice->{id}";
3816             }
3817             $devicestr .= ",id=$id$addr";
3818
3819             if($j == 0){
3820                 $devicestr .= "$rombar$xvga";
3821                 $devicestr .= ",multifunction=on" if $multifunction;
3822                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3823             }
3824
3825             push @$devices, '-device', $devicestr;
3826             $j++;
3827         }
3828     }
3829
3830     # usb devices
3831     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3832     push @$devices, @usbdevices if @usbdevices;
3833     # serial devices
3834     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3835         if (my $path = $conf->{"serial$i"}) {
3836             if ($path eq 'socket') {
3837                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3838                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3839                 # On aarch64, serial0 is the UART device. Qemu only allows
3840                 # connecting UART devices via the '-serial' command line, as
3841                 # the device has a fixed slot on the hardware...
3842                 if ($arch eq 'aarch64' && $i == 0) {
3843                     push @$devices, '-serial', "chardev:serial$i";
3844                 } else {
3845                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3846                 }
3847             } else {
3848                 die "no such serial device\n" if ! -c $path;
3849                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3850                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3851             }
3852         }
3853     }
3854
3855     # parallel devices
3856     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3857         if (my $path = $conf->{"parallel$i"}) {
3858             die "no such parallel device\n" if ! -c $path;
3859             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3860             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3861             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3862         }
3863     }
3864
3865     if (my $audio = conf_has_audio($conf)) {
3866
3867         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3868
3869         my $id = $audio->{dev_id};
3870         if ($audio->{dev} eq 'AC97') {
3871             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3872         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3873             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3874             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3875             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3876         } else {
3877             die "unkown audio device '$audio->{dev}', implement me!";
3878         }
3879
3880         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3881     }
3882
3883     my $sockets = 1;
3884     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3885     $sockets = $conf->{sockets} if  $conf->{sockets};
3886
3887     my $cores = $conf->{cores} || 1;
3888
3889     my $maxcpus = $sockets * $cores;
3890
3891     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3892
3893     my $allowed_vcpus = $cpuinfo->{cpus};
3894
3895     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3896         if ($allowed_vcpus < $maxcpus);
3897
3898     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3899
3900         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3901         for (my $i = 2; $i <= $vcpus; $i++)  {
3902             my $cpustr = print_cpu_device($conf,$i);
3903             push @$cmd, '-device', $cpustr;
3904         }
3905
3906     } else {
3907
3908         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3909     }
3910     push @$cmd, '-nodefaults';
3911
3912     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3913
3914     my $bootindex_hash = {};
3915     my $i = 1;
3916     foreach my $o (split(//, $bootorder)) {
3917         $bootindex_hash->{$o} = $i*100;
3918         $i++;
3919     }
3920
3921     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3922
3923     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3924
3925     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3926
3927     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3928         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3929         my $socket = vnc_socket($vmid);
3930         push @$cmd,  '-vnc', "unix:$socket,password";
3931     } else {
3932         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3933         push @$cmd, '-nographic';
3934     }
3935
3936     # time drift fix
3937     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3938
3939     my $useLocaltime = $conf->{localtime};
3940
3941     if ($winversion >= 5) { # windows
3942         $useLocaltime = 1 if !defined($conf->{localtime});
3943
3944         # use time drift fix when acpi is enabled
3945         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3946             $tdf = 1 if !defined($conf->{tdf});
3947         }
3948     }
3949
3950     if ($winversion >= 6) {
3951         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3952         push @$cmd, '-no-hpet';
3953     }
3954
3955     push @$rtcFlags, 'driftfix=slew' if $tdf;
3956
3957     if (!$kvm) {
3958         push @$machineFlags, 'accel=tcg';
3959     }
3960
3961     if ($machine_type) {
3962         push @$machineFlags, "type=${machine_type}";
3963     }
3964
3965     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3966         push @$rtcFlags, "base=$conf->{startdate}";
3967     } elsif ($useLocaltime) {
3968         push @$rtcFlags, 'base=localtime';
3969     }
3970
3971     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3972
3973     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3974
3975     push @$cmd, '-S' if $conf->{freeze};
3976
3977     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3978
3979     if (parse_guest_agent($conf)->{enabled}) {
3980         my $qgasocket = qmp_socket($vmid, 1);
3981         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3982         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3983         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3984         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3985     }
3986
3987     my $spice_port;
3988
3989     if ($qxlnum) {
3990         if ($qxlnum > 1) {
3991             if ($winversion){
3992                 for(my $i = 1; $i < $qxlnum; $i++){
3993                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3994                 }
3995             } else {
3996                 # assume other OS works like Linux
3997                 my ($ram, $vram) = ("134217728", "67108864");
3998                 if ($vga->{memory}) {
3999                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4000                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4001                 }
4002                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4003                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4004             }
4005         }
4006
4007         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4008
4009         my $nodename = PVE::INotify::nodename();
4010         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4011         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4012         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4013         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4014         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4015
4016         my $spice = {};
4017         my $spice_enhancements = $conf->{spice_enhancements} ? PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements}) : {};
4018         $spice->{videostreaming} = $spice_enhancements->{videostreaming} ? ",streaming-video=$spice_enhancements->{videostreaming}" : '';
4019         $spice->{foldersharing} = $spice_enhancements->{foldersharing};
4020
4021         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on$spice->{videostreaming}";
4022
4023         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4024         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4025         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4026
4027         if ($spice_enhancements->{foldersharing}) {
4028             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4029             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4030         }
4031     }
4032
4033     # enable balloon by default, unless explicitly disabled
4034     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4035         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4036         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4037     }
4038
4039     if ($conf->{watchdog}) {
4040         my $wdopts = parse_watchdog($conf->{watchdog});
4041         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4042         my $watchdog = $wdopts->{model} || 'i6300esb';
4043         push @$devices, '-device', "$watchdog$pciaddr";
4044         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4045     }
4046
4047     my $vollist = [];
4048     my $scsicontroller = {};
4049     my $ahcicontroller = {};
4050     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4051
4052     # Add iscsi initiator name if available
4053     if (my $initiator = get_initiator_name()) {
4054         push @$devices, '-iscsi', "initiator-name=$initiator";
4055     }
4056
4057     foreach_drive($conf, sub {
4058         my ($ds, $drive) = @_;
4059
4060         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4061             push @$vollist, $drive->{file};
4062         }
4063
4064         # ignore efidisk here, already added in bios/fw handling code above
4065         return if $drive->{interface} eq 'efidisk';
4066
4067         $use_virtio = 1 if $ds =~ m/^virtio/;
4068
4069         if (drive_is_cdrom ($drive)) {
4070             if ($bootindex_hash->{d}) {
4071                 $drive->{bootindex} = $bootindex_hash->{d};
4072                 $bootindex_hash->{d} += 1;
4073             }
4074         } else {
4075             if ($bootindex_hash->{c}) {
4076                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4077                 $bootindex_hash->{c} += 1;
4078             }
4079         }
4080
4081         if($drive->{interface} eq 'virtio'){
4082            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4083         }
4084
4085         if ($drive->{interface} eq 'scsi') {
4086
4087             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4088
4089             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4090             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4091
4092             my $iothread = '';
4093             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4094                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4095                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4096             } elsif ($drive->{iothread}) {
4097                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4098             }
4099
4100             my $queues = '';
4101             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4102                 $queues = ",num_queues=$drive->{queues}";
4103             }
4104
4105             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4106             $scsicontroller->{$controller}=1;
4107         }
4108
4109         if ($drive->{interface} eq 'sata') {
4110            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4111            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4112            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4113            $ahcicontroller->{$controller}=1;
4114         }
4115
4116         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4117         push @$devices, '-drive',$drive_cmd;
4118         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4119     });
4120
4121     for (my $i = 0; $i < $MAX_NETS; $i++) {
4122          next if !$conf->{"net$i"};
4123          my $d = parse_net($conf->{"net$i"});
4124          next if !$d;
4125
4126          $use_virtio = 1 if $d->{model} eq 'virtio';
4127
4128          if ($bootindex_hash->{n}) {
4129             $d->{bootindex} = $bootindex_hash->{n};
4130             $bootindex_hash->{n} += 1;
4131          }
4132
4133          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4134          push @$devices, '-netdev', $netdevfull;
4135
4136          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4137          push @$devices, '-device', $netdevicefull;
4138     }
4139
4140     if ($conf->{ivshmem}) {
4141         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4142
4143         my $bus;
4144         if ($q35) {
4145             $bus = print_pcie_addr("ivshmem");
4146         } else {
4147             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4148         }
4149
4150         my $ivshmem_name = $ivshmem->{name} // $vmid;
4151         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4152
4153         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4154         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4155     }
4156
4157     if (!$q35) {
4158         # add pci bridges
4159         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4160            $bridges->{1} = 1;
4161            $bridges->{2} = 1;
4162         }
4163
4164         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4165
4166         while (my ($k, $v) = each %$bridges) {
4167             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4168             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4169         }
4170     }
4171
4172     push @$cmd, @$devices;
4173     push @$cmd, '-rtc', join(',', @$rtcFlags)
4174         if scalar(@$rtcFlags);
4175     push @$cmd, '-machine', join(',', @$machineFlags)
4176         if scalar(@$machineFlags);
4177     push @$cmd, '-global', join(',', @$globalFlags)
4178         if scalar(@$globalFlags);
4179
4180     if (my $vmstate = $conf->{vmstate}) {
4181         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4182         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4183         push @$cmd, '-loadstate', $statepath;
4184     }
4185
4186     # add custom args
4187     if ($conf->{args}) {
4188         my $aa = PVE::Tools::split_args($conf->{args});
4189         push @$cmd, @$aa;
4190     }
4191
4192     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4193 }
4194
4195 sub vnc_socket {
4196     my ($vmid) = @_;
4197     return "${var_run_tmpdir}/$vmid.vnc";
4198 }
4199
4200 sub spice_port {
4201     my ($vmid) = @_;
4202
4203     my $res = vm_mon_cmd($vmid, 'query-spice');
4204
4205     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4206 }
4207
4208 sub qmp_socket {
4209     my ($vmid, $qga, $name) = @_;
4210     my $sockettype = $qga ? 'qga' : 'qmp';
4211     my $ext = $name ? '-'.$name : '';
4212     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4213 }
4214
4215 sub pidfile_name {
4216     my ($vmid) = @_;
4217     return "${var_run_tmpdir}/$vmid.pid";
4218 }
4219
4220 sub vm_devices_list {
4221     my ($vmid) = @_;
4222
4223     my $res = vm_mon_cmd($vmid, 'query-pci');
4224     my $devices_to_check = [];
4225     my $devices = {};
4226     foreach my $pcibus (@$res) {
4227         push @$devices_to_check, @{$pcibus->{devices}},
4228     }
4229
4230     while (@$devices_to_check) {
4231         my $to_check = [];
4232         for my $d (@$devices_to_check) {
4233             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4234             next if !$d->{'pci_bridge'};
4235
4236             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4237             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4238         }
4239         $devices_to_check = $to_check;
4240     }
4241
4242     my $resblock = vm_mon_cmd($vmid, 'query-block');
4243     foreach my $block (@$resblock) {
4244         if($block->{device} =~ m/^drive-(\S+)/){
4245                 $devices->{$1} = 1;
4246         }
4247     }
4248
4249     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4250     foreach my $mice (@$resmice) {
4251         if ($mice->{name} eq 'QEMU HID Tablet') {
4252             $devices->{tablet} = 1;
4253             last;
4254         }
4255     }
4256
4257     # for usb devices there is no query-usb
4258     # but we can iterate over the entries in
4259     # qom-list path=/machine/peripheral
4260     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4261     foreach my $per (@$resperipheral) {
4262         if ($per->{name} =~ m/^usb\d+$/) {
4263             $devices->{$per->{name}} = 1;
4264         }
4265     }
4266
4267     return $devices;
4268 }
4269
4270 sub vm_deviceplug {
4271     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4272
4273     my $q35 = machine_type_is_q35($conf);
4274
4275     my $devices_list = vm_devices_list($vmid);
4276     return 1 if defined($devices_list->{$deviceid});
4277
4278     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4279
4280     if ($deviceid eq 'tablet') {
4281
4282         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4283
4284     } elsif ($deviceid eq 'keyboard') {
4285
4286         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4287
4288     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4289
4290         die "usb hotplug currently not reliable\n";
4291         # since we can't reliably hot unplug all added usb devices
4292         # and usb passthrough disables live migration
4293         # we disable usb hotplugging for now
4294         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4295
4296     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4297
4298         qemu_iothread_add($vmid, $deviceid, $device);
4299
4300         qemu_driveadd($storecfg, $vmid, $device);
4301         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4302
4303         qemu_deviceadd($vmid, $devicefull);
4304         eval { qemu_deviceaddverify($vmid, $deviceid); };
4305         if (my $err = $@) {
4306             eval { qemu_drivedel($vmid, $deviceid); };
4307             warn $@ if $@;
4308             die $err;
4309         }
4310
4311     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4312
4313
4314         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4315         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4316         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4317
4318         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4319
4320         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4321             qemu_iothread_add($vmid, $deviceid, $device);
4322             $devicefull .= ",iothread=iothread-$deviceid";
4323         }
4324
4325         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4326             $devicefull .= ",num_queues=$device->{queues}";
4327         }
4328
4329         qemu_deviceadd($vmid, $devicefull);
4330         qemu_deviceaddverify($vmid, $deviceid);
4331
4332     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4333
4334         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4335         qemu_driveadd($storecfg, $vmid, $device);
4336
4337         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4338         eval { qemu_deviceadd($vmid, $devicefull); };
4339         if (my $err = $@) {
4340             eval { qemu_drivedel($vmid, $deviceid); };
4341             warn $@ if $@;
4342             die $err;
4343         }
4344
4345     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4346
4347         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4348
4349         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4350         my $use_old_bios_files = undef;
4351         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4352
4353         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4354         qemu_deviceadd($vmid, $netdevicefull);
4355         eval {
4356             qemu_deviceaddverify($vmid, $deviceid);
4357             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4358         };
4359         if (my $err = $@) {
4360             eval { qemu_netdevdel($vmid, $deviceid); };
4361             warn $@ if $@;
4362             die $err;
4363         }
4364
4365     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4366
4367         my $bridgeid = $2;
4368         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4369         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4370
4371         qemu_deviceadd($vmid, $devicefull);
4372         qemu_deviceaddverify($vmid, $deviceid);
4373
4374     } else {
4375         die "can't hotplug device '$deviceid'\n";
4376     }
4377
4378     return 1;
4379 }
4380
4381 # fixme: this should raise exceptions on error!
4382 sub vm_deviceunplug {
4383     my ($vmid, $conf, $deviceid) = @_;
4384
4385     my $devices_list = vm_devices_list($vmid);
4386     return 1 if !defined($devices_list->{$deviceid});
4387
4388     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4389
4390     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4391
4392         qemu_devicedel($vmid, $deviceid);
4393
4394     } elsif ($deviceid =~ m/^usb\d+$/) {
4395
4396         die "usb hotplug currently not reliable\n";
4397         # when unplugging usb devices this way,
4398         # there may be remaining usb controllers/hubs
4399         # so we disable it for now
4400         qemu_devicedel($vmid, $deviceid);
4401         qemu_devicedelverify($vmid, $deviceid);
4402
4403     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4404
4405         qemu_devicedel($vmid, $deviceid);
4406         qemu_devicedelverify($vmid, $deviceid);
4407         qemu_drivedel($vmid, $deviceid);
4408         qemu_iothread_del($conf, $vmid, $deviceid);
4409
4410     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4411
4412         qemu_devicedel($vmid, $deviceid);
4413         qemu_devicedelverify($vmid, $deviceid);
4414         qemu_iothread_del($conf, $vmid, $deviceid);
4415
4416     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4417
4418         qemu_devicedel($vmid, $deviceid);
4419         qemu_drivedel($vmid, $deviceid);
4420         qemu_deletescsihw($conf, $vmid, $deviceid);
4421
4422     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4423
4424         qemu_devicedel($vmid, $deviceid);
4425         qemu_devicedelverify($vmid, $deviceid);
4426         qemu_netdevdel($vmid, $deviceid);
4427
4428     } else {
4429         die "can't unplug device '$deviceid'\n";
4430     }
4431
4432     return 1;
4433 }
4434
4435 sub qemu_deviceadd {
4436     my ($vmid, $devicefull) = @_;
4437
4438     $devicefull = "driver=".$devicefull;
4439     my %options =  split(/[=,]/, $devicefull);
4440
4441     vm_mon_cmd($vmid, "device_add" , %options);
4442 }
4443
4444 sub qemu_devicedel {
4445     my ($vmid, $deviceid) = @_;
4446
4447     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4448 }
4449
4450 sub qemu_iothread_add {
4451     my($vmid, $deviceid, $device) = @_;
4452
4453     if ($device->{iothread}) {
4454         my $iothreads = vm_iothreads_list($vmid);
4455         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4456     }
4457 }
4458
4459 sub qemu_iothread_del {
4460     my($conf, $vmid, $deviceid) = @_;
4461
4462     my $confid = $deviceid;
4463     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4464         $confid = 'scsi' . $1;
4465     }
4466     my $device = parse_drive($confid, $conf->{$confid});
4467     if ($device->{iothread}) {
4468         my $iothreads = vm_iothreads_list($vmid);
4469         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4470     }
4471 }
4472
4473 sub qemu_objectadd {
4474     my($vmid, $objectid, $qomtype) = @_;
4475
4476     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4477
4478     return 1;
4479 }
4480
4481 sub qemu_objectdel {
4482     my($vmid, $objectid) = @_;
4483
4484     vm_mon_cmd($vmid, "object-del", id => $objectid);
4485
4486     return 1;
4487 }
4488
4489 sub qemu_driveadd {
4490     my ($storecfg, $vmid, $device) = @_;
4491
4492     my $drive = print_drive_full($storecfg, $vmid, $device);
4493     $drive =~ s/\\/\\\\/g;
4494     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4495
4496     # If the command succeeds qemu prints: "OK"
4497     return 1 if $ret =~ m/OK/s;
4498
4499     die "adding drive failed: $ret\n";
4500 }
4501
4502 sub qemu_drivedel {
4503     my($vmid, $deviceid) = @_;
4504
4505     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4506     $ret =~ s/^\s+//;
4507
4508     return 1 if $ret eq "";
4509
4510     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4511     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4512
4513     die "deleting drive $deviceid failed : $ret\n";
4514 }
4515
4516 sub qemu_deviceaddverify {
4517     my ($vmid, $deviceid) = @_;
4518
4519     for (my $i = 0; $i <= 5; $i++) {
4520          my $devices_list = vm_devices_list($vmid);
4521          return 1 if defined($devices_list->{$deviceid});
4522          sleep 1;
4523     }
4524
4525     die "error on hotplug device '$deviceid'\n";
4526 }
4527
4528
4529 sub qemu_devicedelverify {
4530     my ($vmid, $deviceid) = @_;
4531
4532     # need to verify that the device is correctly removed as device_del
4533     # is async and empty return is not reliable
4534
4535     for (my $i = 0; $i <= 5; $i++) {
4536          my $devices_list = vm_devices_list($vmid);
4537          return 1 if !defined($devices_list->{$deviceid});
4538          sleep 1;
4539     }
4540
4541     die "error on hot-unplugging device '$deviceid'\n";
4542 }
4543
4544 sub qemu_findorcreatescsihw {
4545     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4546
4547     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4548
4549     my $scsihwid="$controller_prefix$controller";
4550     my $devices_list = vm_devices_list($vmid);
4551
4552     if(!defined($devices_list->{$scsihwid})) {
4553         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4554     }
4555
4556     return 1;
4557 }
4558
4559 sub qemu_deletescsihw {
4560     my ($conf, $vmid, $opt) = @_;
4561
4562     my $device = parse_drive($opt, $conf->{$opt});
4563
4564     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4565         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4566         return 1;
4567     }
4568
4569     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4570
4571     my $devices_list = vm_devices_list($vmid);
4572     foreach my $opt (keys %{$devices_list}) {
4573         if (PVE::QemuServer::is_valid_drivename($opt)) {
4574             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4575             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4576                 return 1;
4577             }
4578         }
4579     }
4580
4581     my $scsihwid="scsihw$controller";
4582
4583     vm_deviceunplug($vmid, $conf, $scsihwid);
4584
4585     return 1;
4586 }
4587
4588 sub qemu_add_pci_bridge {
4589     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4590
4591     my $bridges = {};
4592
4593     my $bridgeid;
4594
4595     print_pci_addr($device, $bridges, $arch, $machine_type);
4596
4597     while (my ($k, $v) = each %$bridges) {
4598         $bridgeid = $k;
4599     }
4600     return 1 if !defined($bridgeid) || $bridgeid < 1;
4601
4602     my $bridge = "pci.$bridgeid";
4603     my $devices_list = vm_devices_list($vmid);
4604
4605     if (!defined($devices_list->{$bridge})) {
4606         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4607     }
4608
4609     return 1;
4610 }
4611
4612 sub qemu_set_link_status {
4613     my ($vmid, $device, $up) = @_;
4614
4615     vm_mon_cmd($vmid, "set_link", name => $device,
4616                up => $up ? JSON::true : JSON::false);
4617 }
4618
4619 sub qemu_netdevadd {
4620     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4621
4622     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4623     my %options =  split(/[=,]/, $netdev);
4624
4625     vm_mon_cmd($vmid, "netdev_add",  %options);
4626     return 1;
4627 }
4628
4629 sub qemu_netdevdel {
4630     my ($vmid, $deviceid) = @_;
4631
4632     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4633 }
4634
4635 sub qemu_usb_hotplug {
4636     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4637
4638     return if !$device;
4639
4640     # remove the old one first
4641     vm_deviceunplug($vmid, $conf, $deviceid);
4642
4643     # check if xhci controller is necessary and available
4644     if ($device->{usb3}) {
4645
4646         my $devicelist = vm_devices_list($vmid);
4647
4648         if (!$devicelist->{xhci}) {
4649             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4650             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4651         }
4652     }
4653     my $d = parse_usb_device($device->{host});
4654     $d->{usb3} = $device->{usb3};
4655
4656     # add the new one
4657     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4658 }
4659
4660 sub qemu_cpu_hotplug {
4661     my ($vmid, $conf, $vcpus) = @_;
4662
4663     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4664
4665     my $sockets = 1;
4666     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4667     $sockets = $conf->{sockets} if  $conf->{sockets};
4668     my $cores = $conf->{cores} || 1;
4669     my $maxcpus = $sockets * $cores;
4670
4671     $vcpus = $maxcpus if !$vcpus;
4672
4673     die "you can't add more vcpus than maxcpus\n"
4674         if $vcpus > $maxcpus;
4675
4676     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4677
4678     if ($vcpus < $currentvcpus) {
4679
4680         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4681
4682             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4683                 qemu_devicedel($vmid, "cpu$i");
4684                 my $retry = 0;
4685                 my $currentrunningvcpus = undef;
4686                 while (1) {
4687                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4688                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4689                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4690                     $retry++;
4691                     sleep 1;
4692                 }
4693                 #update conf after each succesfull cpu unplug
4694                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4695                 PVE::QemuConfig->write_config($vmid, $conf);
4696             }
4697         } else {
4698             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4699         }
4700
4701         return;
4702     }
4703
4704     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4705     die "vcpus in running vm does not match its configuration\n"
4706         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4707
4708     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4709
4710         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4711             my $cpustr = print_cpu_device($conf, $i);
4712             qemu_deviceadd($vmid, $cpustr);
4713
4714             my $retry = 0;
4715             my $currentrunningvcpus = undef;
4716             while (1) {
4717                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4718                 last if scalar(@{$currentrunningvcpus}) == $i;
4719                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4720                 sleep 1;
4721                 $retry++;
4722             }
4723             #update conf after each succesfull cpu hotplug
4724             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4725             PVE::QemuConfig->write_config($vmid, $conf);
4726         }
4727     } else {
4728
4729         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4730             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4731         }
4732     }
4733 }
4734
4735 sub qemu_block_set_io_throttle {
4736     my ($vmid, $deviceid,
4737         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4738         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4739         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4740         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4741
4742     return if !check_running($vmid) ;
4743
4744     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4745         bps => int($bps),
4746         bps_rd => int($bps_rd),
4747         bps_wr => int($bps_wr),
4748         iops => int($iops),
4749         iops_rd => int($iops_rd),
4750         iops_wr => int($iops_wr),
4751         bps_max => int($bps_max),
4752         bps_rd_max => int($bps_rd_max),
4753         bps_wr_max => int($bps_wr_max),
4754         iops_max => int($iops_max),
4755         iops_rd_max => int($iops_rd_max),
4756         iops_wr_max => int($iops_wr_max),
4757         bps_max_length => int($bps_max_length),
4758         bps_rd_max_length => int($bps_rd_max_length),
4759         bps_wr_max_length => int($bps_wr_max_length),
4760         iops_max_length => int($iops_max_length),
4761         iops_rd_max_length => int($iops_rd_max_length),
4762         iops_wr_max_length => int($iops_wr_max_length),
4763     );
4764
4765 }
4766
4767 # old code, only used to shutdown old VM after update
4768 sub __read_avail {
4769     my ($fh, $timeout) = @_;
4770
4771     my $sel = new IO::Select;
4772     $sel->add($fh);
4773
4774     my $res = '';
4775     my $buf;
4776
4777     my @ready;
4778     while (scalar (@ready = $sel->can_read($timeout))) {
4779         my $count;
4780         if ($count = $fh->sysread($buf, 8192)) {
4781             if ($buf =~ /^(.*)\(qemu\) $/s) {
4782                 $res .= $1;
4783                 last;
4784             } else {
4785                 $res .= $buf;
4786             }
4787         } else {
4788             if (!defined($count)) {
4789                 die "$!\n";
4790             }
4791             last;
4792         }
4793     }
4794
4795     die "monitor read timeout\n" if !scalar(@ready);
4796
4797     return $res;
4798 }
4799
4800 sub qemu_block_resize {
4801     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4802
4803     my $running = check_running($vmid);
4804
4805     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4806
4807     return if !$running;
4808
4809     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4810
4811 }
4812
4813 sub qemu_volume_snapshot {
4814     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4815
4816     my $running = check_running($vmid);
4817
4818     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4819         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4820     } else {
4821         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4822     }
4823 }
4824
4825 sub qemu_volume_snapshot_delete {
4826     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4827
4828     my $running = check_running($vmid);
4829
4830     if($running) {
4831
4832         $running = undef;
4833         my $conf = PVE::QemuConfig->load_config($vmid);
4834         foreach_drive($conf, sub {
4835             my ($ds, $drive) = @_;
4836             $running = 1 if $drive->{file} eq $volid;
4837         });
4838     }
4839
4840     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4841         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4842     } else {
4843         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4844     }
4845 }
4846
4847 sub set_migration_caps {
4848     my ($vmid) = @_;
4849
4850     my $cap_ref = [];
4851
4852     my $enabled_cap = {
4853         "auto-converge" => 1,
4854         "xbzrle" => 1,
4855         "x-rdma-pin-all" => 0,
4856         "zero-blocks" => 0,
4857         "compress" => 0
4858     };
4859
4860     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4861
4862     for my $supported_capability (@$supported_capabilities) {
4863         push @$cap_ref, {
4864             capability => $supported_capability->{capability},
4865             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4866         };
4867     }
4868
4869     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4870 }
4871
4872 my $fast_plug_option = {
4873     'lock' => 1,
4874     'name' => 1,
4875     'onboot' => 1,
4876     'shares' => 1,
4877     'startup' => 1,
4878     'description' => 1,
4879     'protection' => 1,
4880     'vmstatestorage' => 1,
4881     'hookscript' => 1,
4882 };
4883
4884 # hotplug changes in [PENDING]
4885 # $selection hash can be used to only apply specified options, for
4886 # example: { cores => 1 } (only apply changed 'cores')
4887 # $errors ref is used to return error messages
4888 sub vmconfig_hotplug_pending {
4889     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4890
4891     my $defaults = load_defaults();
4892     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4893
4894     # commit values which do not have any impact on running VM first
4895     # Note: those option cannot raise errors, we we do not care about
4896     # $selection and always apply them.
4897
4898     my $add_error = sub {
4899         my ($opt, $msg) = @_;
4900         $errors->{$opt} = "hotplug problem - $msg";
4901     };
4902
4903     my $changes = 0;
4904     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4905         if ($fast_plug_option->{$opt}) {
4906             $conf->{$opt} = $conf->{pending}->{$opt};
4907             delete $conf->{pending}->{$opt};
4908             $changes = 1;
4909         }
4910     }
4911
4912     if ($changes) {
4913         PVE::QemuConfig->write_config($vmid, $conf);
4914         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4915     }
4916
4917     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4918
4919     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4920     while (my ($opt, $force) = each %$pending_delete_hash) {
4921         next if $selection && !$selection->{$opt};
4922         eval {
4923             if ($opt eq 'hotplug') {
4924                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4925             } elsif ($opt eq 'tablet') {
4926                 die "skip\n" if !$hotplug_features->{usb};
4927                 if ($defaults->{tablet}) {
4928                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4929                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4930                         if $arch eq 'aarch64';
4931                 } else {
4932                     vm_deviceunplug($vmid, $conf, 'tablet');
4933                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4934                 }
4935             } elsif ($opt =~ m/^usb\d+/) {
4936                 die "skip\n";
4937                 # since we cannot reliably hot unplug usb devices
4938                 # we are disabling it
4939                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4940                 vm_deviceunplug($vmid, $conf, $opt);
4941             } elsif ($opt eq 'vcpus') {
4942                 die "skip\n" if !$hotplug_features->{cpu};
4943                 qemu_cpu_hotplug($vmid, $conf, undef);
4944             } elsif ($opt eq 'balloon') {
4945                 # enable balloon device is not hotpluggable
4946                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4947                 # here we reset the ballooning value to memory
4948                 my $balloon = $conf->{memory} || $defaults->{memory};
4949                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4950             } elsif ($fast_plug_option->{$opt}) {
4951                 # do nothing
4952             } elsif ($opt =~ m/^net(\d+)$/) {
4953                 die "skip\n" if !$hotplug_features->{network};
4954                 vm_deviceunplug($vmid, $conf, $opt);
4955             } elsif (is_valid_drivename($opt)) {
4956                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4957                 vm_deviceunplug($vmid, $conf, $opt);
4958                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4959             } elsif ($opt =~ m/^memory$/) {
4960                 die "skip\n" if !$hotplug_features->{memory};
4961                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4962             } elsif ($opt eq 'cpuunits') {
4963                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4964             } elsif ($opt eq 'cpulimit') {
4965                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4966             } else {
4967                 die "skip\n";
4968             }
4969         };
4970         if (my $err = $@) {
4971             &$add_error($opt, $err) if $err ne "skip\n";
4972         } else {
4973             # save new config if hotplug was successful
4974             delete $conf->{$opt};
4975             vmconfig_undelete_pending_option($conf, $opt);
4976             PVE::QemuConfig->write_config($vmid, $conf);
4977             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4978         }
4979     }
4980
4981     my $apply_pending_cloudinit;
4982     $apply_pending_cloudinit = sub {
4983         my ($key, $value) = @_;
4984         $apply_pending_cloudinit = sub {}; # once is enough
4985
4986         my @cloudinit_opts = keys %$confdesc_cloudinit;
4987         foreach my $opt (keys %{$conf->{pending}}) {
4988             next if !grep { $_ eq $opt } @cloudinit_opts;
4989             $conf->{$opt} = delete $conf->{pending}->{$opt};
4990         }
4991
4992         my $new_conf = { %$conf };
4993         $new_conf->{$key} = $value;
4994         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4995     };
4996
4997     foreach my $opt (keys %{$conf->{pending}}) {
4998         next if $selection && !$selection->{$opt};
4999         my $value = $conf->{pending}->{$opt};
5000         eval {
5001             if ($opt eq 'hotplug') {
5002                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5003             } elsif ($opt eq 'tablet') {
5004                 die "skip\n" if !$hotplug_features->{usb};
5005                 if ($value == 1) {
5006                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5007                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5008                         if $arch eq 'aarch64';
5009                 } elsif ($value == 0) {
5010                     vm_deviceunplug($vmid, $conf, 'tablet');
5011                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5012                 }
5013             } elsif ($opt =~ m/^usb\d+$/) {
5014                 die "skip\n";
5015                 # since we cannot reliably hot unplug usb devices
5016                 # we are disabling it
5017                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5018                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5019                 die "skip\n" if !$d;
5020                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5021             } elsif ($opt eq 'vcpus') {
5022                 die "skip\n" if !$hotplug_features->{cpu};
5023                 qemu_cpu_hotplug($vmid, $conf, $value);
5024             } elsif ($opt eq 'balloon') {
5025                 # enable/disable balloning device is not hotpluggable
5026                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5027                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5028                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5029
5030                 # allow manual ballooning if shares is set to zero
5031                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5032                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5033                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5034                 }
5035             } elsif ($opt =~ m/^net(\d+)$/) {
5036                 # some changes can be done without hotplug
5037                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5038                                     $vmid, $opt, $value, $arch, $machine_type);
5039             } elsif (is_valid_drivename($opt)) {
5040                 # some changes can be done without hotplug
5041                 my $drive = parse_drive($opt, $value);
5042                 if (drive_is_cloudinit($drive)) {
5043                     &$apply_pending_cloudinit($opt, $value);
5044                 }
5045                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5046                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5047             } elsif ($opt =~ m/^memory$/) { #dimms
5048                 die "skip\n" if !$hotplug_features->{memory};
5049                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5050             } elsif ($opt eq 'cpuunits') {
5051                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5052             } elsif ($opt eq 'cpulimit') {
5053                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5054                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5055             } else {
5056                 die "skip\n";  # skip non-hot-pluggable options
5057             }
5058         };
5059         if (my $err = $@) {
5060             &$add_error($opt, $err) if $err ne "skip\n";
5061         } else {
5062             # save new config if hotplug was successful
5063             $conf->{$opt} = $value;
5064             delete $conf->{pending}->{$opt};
5065             PVE::QemuConfig->write_config($vmid, $conf);
5066             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5067         }
5068     }
5069 }
5070
5071 sub try_deallocate_drive {
5072     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5073
5074     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5075         my $volid = $drive->{file};
5076         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5077             my $sid = PVE::Storage::parse_volume_id($volid);
5078             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5079
5080             # check if the disk is really unused
5081             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5082                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5083             PVE::Storage::vdisk_free($storecfg, $volid);
5084             return 1;
5085         } else {
5086             # If vm is not owner of this disk remove from config
5087             return 1;
5088         }
5089     }
5090
5091     return undef;
5092 }
5093
5094 sub vmconfig_delete_or_detach_drive {
5095     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5096
5097     my $drive = parse_drive($opt, $conf->{$opt});
5098
5099     my $rpcenv = PVE::RPCEnvironment::get();
5100     my $authuser = $rpcenv->get_user();
5101
5102     if ($force) {
5103         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5104         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5105     } else {
5106         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5107     }
5108 }
5109
5110 sub vmconfig_apply_pending {
5111     my ($vmid, $conf, $storecfg) = @_;
5112
5113     # cold plug
5114
5115     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5116     while (my ($opt, $force) = each %$pending_delete_hash) {
5117         die "internal error" if $opt =~ m/^unused/;
5118         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5119         if (!defined($conf->{$opt})) {
5120             vmconfig_undelete_pending_option($conf, $opt);
5121             PVE::QemuConfig->write_config($vmid, $conf);
5122         } elsif (is_valid_drivename($opt)) {
5123             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5124             vmconfig_undelete_pending_option($conf, $opt);
5125             delete $conf->{$opt};
5126             PVE::QemuConfig->write_config($vmid, $conf);
5127         } else {
5128             vmconfig_undelete_pending_option($conf, $opt);
5129             delete $conf->{$opt};
5130             PVE::QemuConfig->write_config($vmid, $conf);
5131         }
5132     }
5133
5134     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5135
5136     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5137         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5138
5139         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5140             # skip if nothing changed
5141         } elsif (is_valid_drivename($opt)) {
5142             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5143                 if defined($conf->{$opt});
5144             $conf->{$opt} = $conf->{pending}->{$opt};
5145         } else {
5146             $conf->{$opt} = $conf->{pending}->{$opt};
5147         }
5148
5149         delete $conf->{pending}->{$opt};
5150         PVE::QemuConfig->write_config($vmid, $conf);
5151     }
5152 }
5153
5154 my $safe_num_ne = sub {
5155     my ($a, $b) = @_;
5156
5157     return 0 if !defined($a) && !defined($b);
5158     return 1 if !defined($a);
5159     return 1 if !defined($b);
5160
5161     return $a != $b;
5162 };
5163
5164 my $safe_string_ne = sub {
5165     my ($a, $b) = @_;
5166
5167     return 0 if !defined($a) && !defined($b);
5168     return 1 if !defined($a);
5169     return 1 if !defined($b);
5170
5171     return $a ne $b;
5172 };
5173
5174 sub vmconfig_update_net {
5175     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5176
5177     my $newnet = parse_net($value);
5178
5179     if ($conf->{$opt}) {
5180         my $oldnet = parse_net($conf->{$opt});
5181
5182         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5183             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5184             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5185             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5186
5187             # for non online change, we try to hot-unplug
5188             die "skip\n" if !$hotplug;
5189             vm_deviceunplug($vmid, $conf, $opt);
5190         } else {
5191
5192             die "internal error" if $opt !~ m/net(\d+)/;
5193             my $iface = "tap${vmid}i$1";
5194
5195             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5196                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5197                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5198                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5199                 PVE::Network::tap_unplug($iface);
5200                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5201             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5202                 # Rate can be applied on its own but any change above needs to
5203                 # include the rate in tap_plug since OVS resets everything.
5204                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5205             }
5206
5207             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5208                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5209             }
5210
5211             return 1;
5212         }
5213     }
5214
5215     if ($hotplug) {
5216         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5217     } else {
5218         die "skip\n";
5219     }
5220 }
5221
5222 sub vmconfig_update_disk {
5223     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5224
5225     # fixme: do we need force?
5226
5227     my $drive = parse_drive($opt, $value);
5228
5229     if ($conf->{$opt}) {
5230
5231         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5232
5233             my $media = $drive->{media} || 'disk';
5234             my $oldmedia = $old_drive->{media} || 'disk';
5235             die "unable to change media type\n" if $media ne $oldmedia;
5236
5237             if (!drive_is_cdrom($old_drive)) {
5238
5239                 if ($drive->{file} ne $old_drive->{file}) {
5240
5241                     die "skip\n" if !$hotplug;
5242
5243                     # unplug and register as unused
5244                     vm_deviceunplug($vmid, $conf, $opt);
5245                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5246
5247                 } else {
5248                     # update existing disk
5249
5250                     # skip non hotpluggable value
5251                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5252                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5253                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5254                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5255                         die "skip\n";
5256                     }
5257
5258                     # apply throttle
5259                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5260                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5261                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5262                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5263                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5264                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5265                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5266                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5267                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5268                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5269                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5270                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5271                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5272                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5273                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5274                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5275                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5276                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5277
5278                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5279                                                    ($drive->{mbps} || 0)*1024*1024,
5280                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5281                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5282                                                    $drive->{iops} || 0,
5283                                                    $drive->{iops_rd} || 0,
5284                                                    $drive->{iops_wr} || 0,
5285                                                    ($drive->{mbps_max} || 0)*1024*1024,
5286                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5287                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5288                                                    $drive->{iops_max} || 0,
5289                                                    $drive->{iops_rd_max} || 0,
5290                                                    $drive->{iops_wr_max} || 0,
5291                                                    $drive->{bps_max_length} || 1,
5292                                                    $drive->{bps_rd_max_length} || 1,
5293                                                    $drive->{bps_wr_max_length} || 1,
5294                                                    $drive->{iops_max_length} || 1,
5295                                                    $drive->{iops_rd_max_length} || 1,
5296                                                    $drive->{iops_wr_max_length} || 1);
5297
5298                     }
5299
5300                     return 1;
5301                 }
5302
5303             } else { # cdrom
5304
5305                 if ($drive->{file} eq 'none') {
5306                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5307                     if (drive_is_cloudinit($old_drive)) {
5308                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5309                     }
5310                 } else {
5311                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5312                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5313                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5314                 }
5315
5316                 return 1;
5317             }
5318         }
5319     }
5320
5321     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5322     # hotplug new disks
5323     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5324     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5325 }
5326
5327 sub vm_start {
5328     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5329         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5330
5331     PVE::QemuConfig->lock_config($vmid, sub {
5332         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5333
5334         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5335
5336         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5337
5338         PVE::QemuConfig->check_lock($conf)
5339             if !($skiplock || $is_suspended);
5340
5341         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5342
5343         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5344             vmconfig_apply_pending($vmid, $conf, $storecfg);
5345             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5346         }
5347
5348         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5349
5350         my $defaults = load_defaults();
5351
5352         # set environment variable useful inside network script
5353         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5354
5355         my $local_volumes = {};
5356
5357         if ($targetstorage) {
5358             foreach_drive($conf, sub {
5359                 my ($ds, $drive) = @_;
5360
5361                 return if drive_is_cdrom($drive);
5362
5363                 my $volid = $drive->{file};
5364
5365                 return if !$volid;
5366
5367                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5368
5369                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5370                 return if $scfg->{shared};
5371                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5372             });
5373
5374             my $format = undef;
5375
5376             foreach my $opt (sort keys %$local_volumes) {
5377
5378                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5379                 my $drive = parse_drive($opt, $conf->{$opt});
5380
5381                 #if remote storage is specified, use default format
5382                 if ($targetstorage && $targetstorage ne "1") {
5383                     $storeid = $targetstorage;
5384                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5385                     $format = $defFormat;
5386                 } else {
5387                     #else we use same format than original
5388                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5389                     $format = qemu_img_format($scfg, $volid);
5390                 }
5391
5392                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5393                 my $newdrive = $drive;
5394                 $newdrive->{format} = $format;
5395                 $newdrive->{file} = $newvolid;
5396                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5397                 $local_volumes->{$opt} = $drivestr;
5398                 #pass drive to conf for command line
5399                 $conf->{$opt} = $drivestr;
5400             }
5401         }
5402
5403         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5404
5405         if ($is_suspended) {
5406             # enforce machine type on suspended vm to ensure HW compatibility
5407             $forcemachine = $conf->{runningmachine};
5408             print "Resuming suspended VM\n";
5409         }
5410
5411         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5412
5413         my $migrate_port = 0;
5414         my $migrate_uri;
5415         if ($statefile) {
5416             if ($statefile eq 'tcp') {
5417                 my $localip = "localhost";
5418                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5419                 my $nodename = PVE::INotify::nodename();
5420
5421                 if (!defined($migration_type)) {
5422                     if (defined($datacenterconf->{migration}->{type})) {
5423                         $migration_type = $datacenterconf->{migration}->{type};
5424                     } else {
5425                         $migration_type = 'secure';
5426                     }
5427                 }
5428
5429                 if ($migration_type eq 'insecure') {
5430                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5431                     if ($migrate_network_addr) {
5432                         $localip = $migrate_network_addr;
5433                     } else {
5434                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5435                     }
5436
5437                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5438                 }
5439
5440                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5441                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5442                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5443                 push @$cmd, '-incoming', $migrate_uri;
5444                 push @$cmd, '-S';
5445
5446             } elsif ($statefile eq 'unix') {
5447                 # should be default for secure migrations as a ssh TCP forward
5448                 # tunnel is not deterministic reliable ready and fails regurarly
5449                 # to set up in time, so use UNIX socket forwards
5450                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5451                 unlink $socket_addr;
5452
5453                 $migrate_uri = "unix:$socket_addr";
5454
5455                 push @$cmd, '-incoming', $migrate_uri;
5456                 push @$cmd, '-S';
5457
5458             } else {
5459                 push @$cmd, '-loadstate', $statefile;
5460             }
5461         } elsif ($paused) {
5462             push @$cmd, '-S';
5463         }
5464
5465         # host pci devices
5466         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5467           my $d = parse_hostpci($conf->{"hostpci$i"});
5468           next if !$d;
5469           my $pcidevices = $d->{pciid};
5470           foreach my $pcidevice (@$pcidevices) {
5471                 my $pciid = $pcidevice->{id};
5472
5473                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5474                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5475                 die "no pci device info for device '$pciid'\n" if !$info;
5476
5477                 if ($d->{mdev}) {
5478                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5479                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5480                 } else {
5481                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5482                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5483                     die "can't reset pci device '$pciid'\n"
5484                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5485                 }
5486           }
5487         }
5488
5489         PVE::Storage::activate_volumes($storecfg, $vollist);
5490
5491         eval {
5492             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5493                 outfunc => sub {}, errfunc => sub {});
5494         };
5495         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5496         # timeout should be more than enough here...
5497         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5498
5499         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5500                                                   : $defaults->{cpuunits};
5501
5502         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5503         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5504
5505         my %properties = (
5506             Slice => 'qemu.slice',
5507             KillMode => 'none',
5508             CPUShares => $cpuunits
5509         );
5510
5511         if (my $cpulimit = $conf->{cpulimit}) {
5512             $properties{CPUQuota} = int($cpulimit * 100);
5513         }
5514         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5515
5516         my $run_qemu = sub {
5517             PVE::Tools::run_fork sub {
5518                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5519                 run_command($cmd, %run_params);
5520             };
5521         };
5522
5523         if ($conf->{hugepages}) {
5524
5525             my $code = sub {
5526                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5527                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5528
5529                 PVE::QemuServer::Memory::hugepages_mount();
5530                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5531
5532                 eval { $run_qemu->() };
5533                 if (my $err = $@) {
5534                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5535                     die $err;
5536                 }
5537
5538                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5539             };
5540             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5541
5542         } else {
5543             eval { $run_qemu->() };
5544         }
5545
5546         if (my $err = $@) {
5547             # deactivate volumes if start fails
5548             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5549             die "start failed: $err";
5550         }
5551
5552         print "migration listens on $migrate_uri\n" if $migrate_uri;
5553
5554         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5555             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5556             warn $@ if $@;
5557         }
5558
5559         #start nbd server for storage migration
5560         if ($targetstorage) {
5561             my $nodename = PVE::INotify::nodename();
5562             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5563             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5564             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5565             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5566
5567             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5568
5569             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5570
5571             foreach my $opt (sort keys %$local_volumes) {
5572                 my $volid = $local_volumes->{$opt};
5573                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5574                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5575                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5576             }
5577         }
5578
5579         if ($migratedfrom) {
5580             eval {
5581                 set_migration_caps($vmid);
5582             };
5583             warn $@ if $@;
5584
5585             if ($spice_port) {
5586                 print "spice listens on port $spice_port\n";
5587                 if ($spice_ticket) {
5588                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5589                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5590                 }
5591             }
5592
5593         } else {
5594             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5595                 if !$statefile && $conf->{balloon};
5596
5597             foreach my $opt (keys %$conf) {
5598                 next if $opt !~  m/^net\d+$/;
5599                 my $nicconf = parse_net($conf->{$opt});
5600                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5601             }
5602         }
5603
5604         vm_mon_cmd_nocheck($vmid, 'qom-set',
5605                     path => "machine/peripheral/balloon0",
5606                     property => "guest-stats-polling-interval",
5607                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5608
5609         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5610             print "Resumed VM, removing state\n";
5611             delete $conf->@{qw(lock vmstate runningmachine)};
5612             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5613             PVE::Storage::vdisk_free($storecfg, $vmstate);
5614             PVE::QemuConfig->write_config($vmid, $conf);
5615         }
5616
5617         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5618     });
5619 }
5620
5621 sub vm_mon_cmd {
5622     my ($vmid, $execute, %params) = @_;
5623
5624     my $cmd = { execute => $execute, arguments => \%params };
5625     vm_qmp_command($vmid, $cmd);
5626 }
5627
5628 sub vm_mon_cmd_nocheck {
5629     my ($vmid, $execute, %params) = @_;
5630
5631     my $cmd = { execute => $execute, arguments => \%params };
5632     vm_qmp_command($vmid, $cmd, 1);
5633 }
5634
5635 sub vm_qmp_command {
5636     my ($vmid, $cmd, $nocheck) = @_;
5637
5638     my $res;
5639
5640     my $timeout;
5641     if ($cmd->{arguments}) {
5642         $timeout = delete $cmd->{arguments}->{timeout};
5643     }
5644
5645     eval {
5646         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5647         my $sname = qmp_socket($vmid);
5648         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5649             my $qmpclient = PVE::QMPClient->new();
5650
5651             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5652         } else {
5653             die "unable to open monitor socket\n";
5654         }
5655     };
5656     if (my $err = $@) {
5657         syslog("err", "VM $vmid qmp command failed - $err");
5658         die $err;
5659     }
5660
5661     return $res;
5662 }
5663
5664 sub vm_human_monitor_command {
5665     my ($vmid, $cmdline) = @_;
5666
5667     my $cmd = {
5668         execute => 'human-monitor-command',
5669         arguments => { 'command-line' => $cmdline},
5670     };
5671
5672     return vm_qmp_command($vmid, $cmd);
5673 }
5674
5675 sub vm_commandline {
5676     my ($storecfg, $vmid, $snapname) = @_;
5677
5678     my $conf = PVE::QemuConfig->load_config($vmid);
5679
5680     if ($snapname) {
5681         my $snapshot = $conf->{snapshots}->{$snapname};
5682         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5683
5684         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5685
5686         $conf = $snapshot;
5687     }
5688
5689     my $defaults = load_defaults();
5690
5691     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5692
5693     return PVE::Tools::cmd2string($cmd);
5694 }
5695
5696 sub vm_reset {
5697     my ($vmid, $skiplock) = @_;
5698
5699     PVE::QemuConfig->lock_config($vmid, sub {
5700
5701         my $conf = PVE::QemuConfig->load_config($vmid);
5702
5703         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5704
5705         vm_mon_cmd($vmid, "system_reset");
5706     });
5707 }
5708
5709 sub get_vm_volumes {
5710     my ($conf) = @_;
5711
5712     my $vollist = [];
5713     foreach_volid($conf, sub {
5714         my ($volid, $attr) = @_;
5715
5716         return if $volid =~ m|^/|;
5717
5718         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5719         return if !$sid;
5720
5721         push @$vollist, $volid;
5722     });
5723
5724     return $vollist;
5725 }
5726
5727 sub vm_stop_cleanup {
5728     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5729
5730     eval {
5731
5732         if (!$keepActive) {
5733             my $vollist = get_vm_volumes($conf);
5734             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5735         }
5736
5737         foreach my $ext (qw(mon qmp pid vnc qga)) {
5738             unlink "/var/run/qemu-server/${vmid}.$ext";
5739         }
5740
5741         if ($conf->{ivshmem}) {
5742             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5743             # just delete it for now, VMs which have this already open do not
5744             # are affected, but new VMs will get a separated one. If this
5745             # becomes an issue we either add some sort of ref-counting or just
5746             # add a "don't delete on stop" flag to the ivshmem format.
5747             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5748         }
5749
5750         foreach my $key (keys %$conf) {
5751             next if $key !~ m/^hostpci(\d+)$/;
5752             my $hostpciindex = $1;
5753             my $d = parse_hostpci($conf->{$key});
5754             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5755
5756             foreach my $pci (@{$d->{pciid}}) {
5757                 my $pciid = $pci->{id};
5758                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5759             }
5760         }
5761
5762         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5763     };
5764     warn $@ if $@; # avoid errors - just warn
5765 }
5766
5767 # Note: use $nockeck to skip tests if VM configuration file exists.
5768 # We need that when migration VMs to other nodes (files already moved)
5769 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5770 sub vm_stop {
5771     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5772
5773     $force = 1 if !defined($force) && !$shutdown;
5774
5775     if ($migratedfrom){
5776         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5777         kill 15, $pid if $pid;
5778         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5779         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5780         return;
5781     }
5782
5783     PVE::QemuConfig->lock_config($vmid, sub {
5784
5785         my $pid = check_running($vmid, $nocheck);
5786         return if !$pid;
5787
5788         my $conf;
5789         if (!$nocheck) {
5790             $conf = PVE::QemuConfig->load_config($vmid);
5791             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5792             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5793                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5794                 $timeout = $opts->{down} if $opts->{down};
5795             }
5796             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5797         }
5798
5799         eval {
5800             if ($shutdown) {
5801                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5802                     vm_qmp_command($vmid, {
5803                         execute => "guest-shutdown",
5804                         arguments => { timeout => $timeout }
5805                     }, $nocheck);
5806                 } else {
5807                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5808                 }
5809             } else {
5810                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5811             }
5812         };
5813         my $err = $@;
5814
5815         if (!$err) {
5816             $timeout = 60 if !defined($timeout);
5817
5818             my $count = 0;
5819             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5820                 $count++;
5821                 sleep 1;
5822             }
5823
5824             if ($count >= $timeout) {
5825                 if ($force) {
5826                     warn "VM still running - terminating now with SIGTERM\n";
5827                     kill 15, $pid;
5828                 } else {
5829                     die "VM quit/powerdown failed - got timeout\n";
5830                 }
5831             } else {
5832                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5833                 return;
5834             }
5835         } else {
5836             if ($force) {
5837                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5838                 kill 15, $pid;
5839             } else {
5840                 die "VM quit/powerdown failed\n";
5841             }
5842         }
5843
5844         # wait again
5845         $timeout = 10;
5846
5847         my $count = 0;
5848         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5849             $count++;
5850             sleep 1;
5851         }
5852
5853         if ($count >= $timeout) {
5854             warn "VM still running - terminating now with SIGKILL\n";
5855             kill 9, $pid;
5856             sleep 1;
5857         }
5858
5859         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5860    });
5861 }
5862
5863 sub vm_suspend {
5864     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5865
5866     my $conf;
5867     my $path;
5868     my $storecfg;
5869     my $vmstate;
5870
5871     PVE::QemuConfig->lock_config($vmid, sub {
5872
5873         $conf = PVE::QemuConfig->load_config($vmid);
5874
5875         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5876         PVE::QemuConfig->check_lock($conf)
5877             if !($skiplock || $is_backing_up);
5878
5879         die "cannot suspend to disk during backup\n"
5880             if $is_backing_up && $includestate;
5881
5882         if ($includestate) {
5883             $conf->{lock} = 'suspending';
5884             my $date = strftime("%Y-%m-%d", localtime(time()));
5885             $storecfg = PVE::Storage::config();
5886             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5887             $path = PVE::Storage::path($storecfg, $vmstate);
5888             PVE::QemuConfig->write_config($vmid, $conf);
5889         } else {
5890             vm_mon_cmd($vmid, "stop");
5891         }
5892     });
5893
5894     if ($includestate) {
5895         # save vm state
5896         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5897
5898         eval {
5899             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5900             for(;;) {
5901                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5902                 if (!$state->{status}) {
5903                     die "savevm not active\n";
5904                 } elsif ($state->{status} eq 'active') {
5905                     sleep(1);
5906                     next;
5907                 } elsif ($state->{status} eq 'completed') {
5908                     print "State saved, quitting\n";
5909                     last;
5910                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5911                     die "query-savevm failed with error '$state->{error}'\n"
5912                 } else {
5913                     die "query-savevm returned status '$state->{status}'\n";
5914                 }
5915             }
5916         };
5917         my $err = $@;
5918
5919         PVE::QemuConfig->lock_config($vmid, sub {
5920             $conf = PVE::QemuConfig->load_config($vmid);
5921             if ($err) {
5922                 # cleanup, but leave suspending lock, to indicate something went wrong
5923                 eval {
5924                     vm_mon_cmd($vmid, "savevm-end");
5925                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5926                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5927                     delete $conf->@{qw(vmstate runningmachine)};
5928                     PVE::QemuConfig->write_config($vmid, $conf);
5929                 };
5930                 warn $@ if $@;
5931                 die $err;
5932             }
5933
5934             die "lock changed unexpectedly\n"
5935                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5936
5937             vm_qmp_command($vmid, { execute => "quit" });
5938             $conf->{lock} = 'suspended';
5939             PVE::QemuConfig->write_config($vmid, $conf);
5940         });
5941     }
5942 }
5943
5944 sub vm_resume {
5945     my ($vmid, $skiplock, $nocheck) = @_;
5946
5947     PVE::QemuConfig->lock_config($vmid, sub {
5948         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5949         my $res = $vm_mon_cmd->($vmid, 'query-status');
5950         my $resume_cmd = 'cont';
5951
5952         if ($res->{status} && $res->{status} eq 'suspended') {
5953             $resume_cmd = 'system_wakeup';
5954         }
5955
5956         if (!$nocheck) {
5957
5958             my $conf = PVE::QemuConfig->load_config($vmid);
5959
5960             PVE::QemuConfig->check_lock($conf)
5961                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5962         }
5963
5964         $vm_mon_cmd->($vmid, $resume_cmd);
5965     });
5966 }
5967
5968 sub vm_sendkey {
5969     my ($vmid, $skiplock, $key) = @_;
5970
5971     PVE::QemuConfig->lock_config($vmid, sub {
5972
5973         my $conf = PVE::QemuConfig->load_config($vmid);
5974
5975         # there is no qmp command, so we use the human monitor command
5976         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5977         die $res if $res ne '';
5978     });
5979 }
5980
5981 sub vm_destroy {
5982     my ($storecfg, $vmid, $skiplock) = @_;
5983
5984     PVE::QemuConfig->lock_config($vmid, sub {
5985
5986         my $conf = PVE::QemuConfig->load_config($vmid);
5987
5988         if (!check_running($vmid)) {
5989             destroy_vm($storecfg, $vmid, undef, $skiplock);
5990         } else {
5991             die "VM $vmid is running - destroy failed\n";
5992         }
5993     });
5994 }
5995
5996 # vzdump restore implementaion
5997
5998 sub tar_archive_read_firstfile {
5999     my $archive = shift;
6000
6001     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6002
6003     # try to detect archive type first
6004     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6005         die "unable to open file '$archive'\n";
6006     my $firstfile = <$fh>;
6007     kill 15, $pid;
6008     close $fh;
6009
6010     die "ERROR: archive contaions no data\n" if !$firstfile;
6011     chomp $firstfile;
6012
6013     return $firstfile;
6014 }
6015
6016 sub tar_restore_cleanup {
6017     my ($storecfg, $statfile) = @_;
6018
6019     print STDERR "starting cleanup\n";
6020
6021     if (my $fd = IO::File->new($statfile, "r")) {
6022         while (defined(my $line = <$fd>)) {
6023             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6024                 my $volid = $2;
6025                 eval {
6026                     if ($volid =~ m|^/|) {
6027                         unlink $volid || die 'unlink failed\n';
6028                     } else {
6029                         PVE::Storage::vdisk_free($storecfg, $volid);
6030                     }
6031                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6032                 };
6033                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6034             } else {
6035                 print STDERR "unable to parse line in statfile - $line";
6036             }
6037         }
6038         $fd->close();
6039     }
6040 }
6041
6042 sub restore_archive {
6043     my ($archive, $vmid, $user, $opts) = @_;
6044
6045     my $format = $opts->{format};
6046     my $comp;
6047
6048     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6049         $format = 'tar' if !$format;
6050         $comp = 'gzip';
6051     } elsif ($archive =~ m/\.tar$/) {
6052         $format = 'tar' if !$format;
6053     } elsif ($archive =~ m/.tar.lzo$/) {
6054         $format = 'tar' if !$format;
6055         $comp = 'lzop';
6056     } elsif ($archive =~ m/\.vma$/) {
6057         $format = 'vma' if !$format;
6058     } elsif ($archive =~ m/\.vma\.gz$/) {
6059         $format = 'vma' if !$format;
6060         $comp = 'gzip';
6061     } elsif ($archive =~ m/\.vma\.lzo$/) {
6062         $format = 'vma' if !$format;
6063         $comp = 'lzop';
6064     } else {
6065         $format = 'vma' if !$format; # default
6066     }
6067
6068     # try to detect archive format
6069     if ($format eq 'tar') {
6070         return restore_tar_archive($archive, $vmid, $user, $opts);
6071     } else {
6072         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6073     }
6074 }
6075
6076 sub restore_update_config_line {
6077     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6078
6079     return if $line =~ m/^\#qmdump\#/;
6080     return if $line =~ m/^\#vzdump\#/;
6081     return if $line =~ m/^lock:/;
6082     return if $line =~ m/^unused\d+:/;
6083     return if $line =~ m/^parent:/;
6084
6085     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6086     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6087         # try to convert old 1.X settings
6088         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6089         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6090             my ($model, $macaddr) = split(/\=/, $devconfig);
6091             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6092             my $net = {
6093                 model => $model,
6094                 bridge => "vmbr$ind",
6095                 macaddr => $macaddr,
6096             };
6097             my $netstr = print_net($net);
6098
6099             print $outfd "net$cookie->{netcount}: $netstr\n";
6100             $cookie->{netcount}++;
6101         }
6102     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6103         my ($id, $netstr) = ($1, $2);
6104         my $net = parse_net($netstr);
6105         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6106         $netstr = print_net($net);
6107         print $outfd "$id: $netstr\n";
6108     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6109         my $virtdev = $1;
6110         my $value = $3;
6111         my $di = parse_drive($virtdev, $value);
6112         if (defined($di->{backup}) && !$di->{backup}) {
6113             print $outfd "#$line";
6114         } elsif ($map->{$virtdev}) {
6115             delete $di->{format}; # format can change on restore
6116             $di->{file} = $map->{$virtdev};
6117             $value = print_drive($vmid, $di);
6118             print $outfd "$virtdev: $value\n";
6119         } else {
6120             print $outfd $line;
6121         }
6122     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6123         my $vmgenid = $1;
6124         if ($vmgenid ne '0') {
6125             # always generate a new vmgenid if there was a valid one setup
6126             $vmgenid = generate_uuid();
6127         }
6128         print $outfd "vmgenid: $vmgenid\n";
6129     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6130         my ($uuid, $uuid_str);
6131         UUID::generate($uuid);
6132         UUID::unparse($uuid, $uuid_str);
6133         my $smbios1 = parse_smbios1($2);
6134         $smbios1->{uuid} = $uuid_str;
6135         print $outfd $1.print_smbios1($smbios1)."\n";
6136     } else {
6137         print $outfd $line;
6138     }
6139 }
6140
6141 sub scan_volids {
6142     my ($cfg, $vmid) = @_;
6143
6144     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6145
6146     my $volid_hash = {};
6147     foreach my $storeid (keys %$info) {
6148         foreach my $item (@{$info->{$storeid}}) {
6149             next if !($item->{volid} && $item->{size});
6150             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6151             $volid_hash->{$item->{volid}} = $item;
6152         }
6153     }
6154
6155     return $volid_hash;
6156 }
6157
6158 sub is_volume_in_use {
6159     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6160
6161     my $path = PVE::Storage::path($storecfg, $volid);
6162
6163     my $scan_config = sub {
6164         my ($cref, $snapname) = @_;
6165
6166         foreach my $key (keys %$cref) {
6167             my $value = $cref->{$key};
6168             if (is_valid_drivename($key)) {
6169                 next if $skip_drive && $key eq $skip_drive;
6170                 my $drive = parse_drive($key, $value);
6171                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6172                 return 1 if $volid eq $drive->{file};
6173                 if ($drive->{file} =~ m!^/!) {
6174                     return 1 if $drive->{file} eq $path;
6175                 } else {
6176                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6177                     next if !$storeid;
6178                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6179                     next if !$scfg;
6180                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6181                 }
6182             }
6183         }
6184
6185         return 0;
6186     };
6187
6188     return 1 if &$scan_config($conf);
6189
6190     undef $skip_drive;
6191
6192     foreach my $snapname (keys %{$conf->{snapshots}}) {
6193         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6194     }
6195
6196     return 0;
6197 }
6198
6199 sub update_disksize {
6200     my ($vmid, $conf, $volid_hash) = @_;
6201
6202     my $changes;
6203     my $prefix = "VM $vmid:";
6204
6205     # used and unused disks
6206     my $referenced = {};
6207
6208     # Note: it is allowed to define multiple storages with same path (alias), so
6209     # we need to check both 'volid' and real 'path' (two different volid can point
6210     # to the same path).
6211
6212     my $referencedpath = {};
6213
6214     # update size info
6215     foreach my $opt (keys %$conf) {
6216         if (is_valid_drivename($opt)) {
6217             my $drive = parse_drive($opt, $conf->{$opt});
6218             my $volid = $drive->{file};
6219             next if !$volid;
6220
6221             $referenced->{$volid} = 1;
6222             if ($volid_hash->{$volid} &&
6223                 (my $path = $volid_hash->{$volid}->{path})) {
6224                 $referencedpath->{$path} = 1;
6225             }
6226
6227             next if drive_is_cdrom($drive);
6228             next if !$volid_hash->{$volid};
6229
6230             $drive->{size} = $volid_hash->{$volid}->{size};
6231             my $new = print_drive($vmid, $drive);
6232             if ($new ne $conf->{$opt}) {
6233                 $changes = 1;
6234                 $conf->{$opt} = $new;
6235                 print "$prefix update disk '$opt' information.\n";
6236             }
6237         }
6238     }
6239
6240     # remove 'unusedX' entry if volume is used
6241     foreach my $opt (keys %$conf) {
6242         next if $opt !~ m/^unused\d+$/;
6243         my $volid = $conf->{$opt};
6244         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6245         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6246             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6247             $changes = 1;
6248             delete $conf->{$opt};
6249         }
6250
6251         $referenced->{$volid} = 1;
6252         $referencedpath->{$path} = 1 if $path;
6253     }
6254
6255     foreach my $volid (sort keys %$volid_hash) {
6256         next if $volid =~ m/vm-$vmid-state-/;
6257         next if $referenced->{$volid};
6258         my $path = $volid_hash->{$volid}->{path};
6259         next if !$path; # just to be sure
6260         next if $referencedpath->{$path};
6261         $changes = 1;
6262         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6263         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6264         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6265     }
6266
6267     return $changes;
6268 }
6269
6270 sub rescan {
6271     my ($vmid, $nolock, $dryrun) = @_;
6272
6273     my $cfg = PVE::Storage::config();
6274
6275     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6276     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6277     foreach my $stor (keys %{$cfg->{ids}}) {
6278         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6279     }
6280
6281     print "rescan volumes...\n";
6282     my $volid_hash = scan_volids($cfg, $vmid);
6283
6284     my $updatefn =  sub {
6285         my ($vmid) = @_;
6286
6287         my $conf = PVE::QemuConfig->load_config($vmid);
6288
6289         PVE::QemuConfig->check_lock($conf);
6290
6291         my $vm_volids = {};
6292         foreach my $volid (keys %$volid_hash) {
6293             my $info = $volid_hash->{$volid};
6294             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6295         }
6296
6297         my $changes = update_disksize($vmid, $conf, $vm_volids);
6298
6299         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6300     };
6301
6302     if (defined($vmid)) {
6303         if ($nolock) {
6304             &$updatefn($vmid);
6305         } else {
6306             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6307         }
6308     } else {
6309         my $vmlist = config_list();
6310         foreach my $vmid (keys %$vmlist) {
6311             if ($nolock) {
6312                 &$updatefn($vmid);
6313             } else {
6314                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6315             }
6316         }
6317     }
6318 }
6319
6320 sub restore_vma_archive {
6321     my ($archive, $vmid, $user, $opts, $comp) = @_;
6322
6323     my $readfrom = $archive;
6324
6325     my $cfg = PVE::Storage::config();
6326     my $commands = [];
6327     my $bwlimit = $opts->{bwlimit};
6328
6329     my $dbg_cmdstring = '';
6330     my $add_pipe = sub {
6331         my ($cmd) = @_;
6332         push @$commands, $cmd;
6333         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6334         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6335         $readfrom = '-';
6336     };
6337
6338     my $input = undef;
6339     if ($archive eq '-') {
6340         $input = '<&STDIN';
6341     } else {
6342         # If we use a backup from a PVE defined storage we also consider that
6343         # storage's rate limit:
6344         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6345         if (defined($volid)) {
6346             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6347             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6348             if ($readlimit) {
6349                 print STDERR "applying read rate limit: $readlimit\n";
6350                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6351                 $add_pipe->($cstream);
6352             }
6353         }
6354     }
6355
6356     if ($comp) {
6357         my $cmd;
6358         if ($comp eq 'gzip') {
6359             $cmd = ['zcat', $readfrom];
6360         } elsif ($comp eq 'lzop') {
6361             $cmd = ['lzop', '-d', '-c', $readfrom];
6362         } else {
6363             die "unknown compression method '$comp'\n";
6364         }
6365         $add_pipe->($cmd);
6366     }
6367
6368     my $tmpdir = "/var/tmp/vzdumptmp$$";
6369     rmtree $tmpdir;
6370
6371     # disable interrupts (always do cleanups)
6372     local $SIG{INT} =
6373         local $SIG{TERM} =
6374         local $SIG{QUIT} =
6375         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6376
6377     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6378     POSIX::mkfifo($mapfifo, 0600);
6379     my $fifofh;
6380
6381     my $openfifo = sub {
6382         open($fifofh, '>', $mapfifo) || die $!;
6383     };
6384
6385     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6386
6387     my $oldtimeout;
6388     my $timeout = 5;
6389
6390     my $devinfo = {};
6391
6392     my $rpcenv = PVE::RPCEnvironment::get();
6393
6394     my $conffile = PVE::QemuConfig->config_file($vmid);
6395     my $tmpfn = "$conffile.$$.tmp";
6396
6397     # Note: $oldconf is undef if VM does not exists
6398     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6399     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6400
6401     my %storage_limits;
6402
6403     my $print_devmap = sub {
6404         my $virtdev_hash = {};
6405
6406         my $cfgfn = "$tmpdir/qemu-server.conf";
6407
6408         # we can read the config - that is already extracted
6409         my $fh = IO::File->new($cfgfn, "r") ||
6410             "unable to read qemu-server.conf - $!\n";
6411
6412         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6413         if (-f $fwcfgfn) {
6414             my $pve_firewall_dir = '/etc/pve/firewall';
6415             mkdir $pve_firewall_dir; # make sure the dir exists
6416             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6417         }
6418
6419         while (defined(my $line = <$fh>)) {
6420             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6421                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6422                 die "archive does not contain data for drive '$virtdev'\n"
6423                     if !$devinfo->{$devname};
6424                 if (defined($opts->{storage})) {
6425                     $storeid = $opts->{storage} || 'local';
6426                 } elsif (!$storeid) {
6427                     $storeid = 'local';
6428                 }
6429                 $format = 'raw' if !$format;
6430                 $devinfo->{$devname}->{devname} = $devname;
6431                 $devinfo->{$devname}->{virtdev} = $virtdev;
6432                 $devinfo->{$devname}->{format} = $format;
6433                 $devinfo->{$devname}->{storeid} = $storeid;
6434
6435                 # check permission on storage
6436                 my $pool = $opts->{pool}; # todo: do we need that?
6437                 if ($user ne 'root@pam') {
6438                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6439                 }
6440
6441                 $storage_limits{$storeid} = $bwlimit;
6442
6443                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6444             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6445                 my $virtdev = $1;
6446                 my $drive = parse_drive($virtdev, $2);
6447                 if (drive_is_cloudinit($drive)) {
6448                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6449                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6450                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6451
6452                     my $d = {
6453                         format => $format,
6454                         storeid => $opts->{storage} // $storeid,
6455                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6456                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6457                         name => "vm-$vmid-cloudinit",
6458                         is_cloudinit => 1,
6459                     };
6460                     $virtdev_hash->{$virtdev} = $d;
6461                 }
6462             }
6463         }
6464
6465         foreach my $key (keys %storage_limits) {
6466             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6467             next if !$limit;
6468             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6469             $storage_limits{$key} = $limit * 1024;
6470         }
6471
6472         foreach my $devname (keys %$devinfo) {
6473             die "found no device mapping information for device '$devname'\n"
6474                 if !$devinfo->{$devname}->{virtdev};
6475         }
6476
6477         # create empty/temp config
6478         if ($oldconf) {
6479             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6480             foreach_drive($oldconf, sub {
6481                 my ($ds, $drive) = @_;
6482
6483                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6484
6485                 my $volid = $drive->{file};
6486                 return if !$volid || $volid =~ m|^/|;
6487
6488                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6489                 return if !$path || !$owner || ($owner != $vmid);
6490
6491                 # Note: only delete disk we want to restore
6492                 # other volumes will become unused
6493                 if ($virtdev_hash->{$ds}) {
6494                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6495                     if (my $err = $@) {
6496                         warn $err;
6497                     }
6498                 }
6499             });
6500
6501             # delete vmstate files, after the restore we have no snapshots anymore
6502             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6503                 my $snap = $oldconf->{snapshots}->{$snapname};
6504                 if ($snap->{vmstate}) {
6505                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6506                     if (my $err = $@) {
6507                         warn $err;
6508                     }
6509                 }
6510             }
6511         }
6512
6513         my $map = {};
6514         foreach my $virtdev (sort keys %$virtdev_hash) {
6515             my $d = $virtdev_hash->{$virtdev};
6516             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6517             my $storeid = $d->{storeid};
6518             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6519
6520             my $map_opts = '';
6521             if (my $limit = $storage_limits{$storeid}) {
6522                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6523             }
6524
6525             # test if requested format is supported
6526             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6527             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6528             $d->{format} = $defFormat if !$supported;
6529
6530             my $name;
6531             if ($d->{is_cloudinit}) {
6532                 $name = $d->{name};
6533                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6534             }
6535
6536             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6537             print STDERR "new volume ID is '$volid'\n";
6538             $d->{volid} = $volid;
6539
6540             PVE::Storage::activate_volumes($cfg, [$volid]);
6541
6542             my $write_zeros = 1;
6543             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6544                 $write_zeros = 0;
6545             }
6546
6547             if (!$d->{is_cloudinit}) {
6548                 my $path = PVE::Storage::path($cfg, $volid);
6549
6550                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6551
6552                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6553             }
6554             $map->{$virtdev} = $volid;
6555         }
6556
6557         $fh->seek(0, 0) || die "seek failed - $!\n";
6558
6559         my $outfd = new IO::File ($tmpfn, "w") ||
6560             die "unable to write config for VM $vmid\n";
6561
6562         my $cookie = { netcount => 0 };
6563         while (defined(my $line = <$fh>)) {
6564             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6565         }
6566
6567         $fh->close();
6568         $outfd->close();
6569     };
6570
6571     eval {
6572         # enable interrupts
6573         local $SIG{INT} =
6574             local $SIG{TERM} =
6575             local $SIG{QUIT} =
6576             local $SIG{HUP} =
6577             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6578         local $SIG{ALRM} = sub { die "got timeout\n"; };
6579
6580         $oldtimeout = alarm($timeout);
6581
6582         my $parser = sub {
6583             my $line = shift;
6584
6585             print "$line\n";
6586
6587             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6588                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6589                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6590             } elsif ($line =~ m/^CTIME: /) {
6591                 # we correctly received the vma config, so we can disable
6592                 # the timeout now for disk allocation (set to 10 minutes, so
6593                 # that we always timeout if something goes wrong)
6594                 alarm(600);
6595                 &$print_devmap();
6596                 print $fifofh "done\n";
6597                 my $tmp = $oldtimeout || 0;
6598                 $oldtimeout = undef;
6599                 alarm($tmp);
6600                 close($fifofh);
6601             }
6602         };
6603
6604         print "restore vma archive: $dbg_cmdstring\n";
6605         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6606     };
6607     my $err = $@;
6608
6609     alarm($oldtimeout) if $oldtimeout;
6610
6611     my $vollist = [];
6612     foreach my $devname (keys %$devinfo) {
6613         my $volid = $devinfo->{$devname}->{volid};
6614         push @$vollist, $volid if $volid;
6615     }
6616
6617     PVE::Storage::deactivate_volumes($cfg, $vollist);
6618
6619     unlink $mapfifo;
6620
6621     if ($err) {
6622         rmtree $tmpdir;
6623         unlink $tmpfn;
6624
6625         foreach my $devname (keys %$devinfo) {
6626             my $volid = $devinfo->{$devname}->{volid};
6627             next if !$volid;
6628             eval {
6629                 if ($volid =~ m|^/|) {
6630                     unlink $volid || die 'unlink failed\n';
6631                 } else {
6632                     PVE::Storage::vdisk_free($cfg, $volid);
6633                 }
6634                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6635             };
6636             print STDERR "unable to cleanup '$volid' - $@" if $@;
6637         }
6638         die $err;
6639     }
6640
6641     rmtree $tmpdir;
6642
6643     rename($tmpfn, $conffile) ||
6644         die "unable to commit configuration file '$conffile'\n";
6645
6646     PVE::Cluster::cfs_update(); # make sure we read new file
6647
6648     eval { rescan($vmid, 1); };
6649     warn $@ if $@;
6650 }
6651
6652 sub restore_tar_archive {
6653     my ($archive, $vmid, $user, $opts) = @_;
6654
6655     if ($archive ne '-') {
6656         my $firstfile = tar_archive_read_firstfile($archive);
6657         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6658             if $firstfile ne 'qemu-server.conf';
6659     }
6660
6661     my $storecfg = PVE::Storage::config();
6662
6663     # destroy existing data - keep empty config
6664     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6665     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6666
6667     my $tocmd = "/usr/lib/qemu-server/qmextract";
6668
6669     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6670     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6671     $tocmd .= ' --prealloc' if $opts->{prealloc};
6672     $tocmd .= ' --info' if $opts->{info};
6673
6674     # tar option "xf" does not autodetect compression when read from STDIN,
6675     # so we pipe to zcat
6676     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6677         PVE::Tools::shellquote("--to-command=$tocmd");
6678
6679     my $tmpdir = "/var/tmp/vzdumptmp$$";
6680     mkpath $tmpdir;
6681
6682     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6683     local $ENV{VZDUMP_VMID} = $vmid;
6684     local $ENV{VZDUMP_USER} = $user;
6685
6686     my $conffile = PVE::QemuConfig->config_file($vmid);
6687     my $tmpfn = "$conffile.$$.tmp";
6688
6689     # disable interrupts (always do cleanups)
6690     local $SIG{INT} =
6691         local $SIG{TERM} =
6692         local $SIG{QUIT} =
6693         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6694
6695     eval {
6696         # enable interrupts
6697         local $SIG{INT} =
6698             local $SIG{TERM} =
6699             local $SIG{QUIT} =
6700             local $SIG{HUP} =
6701             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6702
6703         if ($archive eq '-') {
6704             print "extracting archive from STDIN\n";
6705             run_command($cmd, input => "<&STDIN");
6706         } else {
6707             print "extracting archive '$archive'\n";
6708             run_command($cmd);
6709         }
6710
6711         return if $opts->{info};
6712
6713         # read new mapping
6714         my $map = {};
6715         my $statfile = "$tmpdir/qmrestore.stat";
6716         if (my $fd = IO::File->new($statfile, "r")) {
6717             while (defined (my $line = <$fd>)) {
6718                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6719                     $map->{$1} = $2 if $1;
6720                 } else {
6721                     print STDERR "unable to parse line in statfile - $line\n";
6722                 }
6723             }
6724             $fd->close();
6725         }
6726
6727         my $confsrc = "$tmpdir/qemu-server.conf";
6728
6729         my $srcfd = new IO::File($confsrc, "r") ||
6730             die "unable to open file '$confsrc'\n";
6731
6732         my $outfd = new IO::File ($tmpfn, "w") ||
6733             die "unable to write config for VM $vmid\n";
6734
6735         my $cookie = { netcount => 0 };
6736         while (defined (my $line = <$srcfd>)) {
6737             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6738         }
6739
6740         $srcfd->close();
6741         $outfd->close();
6742     };
6743     my $err = $@;
6744
6745     if ($err) {
6746
6747         unlink $tmpfn;
6748
6749         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6750
6751         die $err;
6752     }
6753
6754     rmtree $tmpdir;
6755
6756     rename $tmpfn, $conffile ||
6757         die "unable to commit configuration file '$conffile'\n";
6758
6759     PVE::Cluster::cfs_update(); # make sure we read new file
6760
6761     eval { rescan($vmid, 1); };
6762     warn $@ if $@;
6763 };
6764
6765 sub foreach_storage_used_by_vm {
6766     my ($conf, $func) = @_;
6767
6768     my $sidhash = {};
6769
6770     foreach_drive($conf, sub {
6771         my ($ds, $drive) = @_;
6772         return if drive_is_cdrom($drive);
6773
6774         my $volid = $drive->{file};
6775
6776         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6777         $sidhash->{$sid} = $sid if $sid;
6778     });
6779
6780     foreach my $sid (sort keys %$sidhash) {
6781         &$func($sid);
6782     }
6783 }
6784
6785 sub do_snapshots_with_qemu {
6786     my ($storecfg, $volid) = @_;
6787
6788     my $storage_name = PVE::Storage::parse_volume_id($volid);
6789     my $scfg = $storecfg->{ids}->{$storage_name};
6790
6791     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6792         return 1;
6793     }
6794
6795     if ($volid =~ m/\.(qcow2|qed)$/){
6796         return 1;
6797     }
6798
6799     return undef;
6800 }
6801
6802 sub qga_check_running {
6803     my ($vmid, $nowarn) = @_;
6804
6805     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6806     if ($@) {
6807         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6808         return 0;
6809     }
6810     return 1;
6811 }
6812
6813 sub template_create {
6814     my ($vmid, $conf, $disk) = @_;
6815
6816     my $storecfg = PVE::Storage::config();
6817
6818     foreach_drive($conf, sub {
6819         my ($ds, $drive) = @_;
6820
6821         return if drive_is_cdrom($drive);
6822         return if $disk && $ds ne $disk;
6823
6824         my $volid = $drive->{file};
6825         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6826
6827         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6828         $drive->{file} = $voliddst;
6829         $conf->{$ds} = print_drive($vmid, $drive);
6830         PVE::QemuConfig->write_config($vmid, $conf);
6831     });
6832 }
6833
6834 sub convert_iscsi_path {
6835     my ($path) = @_;
6836
6837     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6838         my $portal = $1;
6839         my $target = $2;
6840         my $lun = $3;
6841
6842         my $initiator_name = get_initiator_name();
6843
6844         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6845                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6846     }
6847
6848     die "cannot convert iscsi path '$path', unkown format\n";
6849 }
6850
6851 sub qemu_img_convert {
6852     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6853
6854     my $storecfg = PVE::Storage::config();
6855     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6856     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6857
6858     if ($src_storeid && $dst_storeid) {
6859
6860         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6861
6862         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6863         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6864
6865         my $src_format = qemu_img_format($src_scfg, $src_volname);
6866         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6867
6868         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6869         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6870
6871         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6872         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6873
6874         my $cmd = [];
6875         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6876         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6877         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6878         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6879
6880         if ($src_is_iscsi) {
6881             push @$cmd, '--image-opts';
6882             $src_path = convert_iscsi_path($src_path);
6883         } else {
6884             push @$cmd, '-f', $src_format;
6885         }
6886
6887         if ($dst_is_iscsi) {
6888             push @$cmd, '--target-image-opts';
6889             $dst_path = convert_iscsi_path($dst_path);
6890         } else {
6891             push @$cmd, '-O', $dst_format;
6892         }
6893
6894         push @$cmd, $src_path;
6895
6896         if (!$dst_is_iscsi && $is_zero_initialized) {
6897             push @$cmd, "zeroinit:$dst_path";
6898         } else {
6899             push @$cmd, $dst_path;
6900         }
6901
6902         my $parser = sub {
6903             my $line = shift;
6904             if($line =~ m/\((\S+)\/100\%\)/){
6905                 my $percent = $1;
6906                 my $transferred = int($size * $percent / 100);
6907                 my $remaining = $size - $transferred;
6908
6909                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6910             }
6911
6912         };
6913
6914         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6915         my $err = $@;
6916         die "copy failed: $err" if $err;
6917     }
6918 }
6919
6920 sub qemu_img_format {
6921     my ($scfg, $volname) = @_;
6922
6923     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6924         return $1;
6925     } else {
6926         return "raw";
6927     }
6928 }
6929
6930 sub qemu_drive_mirror {
6931     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6932
6933     $jobs = {} if !$jobs;
6934
6935     my $qemu_target;
6936     my $format;
6937     $jobs->{"drive-$drive"} = {};
6938
6939     if ($dst_volid =~ /^nbd:/) {
6940         $qemu_target = $dst_volid;
6941         $format = "nbd";
6942     } else {
6943         my $storecfg = PVE::Storage::config();
6944         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6945
6946         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6947
6948         $format = qemu_img_format($dst_scfg, $dst_volname);
6949
6950         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6951
6952         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6953     }
6954
6955     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6956     $opts->{format} = $format if $format;
6957
6958     if (defined($bwlimit)) {
6959         $opts->{speed} = $bwlimit * 1024;
6960         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6961     } else {
6962         print "drive mirror is starting for drive-$drive\n";
6963     }
6964
6965     # if a job already runs for this device we get an error, catch it for cleanup
6966     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6967     if (my $err = $@) {
6968         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6969         warn "$@\n" if $@;
6970         die "mirroring error: $err\n";
6971     }
6972
6973     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6974 }
6975
6976 sub qemu_drive_mirror_monitor {
6977     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6978
6979     eval {
6980         my $err_complete = 0;
6981
6982         while (1) {
6983             die "storage migration timed out\n" if $err_complete > 300;
6984
6985             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6986
6987             my $running_mirror_jobs = {};
6988             foreach my $stat (@$stats) {
6989                 next if $stat->{type} ne 'mirror';
6990                 $running_mirror_jobs->{$stat->{device}} = $stat;
6991             }
6992
6993             my $readycounter = 0;
6994
6995             foreach my $job (keys %$jobs) {
6996
6997                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6998                     print "$job : finished\n";
6999                     delete $jobs->{$job};
7000                     next;
7001                 }
7002
7003                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7004
7005                 my $busy = $running_mirror_jobs->{$job}->{busy};
7006                 my $ready = $running_mirror_jobs->{$job}->{ready};
7007                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7008                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7009                     my $remaining = $total - $transferred;
7010                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7011
7012                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7013                 }
7014
7015                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7016             }
7017
7018             last if scalar(keys %$jobs) == 0;
7019
7020             if ($readycounter == scalar(keys %$jobs)) {
7021                 print "all mirroring jobs are ready \n";
7022                 last if $skipcomplete; #do the complete later
7023
7024                 if ($vmiddst && $vmiddst != $vmid) {
7025                     my $agent_running = $qga && qga_check_running($vmid);
7026                     if ($agent_running) {
7027                         print "freeze filesystem\n";
7028                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7029                     } else {
7030                         print "suspend vm\n";
7031                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7032                     }
7033
7034                     # if we clone a disk for a new target vm, we don't switch the disk
7035                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7036
7037                     if ($agent_running) {
7038                         print "unfreeze filesystem\n";
7039                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7040                     } else {
7041                         print "resume vm\n";
7042                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7043                     }
7044
7045                     last;
7046                 } else {
7047
7048                     foreach my $job (keys %$jobs) {
7049                         # try to switch the disk if source and destination are on the same guest
7050                         print "$job: Completing block job...\n";
7051
7052                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7053                         if ($@ =~ m/cannot be completed/) {
7054                             print "$job: Block job cannot be completed, try again.\n";
7055                             $err_complete++;
7056                         }else {
7057                             print "$job: Completed successfully.\n";
7058                             $jobs->{$job}->{complete} = 1;
7059                         }
7060                     }
7061                 }
7062             }
7063             sleep 1;
7064         }
7065     };
7066     my $err = $@;
7067
7068     if ($err) {
7069         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7070         die "mirroring error: $err";
7071     }
7072
7073 }
7074
7075 sub qemu_blockjobs_cancel {
7076     my ($vmid, $jobs) = @_;
7077
7078     foreach my $job (keys %$jobs) {
7079         print "$job: Cancelling block job\n";
7080         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7081         $jobs->{$job}->{cancel} = 1;
7082     }
7083
7084     while (1) {
7085         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7086
7087         my $running_jobs = {};
7088         foreach my $stat (@$stats) {
7089             $running_jobs->{$stat->{device}} = $stat;
7090         }
7091
7092         foreach my $job (keys %$jobs) {
7093
7094             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7095                 print "$job: Done.\n";
7096                 delete $jobs->{$job};
7097             }
7098         }
7099
7100         last if scalar(keys %$jobs) == 0;
7101
7102         sleep 1;
7103     }
7104 }
7105
7106 sub clone_disk {
7107     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7108         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7109
7110     my $newvolid;
7111
7112     if (!$full) {
7113         print "create linked clone of drive $drivename ($drive->{file})\n";
7114         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7115         push @$newvollist, $newvolid;
7116     } else {
7117
7118         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7119         $storeid = $storage if $storage;
7120
7121         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7122         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7123
7124         print "create full clone of drive $drivename ($drive->{file})\n";
7125         my $name = undef;
7126         if (drive_is_cloudinit($drive)) {
7127             $name = "vm-$newvmid-cloudinit";
7128             $snapname = undef;
7129             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7130             if ($dst_format ne 'raw') {
7131                 $name .= ".$dst_format";
7132             }
7133         }
7134         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7135         push @$newvollist, $newvolid;
7136
7137         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7138
7139         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7140         if (!$running || $snapname) {
7141             # TODO: handle bwlimits
7142             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7143         } else {
7144
7145             my $kvmver = get_running_qemu_version ($vmid);
7146             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7147                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7148                     if $drive->{iothread};
7149             }
7150
7151             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7152         }
7153     }
7154
7155     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7156
7157     my $disk = $drive;
7158     $disk->{format} = undef;
7159     $disk->{file} = $newvolid;
7160     $disk->{size} = $size;
7161
7162     return $disk;
7163 }
7164
7165 # this only works if VM is running
7166 sub get_current_qemu_machine {
7167     my ($vmid) = @_;
7168
7169     my $cmd = { execute => 'query-machines', arguments => {} };
7170     my $res = vm_qmp_command($vmid, $cmd);
7171
7172     my ($current, $default);
7173     foreach my $e (@$res) {
7174         $default = $e->{name} if $e->{'is-default'};
7175         $current = $e->{name} if $e->{'is-current'};
7176     }
7177
7178     # fallback to the default machine if current is not supported by qemu
7179     return $current || $default || 'pc';
7180 }
7181
7182 sub get_running_qemu_version {
7183     my ($vmid) = @_;
7184     my $cmd = { execute => 'query-version', arguments => {} };
7185     my $res = vm_qmp_command($vmid, $cmd);
7186     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7187 }
7188
7189 sub qemu_machine_feature_enabled {
7190     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7191
7192     my $current_major;
7193     my $current_minor;
7194
7195     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7196
7197         $current_major = $3;
7198         $current_minor = $4;
7199
7200     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7201
7202         $current_major = $1;
7203         $current_minor = $2;
7204     }
7205
7206     return 1 if $current_major > $version_major ||
7207                 ($current_major == $version_major &&
7208                  $current_minor >= $version_minor);
7209 }
7210
7211 sub qemu_machine_pxe {
7212     my ($vmid, $conf) = @_;
7213
7214     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7215
7216     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7217         $machine .= '.pxe';
7218     }
7219
7220     return $machine;
7221 }
7222
7223 sub qemu_use_old_bios_files {
7224     my ($machine_type) = @_;
7225
7226     return if !$machine_type;
7227
7228     my $use_old_bios_files = undef;
7229
7230     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7231         $machine_type = $1;
7232         $use_old_bios_files = 1;
7233     } else {
7234         my $kvmver = kvm_user_version();
7235         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7236         # load new efi bios files on migration. So this hack is required to allow
7237         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7238         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7239         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7240     }
7241
7242     return ($use_old_bios_files, $machine_type);
7243 }
7244
7245 sub create_efidisk($$$$$) {
7246     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7247
7248     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7249     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7250
7251     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7252     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7253     PVE::Storage::activate_volumes($storecfg, [$volid]);
7254
7255     my $path = PVE::Storage::path($storecfg, $volid);
7256     eval {
7257         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7258     };
7259     die "Copying EFI vars image failed: $@" if $@;
7260
7261     return ($volid, $vars_size);
7262 }
7263
7264 sub vm_iothreads_list {
7265     my ($vmid) = @_;
7266
7267     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7268
7269     my $iothreads = {};
7270     foreach my $iothread (@$res) {
7271         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7272     }
7273
7274     return $iothreads;
7275 }
7276
7277 sub scsihw_infos {
7278     my ($conf, $drive) = @_;
7279
7280     my $maxdev = 0;
7281
7282     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7283         $maxdev = 7;
7284     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7285         $maxdev = 1;
7286     } else {
7287         $maxdev = 256;
7288     }
7289
7290     my $controller = int($drive->{index} / $maxdev);
7291     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7292
7293     return ($maxdev, $controller, $controller_prefix);
7294 }
7295
7296 sub add_hyperv_enlightenments {
7297     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7298
7299     return if $winversion < 6;
7300     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7301
7302     if ($gpu_passthrough || defined($hv_vendor_id)) {
7303         $hv_vendor_id //= 'proxmox';
7304         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7305     }
7306
7307     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7308         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7309         push @$cpuFlags , 'hv_vapic';
7310         push @$cpuFlags , 'hv_time';
7311     } else {
7312         push @$cpuFlags , 'hv_spinlocks=0xffff';
7313     }
7314
7315     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7316         push @$cpuFlags , 'hv_reset';
7317         push @$cpuFlags , 'hv_vpindex';
7318         push @$cpuFlags , 'hv_runtime';
7319     }
7320
7321     if ($winversion >= 7) {
7322         push @$cpuFlags , 'hv_relaxed';
7323
7324         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7325             push @$cpuFlags , 'hv_synic';
7326             push @$cpuFlags , 'hv_stimer';
7327         }
7328
7329         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7330             push @$cpuFlags , 'hv_ipi';
7331         }
7332     }
7333 }
7334
7335 sub windows_version {
7336     my ($ostype) = @_;
7337
7338     return 0 if !$ostype;
7339
7340     my $winversion = 0;
7341
7342     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7343         $winversion = 5;
7344     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7345         $winversion = 6;
7346     } elsif ($ostype =~ m/^win(\d+)$/) {
7347         $winversion = $1;
7348     }
7349
7350     return $winversion;
7351 }
7352
7353 sub resolve_dst_disk_format {
7354         my ($storecfg, $storeid, $src_volname, $format) = @_;
7355         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7356
7357         if (!$format) {
7358             # if no target format is specified, use the source disk format as hint
7359             if ($src_volname) {
7360                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7361                 $format = qemu_img_format($scfg, $src_volname);
7362             } else {
7363                 return $defFormat;
7364             }
7365         }
7366
7367         # test if requested format is supported - else use default
7368         my $supported = grep { $_ eq $format } @$validFormats;
7369         $format = $defFormat if !$supported;
7370         return $format;
7371 }
7372
7373 sub resolve_first_disk {
7374     my $conf = shift;
7375     my @disks = PVE::QemuServer::valid_drive_names();
7376     my $firstdisk;
7377     foreach my $ds (reverse @disks) {
7378         next if !$conf->{$ds};
7379         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7380         next if PVE::QemuServer::drive_is_cdrom($disk);
7381         $firstdisk = $ds;
7382     }
7383     return $firstdisk;
7384 }
7385
7386 sub generate_uuid {
7387     my ($uuid, $uuid_str);
7388     UUID::generate($uuid);
7389     UUID::unparse($uuid, $uuid_str);
7390     return $uuid_str;
7391 }
7392
7393 sub generate_smbios1_uuid {
7394     return "uuid=".generate_uuid();
7395 }
7396
7397 sub nbd_stop {
7398     my ($vmid) = @_;
7399
7400     vm_mon_cmd($vmid, 'nbd-server-stop');
7401 }
7402
7403 # bash completion helper
7404
7405 sub complete_backup_archives {
7406     my ($cmdname, $pname, $cvalue) = @_;
7407
7408     my $cfg = PVE::Storage::config();
7409
7410     my $storeid;
7411
7412     if ($cvalue =~ m/^([^:]+):/) {
7413         $storeid = $1;
7414     }
7415
7416     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7417
7418     my $res = [];
7419     foreach my $id (keys %$data) {
7420         foreach my $item (@{$data->{$id}}) {
7421             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7422             push @$res, $item->{volid} if defined($item->{volid});
7423         }
7424     }
7425
7426     return $res;
7427 }
7428
7429 my $complete_vmid_full = sub {
7430     my ($running) = @_;
7431
7432     my $idlist = vmstatus();
7433
7434     my $res = [];
7435
7436     foreach my $id (keys %$idlist) {
7437         my $d = $idlist->{$id};
7438         if (defined($running)) {
7439             next if $d->{template};
7440             next if $running && $d->{status} ne 'running';
7441             next if !$running && $d->{status} eq 'running';
7442         }
7443         push @$res, $id;
7444
7445     }
7446     return $res;
7447 };
7448
7449 sub complete_vmid {
7450     return &$complete_vmid_full();
7451 }
7452
7453 sub complete_vmid_stopped {
7454     return &$complete_vmid_full(0);
7455 }
7456
7457 sub complete_vmid_running {
7458     return &$complete_vmid_full(1);
7459 }
7460
7461 sub complete_storage {
7462
7463     my $cfg = PVE::Storage::config();
7464     my $ids = $cfg->{ids};
7465
7466     my $res = [];
7467     foreach my $sid (keys %$ids) {
7468         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7469         next if !$ids->{$sid}->{content}->{images};
7470         push @$res, $sid;
7471     }
7472
7473     return $res;
7474 }
7475
7476 1;