PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use POSIX;
   7 use IO::Handle;
   8 use IO::Select;
   9 use IO::File;
  10 use IO::Dir;
  11 use IO::Socket::UNIX;
  12 use File::Basename;
  13 use File::Path;
  14 use File::stat;
  15 use Getopt::Long;
  16 use Digest::SHA;
  17 use Fcntl ':flock';
  18 use Cwd 'abs_path';
  19 use IPC::Open3;
  20 use JSON;
  21 use Fcntl;
  22 use PVE::SafeSyslog;
  23 use Storable qw(dclone);
  24 use MIME::Base64;
  25 use PVE::Exception qw(raise raise_param_exc);
  26 use PVE::Storage;
  27 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  28 use PVE::JSONSchema qw(get_standard_option);
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::INotify;
  31 use PVE::ProcFSTools;
  32 use PVE::QemuConfig;
  33 use PVE::QMPClient;
  34 use PVE::RPCEnvironment;
  35 use PVE::GuestHelpers;
  36 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  37 use PVE::QemuServer::Memory;
  38 use PVE::QemuServer::USB qw(parse_usb_device);
  39 use PVE::QemuServer::Cloudinit;
  40 use PVE::SysFSTools;
  41 use PVE::Systemd;
  42 use Time::HiRes qw(gettimeofday);
  43 use File::Copy qw(copy);
  44 use URI::Escape;
  45
  46 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  47 my $OVMF = {
  48     x86_64 => [
  49         "$EDK2_FW_BASE/OVMF_CODE.fd",
  50         "$EDK2_FW_BASE/OVMF_VARS.fd"
  51     ],
  52     aarch64 => [
  53         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  54         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  55     ],
  56 };
  57
  58 my $qemu_snap_storage = { rbd => 1 };
  59
  60 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  61
  62 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  63
  64 # Note about locking: we use flock on the config file protect
  65 # against concurent actions.
  66 # Aditionaly, we have a 'lock' setting in the config file. This
  67 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  68 # allowed when such lock is set. But you can ignore this kind of
  69 # lock with the --skiplock flag.
  70
  71 cfs_register_file('/qemu-server/',
  72                   \&parse_vm_config,
  73                   \&write_vm_config);
  74
  75 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  76     description => "Some command save/restore state from this location.",
  77     type => 'string',
  78     maxLength => 128,
  79     optional => 1,
  80 });
  81
  82 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  83     type => 'string',
  84     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  85     description => "The drive's backing file's data format.",
  86     optional => 1,
  87 });
  88
  89 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  90         description => "Specifies the Qemu machine type.",
  91         type => 'string',
  92         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?|virt(?:-\d+\.\d+)?)',
  93         maxLength => 40,
  94         optional => 1,
  95 });
  96
  97 #no warnings 'redefine';
  98
  99 sub cgroups_write {
 100    my ($controller, $vmid, $option, $value) = @_;
 101
 102    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 103    PVE::ProcFSTools::write_proc_entry($path, $value);
 104
 105 }
 106
 107 my $nodename = PVE::INotify::nodename();
 108
 109 mkdir "/etc/pve/nodes/$nodename";
 110 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 111 mkdir $confdir;
 112
 113 my $var_run_tmpdir = "/var/run/qemu-server";
 114 mkdir $var_run_tmpdir;
 115
 116 my $lock_dir = "/var/lock/qemu-server";
 117 mkdir $lock_dir;
 118
 119 my $cpu_vendor_list = {
 120     # Intel CPUs
 121     486 => 'GenuineIntel',
 122     pentium => 'GenuineIntel',
 123     pentium2  => 'GenuineIntel',
 124     pentium3  => 'GenuineIntel',
 125     coreduo => 'GenuineIntel',
 126     core2duo => 'GenuineIntel',
 127     Conroe  => 'GenuineIntel',
 128     Penryn  => 'GenuineIntel',
 129     Nehalem  => 'GenuineIntel',
 130     'Nehalem-IBRS'  => 'GenuineIntel',
 131     Westmere => 'GenuineIntel',
 132     'Westmere-IBRS' => 'GenuineIntel',
 133     SandyBridge => 'GenuineIntel',
 134     'SandyBridge-IBRS' => 'GenuineIntel',
 135     IvyBridge => 'GenuineIntel',
 136     'IvyBridge-IBRS' => 'GenuineIntel',
 137     Haswell => 'GenuineIntel',
 138     'Haswell-IBRS' => 'GenuineIntel',
 139     'Haswell-noTSX' => 'GenuineIntel',
 140     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 141     Broadwell => 'GenuineIntel',
 142     'Broadwell-IBRS' => 'GenuineIntel',
 143     'Broadwell-noTSX' => 'GenuineIntel',
 144     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 145     'Skylake-Client' => 'GenuineIntel',
 146     'Skylake-Client-IBRS' => 'GenuineIntel',
 147     'Skylake-Server' => 'GenuineIntel',
 148     'Skylake-Server-IBRS' => 'GenuineIntel',
 149
 150     # AMD CPUs
 151     athlon => 'AuthenticAMD',
 152     phenom  => 'AuthenticAMD',
 153     Opteron_G1  => 'AuthenticAMD',
 154     Opteron_G2  => 'AuthenticAMD',
 155     Opteron_G3  => 'AuthenticAMD',
 156     Opteron_G4  => 'AuthenticAMD',
 157     Opteron_G5  => 'AuthenticAMD',
 158     EPYC => 'AuthenticAMD',
 159     'EPYC-IBPB' => 'AuthenticAMD',
 160
 161     # generic types, use vendor from host node
 162     host => 'default',
 163     kvm32 => 'default',
 164     kvm64 => 'default',
 165     qemu32 => 'default',
 166     qemu64 => 'default',
 167     max => 'default',
 168 };
 169
 170 my @supported_cpu_flags = (
 171     'pcid',
 172     'spec-ctrl',
 173     'ibpb',
 174     'ssbd',
 175     'virt-ssbd',
 176     'amd-ssbd',
 177     'amd-no-ssb',
 178     'pdpe1gb',
 179     'md-clear',
 180     'hv-tlbflush',
 181     'hv-evmcs',
 182     'aes'
 183 );
 184 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 185
 186 my $cpu_fmt = {
 187     cputype => {
 188         description => "Emulated CPU type.",
 189         type => 'string',
 190         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 191         default => 'kvm64',
 192         default_key => 1,
 193     },
 194     hidden => {
 195         description => "Do not identify as a KVM virtual machine.",
 196         type => 'boolean',
 197         optional => 1,
 198         default => 0
 199     },
 200     'hv-vendor-id' => {
 201         type => 'string',
 202         pattern => qr/[a-zA-Z0-9]{1,12}/,
 203         format_description => 'vendor-id',
 204         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 205         optional => 1,
 206     },
 207     flags => {
 208         description => "List of additional CPU flags separated by ';'."
 209                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 210                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 211         format_description => '+FLAG[;-FLAG...]',
 212         type => 'string',
 213         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 214         optional => 1,
 215     },
 216 };
 217
 218 my $watchdog_fmt = {
 219     model => {
 220         default_key => 1,
 221         type => 'string',
 222         enum => [qw(i6300esb ib700)],
 223         description => "Watchdog type to emulate.",
 224         default => 'i6300esb',
 225         optional => 1,
 226     },
 227     action => {
 228         type => 'string',
 229         enum => [qw(reset shutdown poweroff pause debug none)],
 230         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 231         optional => 1,
 232     },
 233 };
 234 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 235
 236 my $agent_fmt = {
 237     enabled => {
 238         description => "Enable/disable Qemu GuestAgent.",
 239         type => 'boolean',
 240         default => 0,
 241         default_key => 1,
 242     },
 243     fstrim_cloned_disks => {
 244         description => "Run fstrim after cloning/moving a disk.",
 245         type => 'boolean',
 246         optional => 1,
 247         default => 0
 248     },
 249 };
 250
 251 my $vga_fmt = {
 252     type => {
 253         description => "Select the VGA type.",
 254         type => 'string',
 255         default => 'std',
 256         optional => 1,
 257         default_key => 1,
 258         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 259     },
 260     memory => {
 261         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 262         type => 'integer',
 263         optional => 1,
 264         minimum => 4,
 265         maximum => 512,
 266     },
 267 };
 268
 269 my $ivshmem_fmt = {
 270     size => {
 271         type => 'integer',
 272         minimum => 1,
 273         description => "The size of the file in MB.",
 274     },
 275     name => {
 276         type => 'string',
 277         pattern => '[a-zA-Z0-9\-]+',
 278         optional => 1,
 279         format_description => 'string',
 280         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 281     },
 282 };
 283
 284 my $audio_fmt = {
 285     device => {
 286         type => 'string',
 287         enum => [qw(ich9-intel-hda intel-hda AC97)],
 288         description =>  "Configure an audio device."
 289     },
 290     driver =>  {
 291         type => 'string',
 292         enum => ['spice'],
 293         default => 'spice',
 294         optional => 1,
 295         description => "Driver backend for the audio device."
 296     },
 297 };
 298
 299 my $spice_enhancements_fmt = {
 300     foldersharing => {
 301         type => 'boolean',
 302         optional => 1,
 303         default => '0',
 304         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 305     },
 306     videostreaming =>  {
 307         type => 'string',
 308         enum => ['off', 'all', 'filter'],
 309         default => 'off',
 310         optional => 1,
 311         description => "Enable video streaming. Uses compression for detected video streams."
 312     },
 313 };
 314
 315 my $confdesc = {
 316     onboot => {
 317         optional => 1,
 318         type => 'boolean',
 319         description => "Specifies whether a VM will be started during system bootup.",
 320         default => 0,
 321     },
 322     autostart => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Automatic restart after crash (currently ignored).",
 326         default => 0,
 327     },
 328     hotplug => {
 329         optional => 1,
 330         type => 'string', format => 'pve-hotplug-features',
 331         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 332         default => 'network,disk,usb',
 333     },
 334     reboot => {
 335         optional => 1,
 336         type => 'boolean',
 337         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 338         default => 1,
 339     },
 340     lock => {
 341         optional => 1,
 342         type => 'string',
 343         description => "Lock/unlock the VM.",
 344         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 345     },
 346     cpulimit => {
 347         optional => 1,
 348         type => 'number',
 349         description => "Limit of CPU usage.",
 350         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 351         minimum => 0,
 352         maximum => 128,
 353         default => 0,
 354     },
 355     cpuunits => {
 356         optional => 1,
 357         type => 'integer',
 358         description => "CPU weight for a VM.",
 359         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 360         minimum => 2,
 361         maximum => 262144,
 362         default => 1024,
 363     },
 364     memory => {
 365         optional => 1,
 366         type => 'integer',
 367         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 368         minimum => 16,
 369         default => 512,
 370     },
 371     balloon => {
 372         optional => 1,
 373         type => 'integer',
 374         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 375         minimum => 0,
 376     },
 377     shares => {
 378         optional => 1,
 379         type => 'integer',
 380         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 381         minimum => 0,
 382         maximum => 50000,
 383         default => 1000,
 384     },
 385     keyboard => {
 386         optional => 1,
 387         type => 'string',
 388         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 389                        "It should not be necessary to set it.",
 390         enum => PVE::Tools::kvmkeymaplist(),
 391         default => undef,
 392     },
 393     name => {
 394         optional => 1,
 395         type => 'string', format => 'dns-name',
 396         description => "Set a name for the VM. Only used on the configuration web interface.",
 397     },
 398     scsihw => {
 399         optional => 1,
 400         type => 'string',
 401         description => "SCSI controller model",
 402         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 403         default => 'lsi',
 404     },
 405     description => {
 406         optional => 1,
 407         type => 'string',
 408         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 409     },
 410     ostype => {
 411         optional => 1,
 412         type => 'string',
 413         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 414         description => "Specify guest operating system.",
 415         verbose_description => <<EODESC,
 416 Specify guest operating system. This is used to enable special
 417 optimization/features for specific operating systems:
 418
 419 [horizontal]
 420 other;; unspecified OS
 421 wxp;; Microsoft Windows XP
 422 w2k;; Microsoft Windows 2000
 423 w2k3;; Microsoft Windows 2003
 424 w2k8;; Microsoft Windows 2008
 425 wvista;; Microsoft Windows Vista
 426 win7;; Microsoft Windows 7
 427 win8;; Microsoft Windows 8/2012/2012r2
 428 win10;; Microsoft Windows 10/2016
 429 l24;; Linux 2.4 Kernel
 430 l26;; Linux 2.6/3.X Kernel
 431 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 432 EODESC
 433     },
 434     boot => {
 435         optional => 1,
 436         type => 'string',
 437         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 438         pattern => '[acdn]{1,4}',
 439         default => 'cdn',
 440     },
 441     bootdisk => {
 442         optional => 1,
 443         type => 'string', format => 'pve-qm-bootdisk',
 444         description => "Enable booting from specified disk.",
 445         pattern => '(ide|sata|scsi|virtio)\d+',
 446     },
 447     smp => {
 448         optional => 1,
 449         type => 'integer',
 450         description => "The number of CPUs. Please use option -sockets instead.",
 451         minimum => 1,
 452         default => 1,
 453     },
 454     sockets => {
 455         optional => 1,
 456         type => 'integer',
 457         description => "The number of CPU sockets.",
 458         minimum => 1,
 459         default => 1,
 460     },
 461     cores => {
 462         optional => 1,
 463         type => 'integer',
 464         description => "The number of cores per socket.",
 465         minimum => 1,
 466         default => 1,
 467     },
 468     numa => {
 469         optional => 1,
 470         type => 'boolean',
 471         description => "Enable/disable NUMA.",
 472         default => 0,
 473     },
 474     hugepages => {
 475         optional => 1,
 476         type => 'string',
 477         description => "Enable/disable hugepages memory.",
 478         enum => [qw(any 2 1024)],
 479     },
 480     vcpus => {
 481         optional => 1,
 482         type => 'integer',
 483         description => "Number of hotplugged vcpus.",
 484         minimum => 1,
 485         default => 0,
 486     },
 487     acpi => {
 488         optional => 1,
 489         type => 'boolean',
 490         description => "Enable/disable ACPI.",
 491         default => 1,
 492     },
 493     agent => {
 494         optional => 1,
 495         description => "Enable/disable Qemu GuestAgent and its properties.",
 496         type => 'string',
 497         format => $agent_fmt,
 498     },
 499     kvm => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Enable/disable KVM hardware virtualization.",
 503         default => 1,
 504     },
 505     tdf => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable time drift fix.",
 509         default => 0,
 510     },
 511     localtime => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 515     },
 516     freeze => {
 517         optional => 1,
 518         type => 'boolean',
 519         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 520     },
 521     vga => {
 522         optional => 1,
 523         type => 'string', format => $vga_fmt,
 524         description => "Configure the VGA hardware.",
 525         verbose_description => "Configure the VGA Hardware. If you want to use ".
 526             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 527             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 528             "is 'std' for all OS types besides some Windows versions (XP and " .
 529             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 530             "display server. For win* OS you can select how many independent " .
 531             "displays you want, Linux guests can add displays them self.\n".
 532             "You can also run without any graphic card, using a serial device as terminal.",
 533     },
 534     watchdog => {
 535         optional => 1,
 536         type => 'string', format => 'pve-qm-watchdog',
 537         description => "Create a virtual hardware watchdog device.",
 538         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 539             " (by a guest action), the watchdog must be periodically polled " .
 540             "by an agent inside the guest or else the watchdog will reset " .
 541             "the guest (or execute the respective action specified)",
 542     },
 543     startdate => {
 544         optional => 1,
 545         type => 'string',
 546         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 547         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 548         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 549         default => 'now',
 550     },
 551     startup =>  get_standard_option('pve-startup-order'),
 552     template => {
 553         optional => 1,
 554         type => 'boolean',
 555         description => "Enable/disable Template.",
 556         default => 0,
 557     },
 558     args => {
 559         optional => 1,
 560         type => 'string',
 561         description => "Arbitrary arguments passed to kvm.",
 562         verbose_description => <<EODESCR,
 563 Arbitrary arguments passed to kvm, for example:
 564
 565 args: -no-reboot -no-hpet
 566
 567 NOTE: this option is for experts only.
 568 EODESCR
 569     },
 570     tablet => {
 571         optional => 1,
 572         type => 'boolean',
 573         default => 1,
 574         description => "Enable/disable the USB tablet device.",
 575         verbose_description => "Enable/disable the USB tablet device. This device is " .
 576             "usually needed to allow absolute mouse positioning with VNC. " .
 577             "Else the mouse runs out of sync with normal VNC clients. " .
 578             "If you're running lots of console-only guests on one host, " .
 579             "you may consider disabling this to save some context switches. " .
 580             "This is turned off by default if you use spice (-vga=qxl).",
 581     },
 582     migrate_speed => {
 583         optional => 1,
 584         type => 'integer',
 585         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 586         minimum => 0,
 587         default => 0,
 588     },
 589     migrate_downtime => {
 590         optional => 1,
 591         type => 'number',
 592         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 593         minimum => 0,
 594         default => 0.1,
 595     },
 596     cdrom => {
 597         optional => 1,
 598         type => 'string', format => 'pve-qm-ide',
 599         typetext => '<volume>',
 600         description => "This is an alias for option -ide2",
 601     },
 602     cpu => {
 603         optional => 1,
 604         description => "Emulated CPU type.",
 605         type => 'string',
 606         format => $cpu_fmt,
 607     },
 608     parent => get_standard_option('pve-snapshot-name', {
 609         optional => 1,
 610         description => "Parent snapshot name. This is used internally, and should not be modified.",
 611     }),
 612     snaptime => {
 613         optional => 1,
 614         description => "Timestamp for snapshots.",
 615         type => 'integer',
 616         minimum => 0,
 617     },
 618     vmstate => {
 619         optional => 1,
 620         type => 'string', format => 'pve-volume-id',
 621         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 622     },
 623     vmstatestorage => get_standard_option('pve-storage-id', {
 624         description => "Default storage for VM state volumes/files.",
 625         optional => 1,
 626     }),
 627     runningmachine => get_standard_option('pve-qemu-machine', {
 628         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 629     }),
 630     machine => get_standard_option('pve-qemu-machine'),
 631     arch => {
 632         description => "Virtual processor architecture. Defaults to the host.",
 633         optional => 1,
 634         type => 'string',
 635         enum => [qw(x86_64 aarch64)],
 636     },
 637     smbios1 => {
 638         description => "Specify SMBIOS type 1 fields.",
 639         type => 'string', format => 'pve-qm-smbios1',
 640         maxLength => 512,
 641         optional => 1,
 642     },
 643     protection => {
 644         optional => 1,
 645         type => 'boolean',
 646         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 647         default => 0,
 648     },
 649     bios => {
 650         optional => 1,
 651         type => 'string',
 652         enum => [ qw(seabios ovmf) ],
 653         description => "Select BIOS implementation.",
 654         default => 'seabios',
 655     },
 656     vmgenid => {
 657         type => 'string',
 658         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 659         format_description => 'UUID',
 660         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 661         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 662             " 128-bit integer value identifier to the guest OS. This allows to".
 663             " notify the guest operating system when the virtual machine is".
 664             " executed with a different configuration (e.g. snapshot execution".
 665             " or creation from a template). The guest operating system notices".
 666             " the change, and is then able to react as appropriate by marking".
 667             " its copies of distributed databases as dirty, re-initializing its".
 668             " random number generator, etc.\n".
 669             "Note that auto-creation only works when done throug API/CLI create".
 670             " or update methods, but not when manually editing the config file.",
 671         default => "1 (autogenerated)",
 672         optional => 1,
 673     },
 674     hookscript => {
 675         type => 'string',
 676         format => 'pve-volume-id',
 677         optional => 1,
 678         description => "Script that will be executed during various steps in the vms lifetime.",
 679     },
 680     ivshmem => {
 681         type => 'string',
 682         format => $ivshmem_fmt,
 683         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 684         optional => 1,
 685     },
 686     audio0 => {
 687         type => 'string',
 688         format => $audio_fmt,
 689         description => "Configure a audio device, useful in combination with QXL/Spice.",
 690         optional => 1
 691     },
 692     spice_enhancements => {
 693         type => 'string',
 694         format => $spice_enhancements_fmt,
 695         description => "Configure additional enhancements for SPICE.",
 696         optional => 1
 697     },
 698 };
 699
 700 my $cicustom_fmt = {
 701     meta => {
 702         type => 'string',
 703         optional => 1,
 704         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 705         format => 'pve-volume-id',
 706         format_description => 'volume',
 707     },
 708     network => {
 709         type => 'string',
 710         optional => 1,
 711         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 712         format => 'pve-volume-id',
 713         format_description => 'volume',
 714     },
 715     user => {
 716         type => 'string',
 717         optional => 1,
 718         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 719         format => 'pve-volume-id',
 720         format_description => 'volume',
 721     },
 722 };
 723 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 724
 725 my $confdesc_cloudinit = {
 726     citype => {
 727         optional => 1,
 728         type => 'string',
 729         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 730         enum => ['configdrive2', 'nocloud'],
 731     },
 732     ciuser => {
 733         optional => 1,
 734         type => 'string',
 735         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 736     },
 737     cipassword => {
 738         optional => 1,
 739         type => 'string',
 740         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 741     },
 742     cicustom => {
 743         optional => 1,
 744         type => 'string',
 745         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 746         format => 'pve-qm-cicustom',
 747     },
 748     searchdomain => {
 749         optional => 1,
 750         type => 'string',
 751         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 752     },
 753     nameserver => {
 754         optional => 1,
 755         type => 'string', format => 'address-list',
 756         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 757     },
 758     sshkeys => {
 759         optional => 1,
 760         type => 'string',
 761         format => 'urlencoded',
 762         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 763     },
 764 };
 765
 766 # what about other qemu settings ?
 767 #cpu => 'string',
 768 #machine => 'string',
 769 #fda => 'file',
 770 #fdb => 'file',
 771 #mtdblock => 'file',
 772 #sd => 'file',
 773 #pflash => 'file',
 774 #snapshot => 'bool',
 775 #bootp => 'file',
 776 ##tftp => 'dir',
 777 ##smb => 'dir',
 778 #kernel => 'file',
 779 #append => 'string',
 780 #initrd => 'file',
 781 ##soundhw => 'string',
 782
 783 while (my ($k, $v) = each %$confdesc) {
 784     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 785 }
 786
 787 my $MAX_IDE_DISKS = 4;
 788 my $MAX_SCSI_DISKS = 14;
 789 my $MAX_VIRTIO_DISKS = 16;
 790 my $MAX_SATA_DISKS = 6;
 791 my $MAX_USB_DEVICES = 5;
 792 my $MAX_NETS = 32;
 793 my $MAX_UNUSED_DISKS = 256;
 794 my $MAX_HOSTPCI_DEVICES = 4;
 795 my $MAX_SERIAL_PORTS = 4;
 796 my $MAX_PARALLEL_PORTS = 3;
 797 my $MAX_NUMA = 8;
 798
 799 my $numa_fmt = {
 800     cpus => {
 801         type => "string",
 802         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 803         description => "CPUs accessing this NUMA node.",
 804         format_description => "id[-id];...",
 805     },
 806     memory => {
 807         type => "number",
 808         description => "Amount of memory this NUMA node provides.",
 809         optional => 1,
 810     },
 811     hostnodes => {
 812         type => "string",
 813         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 814         description => "Host NUMA nodes to use.",
 815         format_description => "id[-id];...",
 816         optional => 1,
 817     },
 818     policy => {
 819         type => 'string',
 820         enum => [qw(preferred bind interleave)],
 821         description => "NUMA allocation policy.",
 822         optional => 1,
 823     },
 824 };
 825 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 826 my $numadesc = {
 827     optional => 1,
 828     type => 'string', format => $numa_fmt,
 829     description => "NUMA topology.",
 830 };
 831 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 832
 833 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 834     $confdesc->{"numa$i"} = $numadesc;
 835 }
 836
 837 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 838                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 839                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 840 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 841
 842 my $net_fmt_bridge_descr = <<__EOD__;
 843 Bridge to attach the network device to. The Proxmox VE standard bridge
 844 is called 'vmbr0'.
 845
 846 If you do not specify a bridge, we create a kvm user (NATed) network
 847 device, which provides DHCP and DNS services. The following addresses
 848 are used:
 849
 850  10.0.2.2   Gateway
 851  10.0.2.3   DNS Server
 852  10.0.2.4   SMB Server
 853
 854 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 855 __EOD__
 856
 857 my $net_fmt = {
 858     macaddr  => get_standard_option('mac-addr', {
 859         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 860     }),
 861     model => {
 862         type => 'string',
 863         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 864         enum => $nic_model_list,
 865         default_key => 1,
 866     },
 867     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 868     bridge => {
 869         type => 'string',
 870         description => $net_fmt_bridge_descr,
 871         format_description => 'bridge',
 872         optional => 1,
 873     },
 874     queues => {
 875         type => 'integer',
 876         minimum => 0, maximum => 16,
 877         description => 'Number of packet queues to be used on the device.',
 878         optional => 1,
 879     },
 880     rate => {
 881         type => 'number',
 882         minimum => 0,
 883         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 884         optional => 1,
 885     },
 886     tag => {
 887         type => 'integer',
 888         minimum => 1, maximum => 4094,
 889         description => 'VLAN tag to apply to packets on this interface.',
 890         optional => 1,
 891     },
 892     trunks => {
 893         type => 'string',
 894         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 895         description => 'VLAN trunks to pass through this interface.',
 896         format_description => 'vlanid[;vlanid...]',
 897         optional => 1,
 898     },
 899     firewall => {
 900         type => 'boolean',
 901         description => 'Whether this interface should be protected by the firewall.',
 902         optional => 1,
 903     },
 904     link_down => {
 905         type => 'boolean',
 906         description => 'Whether this interface should be disconnected (like pulling the plug).',
 907         optional => 1,
 908     },
 909 };
 910
 911 my $netdesc = {
 912     optional => 1,
 913     type => 'string', format => $net_fmt,
 914     description => "Specify network devices.",
 915 };
 916
 917 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 918
 919 my $ipconfig_fmt = {
 920     ip => {
 921         type => 'string',
 922         format => 'pve-ipv4-config',
 923         format_description => 'IPv4Format/CIDR',
 924         description => 'IPv4 address in CIDR format.',
 925         optional => 1,
 926         default => 'dhcp',
 927     },
 928     gw => {
 929         type => 'string',
 930         format => 'ipv4',
 931         format_description => 'GatewayIPv4',
 932         description => 'Default gateway for IPv4 traffic.',
 933         optional => 1,
 934         requires => 'ip',
 935     },
 936     ip6 => {
 937         type => 'string',
 938         format => 'pve-ipv6-config',
 939         format_description => 'IPv6Format/CIDR',
 940         description => 'IPv6 address in CIDR format.',
 941         optional => 1,
 942         default => 'dhcp',
 943     },
 944     gw6 => {
 945         type => 'string',
 946         format => 'ipv6',
 947         format_description => 'GatewayIPv6',
 948         description => 'Default gateway for IPv6 traffic.',
 949         optional => 1,
 950         requires => 'ip6',
 951     },
 952 };
 953 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 954 my $ipconfigdesc = {
 955     optional => 1,
 956     type => 'string', format => 'pve-qm-ipconfig',
 957     description => <<'EODESCR',
 958 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 959
 960 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 961
 962 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 963 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 964
 965 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 966 EODESCR
 967 };
 968 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 969
 970 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 971     $confdesc->{"net$i"} = $netdesc;
 972     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 973 }
 974
 975 foreach my $key (keys %$confdesc_cloudinit) {
 976     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 977 }
 978
 979 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 980 sub verify_volume_id_or_qm_path {
 981     my ($volid, $noerr) = @_;
 982
 983     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 984         return $volid;
 985     }
 986
 987     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 988     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 989     if ($@) {
 990         return undef if $noerr;
 991         die $@;
 992     }
 993     return $volid;
 994 }
 995
 996 my $drivename_hash;
 997
 998 my %drivedesc_base = (
 999     volume => { alias => 'file' },
1000     file => {
1001         type => 'string',
1002         format => 'pve-volume-id-or-qm-path',
1003         default_key => 1,
1004         format_description => 'volume',
1005         description => "The drive's backing volume.",
1006     },
1007     media => {
1008         type => 'string',
1009         enum => [qw(cdrom disk)],
1010         description => "The drive's media type.",
1011         default => 'disk',
1012         optional => 1
1013     },
1014     cyls => {
1015         type => 'integer',
1016         description => "Force the drive's physical geometry to have a specific cylinder count.",
1017         optional => 1
1018     },
1019     heads => {
1020         type => 'integer',
1021         description => "Force the drive's physical geometry to have a specific head count.",
1022         optional => 1
1023     },
1024     secs => {
1025         type => 'integer',
1026         description => "Force the drive's physical geometry to have a specific sector count.",
1027         optional => 1
1028     },
1029     trans => {
1030         type => 'string',
1031         enum => [qw(none lba auto)],
1032         description => "Force disk geometry bios translation mode.",
1033         optional => 1,
1034     },
1035     snapshot => {
1036         type => 'boolean',
1037         description => "Controls qemu's snapshot mode feature."
1038             . " If activated, changes made to the disk are temporary and will"
1039             . " be discarded when the VM is shutdown.",
1040         optional => 1,
1041     },
1042     cache => {
1043         type => 'string',
1044         enum => [qw(none writethrough writeback unsafe directsync)],
1045         description => "The drive's cache mode",
1046         optional => 1,
1047     },
1048     format => get_standard_option('pve-qm-image-format'),
1049     size => {
1050         type => 'string',
1051         format => 'disk-size',
1052         format_description => 'DiskSize',
1053         description => "Disk size. This is purely informational and has no effect.",
1054         optional => 1,
1055     },
1056     backup => {
1057         type => 'boolean',
1058         description => "Whether the drive should be included when making backups.",
1059         optional => 1,
1060     },
1061     replicate => {
1062         type => 'boolean',
1063         description => 'Whether the drive should considered for replication jobs.',
1064         optional => 1,
1065         default => 1,
1066     },
1067     rerror => {
1068         type => 'string',
1069         enum => [qw(ignore report stop)],
1070         description => 'Read error action.',
1071         optional => 1,
1072     },
1073     werror => {
1074         type => 'string',
1075         enum => [qw(enospc ignore report stop)],
1076         description => 'Write error action.',
1077         optional => 1,
1078     },
1079     aio => {
1080         type => 'string',
1081         enum => [qw(native threads)],
1082         description => 'AIO type to use.',
1083         optional => 1,
1084     },
1085     discard => {
1086         type => 'string',
1087         enum => [qw(ignore on)],
1088         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1089         optional => 1,
1090     },
1091     detect_zeroes => {
1092         type => 'boolean',
1093         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1094         optional => 1,
1095     },
1096     serial => {
1097         type => 'string',
1098         format => 'urlencoded',
1099         format_description => 'serial',
1100         maxLength => 20*3, # *3 since it's %xx url enoded
1101         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1102         optional => 1,
1103     },
1104     shared => {
1105         type => 'boolean',
1106         description => 'Mark this locally-managed volume as available on all nodes',
1107         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1108         optional => 1,
1109         default => 0,
1110     }
1111 );
1112
1113 my %iothread_fmt = ( iothread => {
1114         type => 'boolean',
1115         description => "Whether to use iothreads for this drive",
1116         optional => 1,
1117 });
1118
1119 my %model_fmt = (
1120     model => {
1121         type => 'string',
1122         format => 'urlencoded',
1123         format_description => 'model',
1124         maxLength => 40*3, # *3 since it's %xx url enoded
1125         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1126         optional => 1,
1127     },
1128 );
1129
1130 my %queues_fmt = (
1131     queues => {
1132         type => 'integer',
1133         description => "Number of queues.",
1134         minimum => 2,
1135         optional => 1
1136     }
1137 );
1138
1139 my %scsiblock_fmt = (
1140     scsiblock => {
1141         type => 'boolean',
1142         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1143         optional => 1,
1144         default => 0,
1145     },
1146 );
1147
1148 my %ssd_fmt = (
1149     ssd => {
1150         type => 'boolean',
1151         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1152         optional => 1,
1153     },
1154 );
1155
1156 my %wwn_fmt = (
1157     wwn => {
1158         type => 'string',
1159         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1160         format_description => 'wwn',
1161         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1162         optional => 1,
1163     },
1164 );
1165
1166 my $add_throttle_desc = sub {
1167     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1168     my $d = {
1169         type => $type,
1170         format_description => $unit,
1171         description => "Maximum $what in $longunit.",
1172         optional => 1,
1173     };
1174     $d->{minimum} = $minimum if defined($minimum);
1175     $drivedesc_base{$key} = $d;
1176 };
1177 # throughput: (leaky bucket)
1178 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1179 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1180 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1181 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1182 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1183 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1184 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1185 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1186 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1187
1188 # pools: (pool of IO before throttling starts taking effect)
1189 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1190 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1191 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1192 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1193 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1194 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1195
1196 # burst lengths
1197 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1198 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1199 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1200 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1201 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1202 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1203
1204 # legacy support
1205 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1206 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1207 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1208 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1209
1210 my $ide_fmt = {
1211     %drivedesc_base,
1212     %model_fmt,
1213     %ssd_fmt,
1214     %wwn_fmt,
1215 };
1216 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1217
1218 my $idedesc = {
1219     optional => 1,
1220     type => 'string', format => $ide_fmt,
1221     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1222 };
1223 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1224
1225 my $scsi_fmt = {
1226     %drivedesc_base,
1227     %iothread_fmt,
1228     %queues_fmt,
1229     %scsiblock_fmt,
1230     %ssd_fmt,
1231     %wwn_fmt,
1232 };
1233 my $scsidesc = {
1234     optional => 1,
1235     type => 'string', format => $scsi_fmt,
1236     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1237 };
1238 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1239
1240 my $sata_fmt = {
1241     %drivedesc_base,
1242     %ssd_fmt,
1243     %wwn_fmt,
1244 };
1245 my $satadesc = {
1246     optional => 1,
1247     type => 'string', format => $sata_fmt,
1248     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1249 };
1250 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1251
1252 my $virtio_fmt = {
1253     %drivedesc_base,
1254     %iothread_fmt,
1255 };
1256 my $virtiodesc = {
1257     optional => 1,
1258     type => 'string', format => $virtio_fmt,
1259     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1260 };
1261 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1262
1263 my $alldrive_fmt = {
1264     %drivedesc_base,
1265     %iothread_fmt,
1266     %model_fmt,
1267     %queues_fmt,
1268     %scsiblock_fmt,
1269     %ssd_fmt,
1270     %wwn_fmt,
1271 };
1272
1273 my $efidisk_fmt = {
1274     volume => { alias => 'file' },
1275     file => {
1276         type => 'string',
1277         format => 'pve-volume-id-or-qm-path',
1278         default_key => 1,
1279         format_description => 'volume',
1280         description => "The drive's backing volume.",
1281     },
1282     format => get_standard_option('pve-qm-image-format'),
1283     size => {
1284         type => 'string',
1285         format => 'disk-size',
1286         format_description => 'DiskSize',
1287         description => "Disk size. This is purely informational and has no effect.",
1288         optional => 1,
1289     },
1290 };
1291
1292 my $efidisk_desc = {
1293     optional => 1,
1294     type => 'string', format => $efidisk_fmt,
1295     description => "Configure a Disk for storing EFI vars",
1296 };
1297
1298 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1299
1300 my $usb_fmt = {
1301     host => {
1302         default_key => 1,
1303         type => 'string', format => 'pve-qm-usb-device',
1304         format_description => 'HOSTUSBDEVICE|spice',
1305         description => <<EODESCR,
1306 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1307
1308  'bus-port(.port)*' (decimal numbers) or
1309  'vendor_id:product_id' (hexadeciaml numbers) or
1310  'spice'
1311
1312 You can use the 'lsusb -t' command to list existing usb devices.
1313
1314 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1315
1316 The value 'spice' can be used to add a usb redirection devices for spice.
1317 EODESCR
1318     },
1319     usb3 => {
1320         optional => 1,
1321         type => 'boolean',
1322         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1323         default => 0,
1324     },
1325 };
1326
1327 my $usbdesc = {
1328     optional => 1,
1329     type => 'string', format => $usb_fmt,
1330     description => "Configure an USB device (n is 0 to 4).",
1331 };
1332 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1333
1334 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1335 my $hostpci_fmt = {
1336     host => {
1337         default_key => 1,
1338         type => 'string',
1339         pattern => qr/$PCIRE(;$PCIRE)*/,
1340         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1341         description => <<EODESCR,
1342 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1343 of PCI virtual functions of the host. HOSTPCIID syntax is:
1344
1345 'bus:dev.func' (hexadecimal numbers)
1346
1347 You can us the 'lspci' command to list existing PCI devices.
1348 EODESCR
1349     },
1350     rombar => {
1351         type => 'boolean',
1352         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1353         optional => 1,
1354         default => 1,
1355     },
1356     romfile => {
1357         type => 'string',
1358         pattern => '[^,;]+',
1359         format_description => 'string',
1360         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1361         optional => 1,
1362     },
1363     pcie => {
1364         type => 'boolean',
1365         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1366         optional => 1,
1367         default => 0,
1368     },
1369     'x-vga' => {
1370         type => 'boolean',
1371         description =>  "Enable vfio-vga device support.",
1372         optional => 1,
1373         default => 0,
1374     },
1375     'mdev' => {
1376         type => 'string',
1377         format_description => 'string',
1378         pattern => '[^/\.:]+',
1379         optional => 1,
1380         description => <<EODESCR
1381 The type of mediated device to use.
1382 An instance of this type will be created on startup of the VM and
1383 will be cleaned up when the VM stops.
1384 EODESCR
1385     }
1386 };
1387 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1388
1389 my $hostpcidesc = {
1390         optional => 1,
1391         type => 'string', format => 'pve-qm-hostpci',
1392         description => "Map host PCI devices into guest.",
1393         verbose_description =>  <<EODESCR,
1394 Map host PCI devices into guest.
1395
1396 NOTE: This option allows direct access to host hardware. So it is no longer
1397 possible to migrate such machines - use with special care.
1398
1399 CAUTION: Experimental! User reported problems with this option.
1400 EODESCR
1401 };
1402 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1403
1404 my $serialdesc = {
1405         optional => 1,
1406         type => 'string',
1407         pattern => '(/dev/.+|socket)',
1408         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1409         verbose_description =>  <<EODESCR,
1410 Create a serial device inside the VM (n is 0 to 3), and pass through a
1411 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1412 host side (use 'qm terminal' to open a terminal connection).
1413
1414 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1415
1416 CAUTION: Experimental! User reported problems with this option.
1417 EODESCR
1418 };
1419
1420 my $paralleldesc= {
1421         optional => 1,
1422         type => 'string',
1423         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1424         description =>  "Map host parallel devices (n is 0 to 2).",
1425         verbose_description =>  <<EODESCR,
1426 Map host parallel devices (n is 0 to 2).
1427
1428 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1429
1430 CAUTION: Experimental! User reported problems with this option.
1431 EODESCR
1432 };
1433
1434 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1435     $confdesc->{"parallel$i"} = $paralleldesc;
1436 }
1437
1438 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1439     $confdesc->{"serial$i"} = $serialdesc;
1440 }
1441
1442 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1443     $confdesc->{"hostpci$i"} = $hostpcidesc;
1444 }
1445
1446 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1447     $drivename_hash->{"ide$i"} = 1;
1448     $confdesc->{"ide$i"} = $idedesc;
1449 }
1450
1451 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1452     $drivename_hash->{"sata$i"} = 1;
1453     $confdesc->{"sata$i"} = $satadesc;
1454 }
1455
1456 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1457     $drivename_hash->{"scsi$i"} = 1;
1458     $confdesc->{"scsi$i"} = $scsidesc ;
1459 }
1460
1461 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1462     $drivename_hash->{"virtio$i"} = 1;
1463     $confdesc->{"virtio$i"} = $virtiodesc;
1464 }
1465
1466 $drivename_hash->{efidisk0} = 1;
1467 $confdesc->{efidisk0} = $efidisk_desc;
1468
1469 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1470     $confdesc->{"usb$i"} = $usbdesc;
1471 }
1472
1473 my $unuseddesc = {
1474     optional => 1,
1475     type => 'string', format => 'pve-volume-id',
1476     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1477 };
1478
1479 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1480     $confdesc->{"unused$i"} = $unuseddesc;
1481 }
1482
1483 my $kvm_api_version = 0;
1484
1485 sub kvm_version {
1486     return $kvm_api_version if $kvm_api_version;
1487
1488     open my $fh, '<', '/dev/kvm'
1489         or return undef;
1490
1491     # 0xae00 => KVM_GET_API_VERSION
1492     $kvm_api_version = ioctl($fh, 0xae00, 0);
1493
1494     return $kvm_api_version;
1495 }
1496
1497 my $kvm_user_version = {};
1498 my $kvm_mtime = {};
1499
1500 sub kvm_user_version {
1501     my ($binary) = @_;
1502
1503     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1504     my $st = stat($binary);
1505
1506     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1507     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1508         $cachedmtime == $st->mtime;
1509
1510     $kvm_user_version->{$binary} = 'unknown';
1511     $kvm_mtime->{$binary} = $st->mtime;
1512
1513     my $code = sub {
1514         my $line = shift;
1515         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1516             $kvm_user_version->{$binary} = $2;
1517         }
1518     };
1519
1520     eval { run_command([$binary, '--version'], outfunc => $code); };
1521     warn $@ if $@;
1522
1523     return $kvm_user_version->{$binary};
1524
1525 }
1526
1527 sub kernel_has_vhost_net {
1528     return -c '/dev/vhost-net';
1529 }
1530
1531 sub valid_drive_names {
1532     # order is important - used to autoselect boot disk
1533     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1534             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1535             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1536             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1537             'efidisk0');
1538 }
1539
1540 sub is_valid_drivename {
1541     my $dev = shift;
1542
1543     return defined($drivename_hash->{$dev});
1544 }
1545
1546 sub option_exists {
1547     my $key = shift;
1548     return defined($confdesc->{$key});
1549 }
1550
1551 sub nic_models {
1552     return $nic_model_list;
1553 }
1554
1555 sub os_list_description {
1556
1557     return {
1558         other => 'Other',
1559         wxp => 'Windows XP',
1560         w2k => 'Windows 2000',
1561         w2k3 =>, 'Windows 2003',
1562         w2k8 => 'Windows 2008',
1563         wvista => 'Windows Vista',
1564         win7 => 'Windows 7',
1565         win8 => 'Windows 8/2012',
1566         win10 => 'Windows 10/2016',
1567         l24 => 'Linux 2.4',
1568         l26 => 'Linux 2.6',
1569     };
1570 }
1571
1572 my $cdrom_path;
1573
1574 sub get_cdrom_path {
1575
1576     return  $cdrom_path if $cdrom_path;
1577
1578     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1579     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1580     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1581 }
1582
1583 sub get_iso_path {
1584     my ($storecfg, $vmid, $cdrom) = @_;
1585
1586     if ($cdrom eq 'cdrom') {
1587         return get_cdrom_path();
1588     } elsif ($cdrom eq 'none') {
1589         return '';
1590     } elsif ($cdrom =~ m|^/|) {
1591         return $cdrom;
1592     } else {
1593         return PVE::Storage::path($storecfg, $cdrom);
1594     }
1595 }
1596
1597 # try to convert old style file names to volume IDs
1598 sub filename_to_volume_id {
1599     my ($vmid, $file, $media) = @_;
1600
1601      if (!($file eq 'none' || $file eq 'cdrom' ||
1602           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1603
1604         return undef if $file =~ m|/|;
1605
1606         if ($media && $media eq 'cdrom') {
1607             $file = "local:iso/$file";
1608         } else {
1609             $file = "local:$vmid/$file";
1610         }
1611     }
1612
1613     return $file;
1614 }
1615
1616 sub verify_media_type {
1617     my ($opt, $vtype, $media) = @_;
1618
1619     return if !$media;
1620
1621     my $etype;
1622     if ($media eq 'disk') {
1623         $etype = 'images';
1624     } elsif ($media eq 'cdrom') {
1625         $etype = 'iso';
1626     } else {
1627         die "internal error";
1628     }
1629
1630     return if ($vtype eq $etype);
1631
1632     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1633 }
1634
1635 sub cleanup_drive_path {
1636     my ($opt, $storecfg, $drive) = @_;
1637
1638     # try to convert filesystem paths to volume IDs
1639
1640     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1641         ($drive->{file} !~ m|^/dev/.+|) &&
1642         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1643         ($drive->{file} !~ m/^\d+$/)) {
1644         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1645         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1646         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1647         verify_media_type($opt, $vtype, $drive->{media});
1648         $drive->{file} = $volid;
1649     }
1650
1651     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1652 }
1653
1654 sub parse_hotplug_features {
1655     my ($data) = @_;
1656
1657     my $res = {};
1658
1659     return $res if $data eq '0';
1660
1661     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1662
1663     foreach my $feature (PVE::Tools::split_list($data)) {
1664         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1665             $res->{$1} = 1;
1666         } else {
1667             die "invalid hotplug feature '$feature'\n";
1668         }
1669     }
1670     return $res;
1671 }
1672
1673 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1674 sub pve_verify_hotplug_features {
1675     my ($value, $noerr) = @_;
1676
1677     return $value if parse_hotplug_features($value);
1678
1679     return undef if $noerr;
1680
1681     die "unable to parse hotplug option\n";
1682 }
1683
1684 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1685 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1686 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1687 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1688 #        [,iothread=on][,serial=serial][,model=model]
1689
1690 sub parse_drive {
1691     my ($key, $data) = @_;
1692
1693     my ($interface, $index);
1694
1695     if ($key =~ m/^([^\d]+)(\d+)$/) {
1696         $interface = $1;
1697         $index = $2;
1698     } else {
1699         return undef;
1700     }
1701
1702     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1703                                      : $confdesc->{$key}->{format};
1704     if (!$desc) {
1705         warn "invalid drive key: $key\n";
1706         return undef;
1707     }
1708     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1709     return undef if !$res;
1710     $res->{interface} = $interface;
1711     $res->{index} = $index;
1712
1713     my $error = 0;
1714     foreach my $opt (qw(bps bps_rd bps_wr)) {
1715         if (my $bps = defined(delete $res->{$opt})) {
1716             if (defined($res->{"m$opt"})) {
1717                 warn "both $opt and m$opt specified\n";
1718                 ++$error;
1719                 next;
1720             }
1721             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1722         }
1723     }
1724
1725     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1726     for my $requirement (
1727         [mbps_max => 'mbps'],
1728         [mbps_rd_max => 'mbps_rd'],
1729         [mbps_wr_max => 'mbps_wr'],
1730         [miops_max => 'miops'],
1731         [miops_rd_max => 'miops_rd'],
1732         [miops_wr_max => 'miops_wr'],
1733         [bps_max_length => 'mbps_max'],
1734         [bps_rd_max_length => 'mbps_rd_max'],
1735         [bps_wr_max_length => 'mbps_wr_max'],
1736         [iops_max_length => 'iops_max'],
1737         [iops_rd_max_length => 'iops_rd_max'],
1738         [iops_wr_max_length => 'iops_wr_max']) {
1739         my ($option, $requires) = @$requirement;
1740         if ($res->{$option} && !$res->{$requires}) {
1741             warn "$option requires $requires\n";
1742             ++$error;
1743         }
1744     }
1745
1746     return undef if $error;
1747
1748     return undef if $res->{mbps_rd} && $res->{mbps};
1749     return undef if $res->{mbps_wr} && $res->{mbps};
1750     return undef if $res->{iops_rd} && $res->{iops};
1751     return undef if $res->{iops_wr} && $res->{iops};
1752
1753     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1754         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1755         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1756         return undef if $res->{interface} eq 'virtio';
1757     }
1758
1759     if (my $size = $res->{size}) {
1760         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1761     }
1762
1763     return $res;
1764 }
1765
1766 sub print_drive {
1767     my ($vmid, $drive) = @_;
1768     my $data = { %$drive };
1769     delete $data->{$_} for qw(index interface);
1770     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1771 }
1772
1773 sub scsi_inquiry {
1774     my($fh, $noerr) = @_;
1775
1776     my $SG_IO = 0x2285;
1777     my $SG_GET_VERSION_NUM = 0x2282;
1778
1779     my $versionbuf = "\x00" x 8;
1780     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1781     if (!$ret) {
1782         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1783         return undef;
1784     }
1785     my $version = unpack("I", $versionbuf);
1786     if ($version < 30000) {
1787         die "scsi generic interface too old\n"  if !$noerr;
1788         return undef;
1789     }
1790
1791     my $buf = "\x00" x 36;
1792     my $sensebuf = "\x00" x 8;
1793     my $cmd = pack("C x3 C x1", 0x12, 36);
1794
1795     # see /usr/include/scsi/sg.h
1796     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1797
1798     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1799                       length($sensebuf), 0, length($buf), $buf,
1800                       $cmd, $sensebuf, 6000);
1801
1802     $ret = ioctl($fh, $SG_IO, $packet);
1803     if (!$ret) {
1804         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1805         return undef;
1806     }
1807
1808     my @res = unpack($sg_io_hdr_t, $packet);
1809     if ($res[17] || $res[18]) {
1810         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1811         return undef;
1812     }
1813
1814     my $res = {};
1815     (my $byte0, my $byte1, $res->{vendor},
1816      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1817
1818     $res->{removable} = $byte1 & 128 ? 1 : 0;
1819     $res->{type} = $byte0 & 31;
1820
1821     return $res;
1822 }
1823
1824 sub path_is_scsi {
1825     my ($path) = @_;
1826
1827     my $fh = IO::File->new("+<$path") || return undef;
1828     my $res = scsi_inquiry($fh, 1);
1829     close($fh);
1830
1831     return $res;
1832 }
1833
1834 sub machine_type_is_q35 {
1835     my ($conf) = @_;
1836
1837     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1838 }
1839
1840 sub print_tabletdevice_full {
1841     my ($conf, $arch) = @_;
1842
1843     my $q35 = machine_type_is_q35($conf);
1844
1845     # we use uhci for old VMs because tablet driver was buggy in older qemu
1846     my $usbbus;
1847     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1848         $usbbus = 'ehci';
1849     } else {
1850         $usbbus = 'uhci';
1851     }
1852
1853     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1854 }
1855
1856 sub print_keyboarddevice_full {
1857     my ($conf, $arch, $machine) = @_;
1858
1859     return undef if $arch ne 'aarch64';
1860
1861     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1862 }
1863
1864 sub print_drivedevice_full {
1865     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1866
1867     my $device = '';
1868     my $maxdev = 0;
1869
1870     if ($drive->{interface} eq 'virtio') {
1871         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1872         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1873         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1874     } elsif ($drive->{interface} eq 'scsi') {
1875
1876         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1877         my $unit = $drive->{index} % $maxdev;
1878         my $devicetype = 'hd';
1879         my $path = '';
1880         if (drive_is_cdrom($drive)) {
1881             $devicetype = 'cd';
1882         } else {
1883             if ($drive->{file} =~ m|^/|) {
1884                 $path = $drive->{file};
1885                 if (my $info = path_is_scsi($path)) {
1886                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1887                         $devicetype = 'block';
1888                     } elsif ($info->{type} == 1) { # tape
1889                         $devicetype = 'generic';
1890                     }
1891                 }
1892             } else {
1893                  $path = PVE::Storage::path($storecfg, $drive->{file});
1894             }
1895
1896             if($path =~ m/^iscsi\:\/\//){
1897                 $devicetype = 'generic';
1898             }
1899         }
1900
1901         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1902             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1903         } else {
1904             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1905         }
1906
1907         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1908             $device .= ",rotation_rate=1";
1909         }
1910         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1911
1912     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1913         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1914         my $controller = int($drive->{index} / $maxdev);
1915         my $unit = $drive->{index} % $maxdev;
1916         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1917
1918         $device = "ide-$devicetype";
1919         if ($drive->{interface} eq 'ide') {
1920             $device .= ",bus=ide.$controller,unit=$unit";
1921         } else {
1922             $device .= ",bus=ahci$controller.$unit";
1923         }
1924         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1925
1926         if ($devicetype eq 'hd') {
1927             if (my $model = $drive->{model}) {
1928                 $model = URI::Escape::uri_unescape($model);
1929                 $device .= ",model=$model";
1930             }
1931             if ($drive->{ssd}) {
1932                 $device .= ",rotation_rate=1";
1933             }
1934         }
1935         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1936     } elsif ($drive->{interface} eq 'usb') {
1937         die "implement me";
1938         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1939     } else {
1940         die "unsupported interface type";
1941     }
1942
1943     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1944
1945     if (my $serial = $drive->{serial}) {
1946         $serial = URI::Escape::uri_unescape($serial);
1947         $device .= ",serial=$serial";
1948     }
1949
1950
1951     return $device;
1952 }
1953
1954 sub get_initiator_name {
1955     my $initiator;
1956
1957     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1958     while (defined(my $line = <$fh>)) {
1959         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1960         $initiator = $1;
1961         last;
1962     }
1963     $fh->close();
1964
1965     return $initiator;
1966 }
1967
1968 sub print_drive_full {
1969     my ($storecfg, $vmid, $drive) = @_;
1970
1971     my $path;
1972     my $volid = $drive->{file};
1973     my $format;
1974
1975     if (drive_is_cdrom($drive)) {
1976         $path = get_iso_path($storecfg, $vmid, $volid);
1977     } else {
1978         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1979         if ($storeid) {
1980             $path = PVE::Storage::path($storecfg, $volid);
1981             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1982             $format = qemu_img_format($scfg, $volname);
1983         } else {
1984             $path = $volid;
1985             $format = "raw";
1986         }
1987    }
1988
1989     my $opts = '';
1990     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1991     foreach my $o (@qemu_drive_options) {
1992         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1993     }
1994
1995     # snapshot only accepts on|off
1996     if (defined($drive->{snapshot})) {
1997         my $v = $drive->{snapshot} ? 'on' : 'off';
1998         $opts .= ",snapshot=$v";
1999     }
2000
2001     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2002         my ($dir, $qmpname) = @$type;
2003         if (my $v = $drive->{"mbps$dir"}) {
2004             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2005         }
2006         if (my $v = $drive->{"mbps${dir}_max"}) {
2007             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2008         }
2009         if (my $v = $drive->{"bps${dir}_max_length"}) {
2010             $opts .= ",throttling.bps$qmpname-max-length=$v";
2011         }
2012         if (my $v = $drive->{"iops${dir}"}) {
2013             $opts .= ",throttling.iops$qmpname=$v";
2014         }
2015         if (my $v = $drive->{"iops${dir}_max"}) {
2016             $opts .= ",throttling.iops$qmpname-max=$v";
2017         }
2018         if (my $v = $drive->{"iops${dir}_max_length"}) {
2019             $opts .= ",throttling.iops$qmpname-max-length=$v";
2020         }
2021     }
2022
2023     $opts .= ",format=$format" if $format && !$drive->{format};
2024
2025     my $cache_direct = 0;
2026
2027     if (my $cache = $drive->{cache}) {
2028         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2029     } elsif (!drive_is_cdrom($drive)) {
2030         $opts .= ",cache=none";
2031         $cache_direct = 1;
2032     }
2033
2034     # aio native works only with O_DIRECT
2035     if (!$drive->{aio}) {
2036         if($cache_direct) {
2037             $opts .= ",aio=native";
2038         } else {
2039             $opts .= ",aio=threads";
2040         }
2041     }
2042
2043     if (!drive_is_cdrom($drive)) {
2044         my $detectzeroes;
2045         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2046             $detectzeroes = 'off';
2047         } elsif ($drive->{discard}) {
2048             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2049         } else {
2050             # This used to be our default with discard not being specified:
2051             $detectzeroes = 'on';
2052         }
2053         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2054     }
2055
2056     my $pathinfo = $path ? "file=$path," : '';
2057
2058     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2059 }
2060
2061 sub print_netdevice_full {
2062     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2063
2064     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2065
2066     my $device = $net->{model};
2067     if ($net->{model} eq 'virtio') {
2068          $device = 'virtio-net-pci';
2069      };
2070
2071     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2072     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2073     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2074         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2075         my $vectors = $net->{queues} * 2 + 2;
2076         $tmpstr .= ",vectors=$vectors,mq=on";
2077     }
2078     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2079
2080     if ($use_old_bios_files) {
2081         my $romfile;
2082         if ($device eq 'virtio-net-pci') {
2083             $romfile = 'pxe-virtio.rom';
2084         } elsif ($device eq 'e1000') {
2085             $romfile = 'pxe-e1000.rom';
2086         } elsif ($device eq 'ne2k') {
2087             $romfile = 'pxe-ne2k_pci.rom';
2088         } elsif ($device eq 'pcnet') {
2089             $romfile = 'pxe-pcnet.rom';
2090         } elsif ($device eq 'rtl8139') {
2091             $romfile = 'pxe-rtl8139.rom';
2092         }
2093         $tmpstr .= ",romfile=$romfile" if $romfile;
2094     }
2095
2096     return $tmpstr;
2097 }
2098
2099 sub print_netdev_full {
2100     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2101
2102     my $i = '';
2103     if ($netid =~ m/^net(\d+)$/) {
2104         $i = int($1);
2105     }
2106
2107     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2108
2109     my $ifname = "tap${vmid}i$i";
2110
2111     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2112     die "interface name '$ifname' is too long (max 15 character)\n"
2113         if length($ifname) >= 16;
2114
2115     my $vhostparam = '';
2116     if (is_native($arch)) {
2117         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2118     }
2119
2120     my $vmname = $conf->{name} || "vm$vmid";
2121
2122     my $netdev = "";
2123     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2124
2125     if ($net->{bridge}) {
2126         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2127     } else {
2128         $netdev = "type=user,id=$netid,hostname=$vmname";
2129     }
2130
2131     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2132
2133     return $netdev;
2134 }
2135
2136
2137 sub print_cpu_device {
2138     my ($conf, $id) = @_;
2139
2140     my $kvm = $conf->{kvm} // 1;
2141     my $cpu = $kvm ? "kvm64" : "qemu64";
2142     if (my $cputype = $conf->{cpu}) {
2143         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2144             or die "Cannot parse cpu description: $cputype\n";
2145         $cpu = $cpuconf->{cputype};
2146     }
2147
2148     my $cores = $conf->{cores} || 1;
2149
2150     my $current_core = ($id - 1) % $cores;
2151     my $current_socket = int(($id - 1 - $current_core)/$cores);
2152
2153     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2154 }
2155
2156 my $vga_map = {
2157     'cirrus' => 'cirrus-vga',
2158     'std' => 'VGA',
2159     'vmware' => 'vmware-svga',
2160     'virtio' => 'virtio-vga',
2161 };
2162
2163 sub print_vga_device {
2164     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2165
2166     my $type = $vga_map->{$vga->{type}};
2167     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2168         $type = 'virtio-gpu';
2169     }
2170     my $vgamem_mb = $vga->{memory};
2171     if ($qxlnum) {
2172         $type = $id ? 'qxl' : 'qxl-vga';
2173     }
2174     die "no devicetype for $vga->{type}\n" if !$type;
2175
2176     my $memory = "";
2177     if ($vgamem_mb) {
2178         if ($vga->{type} eq 'virtio') {
2179             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2180             $memory = ",max_hostmem=$bytes";
2181         } elsif ($qxlnum) {
2182             # from https://www.spice-space.org/multiple-monitors.html
2183             $memory = ",vgamem_mb=$vga->{memory}";
2184             my $ram = $vgamem_mb * 4;
2185             my $vram = $vgamem_mb * 2;
2186             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2187         } else {
2188             $memory = ",vgamem_mb=$vga->{memory}";
2189         }
2190     } elsif ($qxlnum && $id) {
2191         $memory = ",ram_size=67108864,vram_size=33554432";
2192     }
2193
2194     my $q35 = machine_type_is_q35($conf);
2195     my $vgaid = "vga" . ($id // '');
2196     my $pciaddr;
2197
2198     if ($q35 && $vgaid eq 'vga') {
2199         # the first display uses pcie.0 bus on q35 machines
2200         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2201     } else {
2202         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2203     }
2204
2205     return "$type,id=${vgaid}${memory}${pciaddr}";
2206 }
2207
2208 sub drive_is_cloudinit {
2209     my ($drive) = @_;
2210     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2211 }
2212
2213 sub drive_is_cdrom {
2214     my ($drive, $exclude_cloudinit) = @_;
2215
2216     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2217
2218     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2219
2220 }
2221
2222 sub parse_number_sets {
2223     my ($set) = @_;
2224     my $res = [];
2225     foreach my $part (split(/;/, $set)) {
2226         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2227             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2228             push @$res, [ $1, $2 ];
2229         } else {
2230             die "invalid range: $part\n";
2231         }
2232     }
2233     return $res;
2234 }
2235
2236 sub parse_numa {
2237     my ($data) = @_;
2238
2239     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2240     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2241     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2242     return $res;
2243 }
2244
2245 sub parse_hostpci {
2246     my ($value) = @_;
2247
2248     return undef if !$value;
2249
2250     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2251
2252     my @idlist = split(/;/, $res->{host});
2253     delete $res->{host};
2254     foreach my $id (@idlist) {
2255         if ($id =~ m/\./) { # full id 00:00.1
2256             push @{$res->{pciid}}, {
2257                 id => $id,
2258             };
2259         } else { # partial id 00:00
2260             $res->{pciid} = PVE::SysFSTools::lspci($id);
2261         }
2262     }
2263     return $res;
2264 }
2265
2266 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2267 sub parse_net {
2268     my ($data) = @_;
2269
2270     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2271     if ($@) {
2272         warn $@;
2273         return undef;
2274     }
2275     if (!defined($res->{macaddr})) {
2276         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2277         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2278     }
2279     return $res;
2280 }
2281
2282 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2283 sub parse_ipconfig {
2284     my ($data) = @_;
2285
2286     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2287     if ($@) {
2288         warn $@;
2289         return undef;
2290     }
2291
2292     if ($res->{gw} && !$res->{ip}) {
2293         warn 'gateway specified without specifying an IP address';
2294         return undef;
2295     }
2296     if ($res->{gw6} && !$res->{ip6}) {
2297         warn 'IPv6 gateway specified without specifying an IPv6 address';
2298         return undef;
2299     }
2300     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2301         warn 'gateway specified together with DHCP';
2302         return undef;
2303     }
2304     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2305         # gw6 + auto/dhcp
2306         warn "IPv6 gateway specified together with $res->{ip6} address";
2307         return undef;
2308     }
2309
2310     if (!$res->{ip} && !$res->{ip6}) {
2311         return { ip => 'dhcp', ip6 => 'dhcp' };
2312     }
2313
2314     return $res;
2315 }
2316
2317 sub print_net {
2318     my $net = shift;
2319
2320     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2321 }
2322
2323 sub add_random_macs {
2324     my ($settings) = @_;
2325
2326     foreach my $opt (keys %$settings) {
2327         next if $opt !~ m/^net(\d+)$/;
2328         my $net = parse_net($settings->{$opt});
2329         next if !$net;
2330         $settings->{$opt} = print_net($net);
2331     }
2332 }
2333
2334 sub vm_is_volid_owner {
2335     my ($storecfg, $vmid, $volid) = @_;
2336
2337     if ($volid !~  m|^/|) {
2338         my ($path, $owner);
2339         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2340         if ($owner && ($owner == $vmid)) {
2341             return 1;
2342         }
2343     }
2344
2345     return undef;
2346 }
2347
2348 sub split_flagged_list {
2349     my $text = shift || '';
2350     $text =~ s/[,;]/ /g;
2351     $text =~ s/^\s+//;
2352     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2353 }
2354
2355 sub join_flagged_list {
2356     my ($how, $lst) = @_;
2357     join $how, map { $lst->{$_} . $_ } keys %$lst;
2358 }
2359
2360 sub vmconfig_delete_pending_option {
2361     my ($conf, $key, $force) = @_;
2362
2363     delete $conf->{pending}->{$key};
2364     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2365     $pending_delete_hash->{$key} = $force ? '!' : '';
2366     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2367 }
2368
2369 sub vmconfig_undelete_pending_option {
2370     my ($conf, $key) = @_;
2371
2372     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2373     delete $pending_delete_hash->{$key};
2374
2375     if (%$pending_delete_hash) {
2376         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2377     } else {
2378         delete $conf->{pending}->{delete};
2379     }
2380 }
2381
2382 sub vmconfig_register_unused_drive {
2383     my ($storecfg, $vmid, $conf, $drive) = @_;
2384
2385     if (drive_is_cloudinit($drive)) {
2386         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2387         warn $@ if $@;
2388     } elsif (!drive_is_cdrom($drive)) {
2389         my $volid = $drive->{file};
2390         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2391             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2392         }
2393     }
2394 }
2395
2396 sub vmconfig_cleanup_pending {
2397     my ($conf) = @_;
2398
2399     # remove pending changes when nothing changed
2400     my $changes;
2401     foreach my $opt (keys %{$conf->{pending}}) {
2402         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2403             $changes = 1;
2404             delete $conf->{pending}->{$opt};
2405         }
2406     }
2407
2408     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2409     my $pending_delete_hash = {};
2410     while (my ($opt, $force) = each %$current_delete_hash) {
2411         if (defined($conf->{$opt})) {
2412             $pending_delete_hash->{$opt} = $force;
2413         } else {
2414             $changes = 1;
2415         }
2416     }
2417
2418     if (%$pending_delete_hash) {
2419         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2420     } else {
2421         delete $conf->{pending}->{delete};
2422     }
2423
2424     return $changes;
2425 }
2426
2427 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2428 my $smbios1_fmt = {
2429     uuid => {
2430         type => 'string',
2431         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2432         format_description => 'UUID',
2433         description => "Set SMBIOS1 UUID.",
2434         optional => 1,
2435     },
2436     version => {
2437         type => 'string',
2438         pattern => '[A-Za-z0-9+\/]+={0,2}',
2439         format_description => 'Base64 encoded string',
2440         description => "Set SMBIOS1 version.",
2441         optional => 1,
2442     },
2443     serial => {
2444         type => 'string',
2445         pattern => '[A-Za-z0-9+\/]+={0,2}',
2446         format_description => 'Base64 encoded string',
2447         description => "Set SMBIOS1 serial number.",
2448         optional => 1,
2449     },
2450     manufacturer => {
2451         type => 'string',
2452         pattern => '[A-Za-z0-9+\/]+={0,2}',
2453         format_description => 'Base64 encoded string',
2454         description => "Set SMBIOS1 manufacturer.",
2455         optional => 1,
2456     },
2457     product => {
2458         type => 'string',
2459         pattern => '[A-Za-z0-9+\/]+={0,2}',
2460         format_description => 'Base64 encoded string',
2461         description => "Set SMBIOS1 product ID.",
2462         optional => 1,
2463     },
2464     sku => {
2465         type => 'string',
2466         pattern => '[A-Za-z0-9+\/]+={0,2}',
2467         format_description => 'Base64 encoded string',
2468         description => "Set SMBIOS1 SKU string.",
2469         optional => 1,
2470     },
2471     family => {
2472         type => 'string',
2473         pattern => '[A-Za-z0-9+\/]+={0,2}',
2474         format_description => 'Base64 encoded string',
2475         description => "Set SMBIOS1 family string.",
2476         optional => 1,
2477     },
2478     base64 => {
2479         type => 'boolean',
2480         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2481         optional => 1,
2482     },
2483 };
2484
2485 sub parse_smbios1 {
2486     my ($data) = @_;
2487
2488     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2489     warn $@ if $@;
2490     return $res;
2491 }
2492
2493 sub print_smbios1 {
2494     my ($smbios1) = @_;
2495     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2496 }
2497
2498 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2499
2500 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2501 sub verify_bootdisk {
2502     my ($value, $noerr) = @_;
2503
2504     return $value if is_valid_drivename($value);
2505
2506     return undef if $noerr;
2507
2508     die "invalid boot disk '$value'\n";
2509 }
2510
2511 sub parse_watchdog {
2512     my ($value) = @_;
2513
2514     return undef if !$value;
2515
2516     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2517     warn $@ if $@;
2518     return $res;
2519 }
2520
2521 sub parse_guest_agent {
2522     my ($value) = @_;
2523
2524     return {} if !defined($value->{agent});
2525
2526     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2527     warn $@ if $@;
2528
2529     # if the agent is disabled ignore the other potentially set properties
2530     return {} if !$res->{enabled};
2531     return $res;
2532 }
2533
2534 sub parse_vga {
2535     my ($value) = @_;
2536
2537     return {} if !$value;
2538     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2539     warn $@ if $@;
2540     return $res;
2541 }
2542
2543 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2544 sub verify_usb_device {
2545     my ($value, $noerr) = @_;
2546
2547     return $value if parse_usb_device($value);
2548
2549     return undef if $noerr;
2550
2551     die "unable to parse usb device\n";
2552 }
2553
2554 # add JSON properties for create and set function
2555 sub json_config_properties {
2556     my $prop = shift;
2557
2558     foreach my $opt (keys %$confdesc) {
2559         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2560         $prop->{$opt} = $confdesc->{$opt};
2561     }
2562
2563     return $prop;
2564 }
2565
2566 # return copy of $confdesc_cloudinit to generate documentation
2567 sub cloudinit_config_properties {
2568
2569     return dclone($confdesc_cloudinit);
2570 }
2571
2572 sub check_type {
2573     my ($key, $value) = @_;
2574
2575     die "unknown setting '$key'\n" if !$confdesc->{$key};
2576
2577     my $type = $confdesc->{$key}->{type};
2578
2579     if (!defined($value)) {
2580         die "got undefined value\n";
2581     }
2582
2583     if ($value =~ m/[\n\r]/) {
2584         die "property contains a line feed\n";
2585     }
2586
2587     if ($type eq 'boolean') {
2588         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2589         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2590         die "type check ('boolean') failed - got '$value'\n";
2591     } elsif ($type eq 'integer') {
2592         return int($1) if $value =~ m/^(\d+)$/;
2593         die "type check ('integer') failed - got '$value'\n";
2594     } elsif ($type eq 'number') {
2595         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2596         die "type check ('number') failed - got '$value'\n";
2597     } elsif ($type eq 'string') {
2598         if (my $fmt = $confdesc->{$key}->{format}) {
2599             PVE::JSONSchema::check_format($fmt, $value);
2600             return $value;
2601         }
2602         $value =~ s/^\"(.*)\"$/$1/;
2603         return $value;
2604     } else {
2605         die "internal error"
2606     }
2607 }
2608
2609 sub touch_config {
2610     my ($vmid) = @_;
2611
2612     my $conf = PVE::QemuConfig->config_file($vmid);
2613     utime undef, undef, $conf;
2614 }
2615
2616 sub destroy_vm {
2617     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2618
2619     my $conffile = PVE::QemuConfig->config_file($vmid);
2620
2621     my $conf = PVE::QemuConfig->load_config($vmid);
2622
2623     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2624
2625     if ($conf->{template}) {
2626         # check if any base image is still used by a linked clone
2627         foreach_drive($conf, sub {
2628                 my ($ds, $drive) = @_;
2629
2630                 return if drive_is_cdrom($drive);
2631
2632                 my $volid = $drive->{file};
2633
2634                 return if !$volid || $volid =~ m|^/|;
2635
2636                 die "base volume '$volid' is still in use by linked cloned\n"
2637                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2638
2639         });
2640     }
2641
2642     # only remove disks owned by this VM
2643     foreach_drive($conf, sub {
2644         my ($ds, $drive) = @_;
2645
2646         return if drive_is_cdrom($drive, 1);
2647
2648         my $volid = $drive->{file};
2649
2650         return if !$volid || $volid =~ m|^/|;
2651
2652         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2653         return if !$path || !$owner || ($owner != $vmid);
2654
2655         eval {
2656             PVE::Storage::vdisk_free($storecfg, $volid);
2657         };
2658         warn "Could not remove disk '$volid', check manually: $@" if $@;
2659
2660     });
2661
2662     if ($keep_empty_config) {
2663         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2664     } else {
2665         unlink $conffile;
2666     }
2667
2668     # also remove unused disk
2669     eval {
2670         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2671
2672         eval {
2673             PVE::Storage::foreach_volid($dl, sub {
2674                 my ($volid, $sid, $volname, $d) = @_;
2675                 PVE::Storage::vdisk_free($storecfg, $volid);
2676             });
2677         };
2678         warn $@ if $@;
2679
2680     };
2681     warn $@ if $@;
2682 }
2683
2684 sub parse_vm_config {
2685     my ($filename, $raw) = @_;
2686
2687     return undef if !defined($raw);
2688
2689     my $res = {
2690         digest => Digest::SHA::sha1_hex($raw),
2691         snapshots => {},
2692         pending => {},
2693     };
2694
2695     $filename =~ m|/qemu-server/(\d+)\.conf$|
2696         || die "got strange filename '$filename'";
2697
2698     my $vmid = $1;
2699
2700     my $conf = $res;
2701     my $descr;
2702     my $section = '';
2703
2704     my @lines = split(/\n/, $raw);
2705     foreach my $line (@lines) {
2706         next if $line =~ m/^\s*$/;
2707
2708         if ($line =~ m/^\[PENDING\]\s*$/i) {
2709             $section = 'pending';
2710             if (defined($descr)) {
2711                 $descr =~ s/\s+$//;
2712                 $conf->{description} = $descr;
2713             }
2714             $descr = undef;
2715             $conf = $res->{$section} = {};
2716             next;
2717
2718         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2719             $section = $1;
2720             if (defined($descr)) {
2721                 $descr =~ s/\s+$//;
2722                 $conf->{description} = $descr;
2723             }
2724             $descr = undef;
2725             $conf = $res->{snapshots}->{$section} = {};
2726             next;
2727         }
2728
2729         if ($line =~ m/^\#(.*)\s*$/) {
2730             $descr = '' if !defined($descr);
2731             $descr .= PVE::Tools::decode_text($1) . "\n";
2732             next;
2733         }
2734
2735         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2736             $descr = '' if !defined($descr);
2737             $descr .= PVE::Tools::decode_text($2);
2738         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2739             $conf->{snapstate} = $1;
2740         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2741             my $key = $1;
2742             my $value = $2;
2743             $conf->{$key} = $value;
2744         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2745             my $value = $1;
2746             if ($section eq 'pending') {
2747                 $conf->{delete} = $value; # we parse this later
2748             } else {
2749                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2750             }
2751         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2752             my $key = $1;
2753             my $value = $2;
2754             eval { $value = check_type($key, $value); };
2755             if ($@) {
2756                 warn "vm $vmid - unable to parse value of '$key' - $@";
2757             } else {
2758                 $key = 'ide2' if $key eq 'cdrom';
2759                 my $fmt = $confdesc->{$key}->{format};
2760                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2761                     my $v = parse_drive($key, $value);
2762                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2763                         $v->{file} = $volid;
2764                         $value = print_drive($vmid, $v);
2765                     } else {
2766                         warn "vm $vmid - unable to parse value of '$key'\n";
2767                         next;
2768                     }
2769                 }
2770
2771                 $conf->{$key} = $value;
2772             }
2773         }
2774     }
2775
2776     if (defined($descr)) {
2777         $descr =~ s/\s+$//;
2778         $conf->{description} = $descr;
2779     }
2780     delete $res->{snapstate}; # just to be sure
2781
2782     return $res;
2783 }
2784
2785 sub write_vm_config {
2786     my ($filename, $conf) = @_;
2787
2788     delete $conf->{snapstate}; # just to be sure
2789
2790     if ($conf->{cdrom}) {
2791         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2792         $conf->{ide2} = $conf->{cdrom};
2793         delete $conf->{cdrom};
2794     }
2795
2796     # we do not use 'smp' any longer
2797     if ($conf->{sockets}) {
2798         delete $conf->{smp};
2799     } elsif ($conf->{smp}) {
2800         $conf->{sockets} = $conf->{smp};
2801         delete $conf->{cores};
2802         delete $conf->{smp};
2803     }
2804
2805     my $used_volids = {};
2806
2807     my $cleanup_config = sub {
2808         my ($cref, $pending, $snapname) = @_;
2809
2810         foreach my $key (keys %$cref) {
2811             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2812                 $key eq 'snapstate' || $key eq 'pending';
2813             my $value = $cref->{$key};
2814             if ($key eq 'delete') {
2815                 die "propertry 'delete' is only allowed in [PENDING]\n"
2816                     if !$pending;
2817                 # fixme: check syntax?
2818                 next;
2819             }
2820             eval { $value = check_type($key, $value); };
2821             die "unable to parse value of '$key' - $@" if $@;
2822
2823             $cref->{$key} = $value;
2824
2825             if (!$snapname && is_valid_drivename($key)) {
2826                 my $drive = parse_drive($key, $value);
2827                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2828             }
2829         }
2830     };
2831
2832     &$cleanup_config($conf);
2833
2834     &$cleanup_config($conf->{pending}, 1);
2835
2836     foreach my $snapname (keys %{$conf->{snapshots}}) {
2837         die "internal error" if $snapname eq 'pending';
2838         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2839     }
2840
2841     # remove 'unusedX' settings if we re-add a volume
2842     foreach my $key (keys %$conf) {
2843         my $value = $conf->{$key};
2844         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2845             delete $conf->{$key};
2846         }
2847     }
2848
2849     my $generate_raw_config = sub {
2850         my ($conf, $pending) = @_;
2851
2852         my $raw = '';
2853
2854         # add description as comment to top of file
2855         if (defined(my $descr = $conf->{description})) {
2856             if ($descr) {
2857                 foreach my $cl (split(/\n/, $descr)) {
2858                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2859                 }
2860             } else {
2861                 $raw .= "#\n" if $pending;
2862             }
2863         }
2864
2865         foreach my $key (sort keys %$conf) {
2866             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2867             $raw .= "$key: $conf->{$key}\n";
2868         }
2869         return $raw;
2870     };
2871
2872     my $raw = &$generate_raw_config($conf);
2873
2874     if (scalar(keys %{$conf->{pending}})){
2875         $raw .= "\n[PENDING]\n";
2876         $raw .= &$generate_raw_config($conf->{pending}, 1);
2877     }
2878
2879     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2880         $raw .= "\n[$snapname]\n";
2881         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2882     }
2883
2884     return $raw;
2885 }
2886
2887 sub load_defaults {
2888
2889     my $res = {};
2890
2891     # we use static defaults from our JSON schema configuration
2892     foreach my $key (keys %$confdesc) {
2893         if (defined(my $default = $confdesc->{$key}->{default})) {
2894             $res->{$key} = $default;
2895         }
2896     }
2897
2898     return $res;
2899 }
2900
2901 sub config_list {
2902     my $vmlist = PVE::Cluster::get_vmlist();
2903     my $res = {};
2904     return $res if !$vmlist || !$vmlist->{ids};
2905     my $ids = $vmlist->{ids};
2906
2907     foreach my $vmid (keys %$ids) {
2908         my $d = $ids->{$vmid};
2909         next if !$d->{node} || $d->{node} ne $nodename;
2910         next if !$d->{type} || $d->{type} ne 'qemu';
2911         $res->{$vmid}->{exists} = 1;
2912     }
2913     return $res;
2914 }
2915
2916 # test if VM uses local resources (to prevent migration)
2917 sub check_local_resources {
2918     my ($conf, $noerr) = @_;
2919
2920     my @loc_res = ();
2921
2922     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2923     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2924
2925     push @loc_res, "ivshmem" if $conf->{ivshmem};
2926
2927     foreach my $k (keys %$conf) {
2928         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2929         # sockets are safe: they will recreated be on the target side post-migrate
2930         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2931         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2932     }
2933
2934     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2935
2936     return \@loc_res;
2937 }
2938
2939 # check if used storages are available on all nodes (use by migrate)
2940 sub check_storage_availability {
2941     my ($storecfg, $conf, $node) = @_;
2942
2943     foreach_drive($conf, sub {
2944         my ($ds, $drive) = @_;
2945
2946         my $volid = $drive->{file};
2947         return if !$volid;
2948
2949         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2950         return if !$sid;
2951
2952         # check if storage is available on both nodes
2953         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2954         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2955    });
2956 }
2957
2958 # list nodes where all VM images are available (used by has_feature API)
2959 sub shared_nodes {
2960     my ($conf, $storecfg) = @_;
2961
2962     my $nodelist = PVE::Cluster::get_nodelist();
2963     my $nodehash = { map { $_ => 1 } @$nodelist };
2964     my $nodename = PVE::INotify::nodename();
2965
2966     foreach_drive($conf, sub {
2967         my ($ds, $drive) = @_;
2968
2969         my $volid = $drive->{file};
2970         return if !$volid;
2971
2972         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2973         if ($storeid) {
2974             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2975             if ($scfg->{disable}) {
2976                 $nodehash = {};
2977             } elsif (my $avail = $scfg->{nodes}) {
2978                 foreach my $node (keys %$nodehash) {
2979                     delete $nodehash->{$node} if !$avail->{$node};
2980                 }
2981             } elsif (!$scfg->{shared}) {
2982                 foreach my $node (keys %$nodehash) {
2983                     delete $nodehash->{$node} if $node ne $nodename
2984                 }
2985             }
2986         }
2987     });
2988
2989     return $nodehash
2990 }
2991
2992 sub check_local_storage_availability {
2993     my ($conf, $storecfg) = @_;
2994
2995     my $nodelist = PVE::Cluster::get_nodelist();
2996     my $nodehash = { map { $_ => {} } @$nodelist };
2997
2998     foreach_drive($conf, sub {
2999         my ($ds, $drive) = @_;
3000
3001         my $volid = $drive->{file};
3002         return if !$volid;
3003
3004         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
3005         if ($storeid) {
3006             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3007
3008             if ($scfg->{disable}) {
3009                 foreach my $node (keys %$nodehash) {
3010                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3011                 }
3012             } elsif (my $avail = $scfg->{nodes}) {
3013                 foreach my $node (keys %$nodehash) {
3014                     if (!$avail->{$node}) {
3015                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
3016                     }
3017                 }
3018             }
3019         }
3020     });
3021
3022     foreach my $node (values %$nodehash) {
3023         if (my $unavail = $node->{unavailable_storages}) {
3024             $node->{unavailable_storages} = [ sort keys %$unavail ];
3025         }
3026     }
3027
3028     return $nodehash
3029 }
3030
3031 sub check_cmdline {
3032     my ($pidfile, $pid) = @_;
3033
3034     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
3035     if (defined($fh)) {
3036         my $line = <$fh>;
3037         $fh->close;
3038         return undef if !$line;
3039         my @param = split(/\0/, $line);
3040
3041         my $cmd = $param[0];
3042         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
3043
3044         for (my $i = 0; $i < scalar (@param); $i++) {
3045             my $p = $param[$i];
3046             next if !$p;
3047             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
3048                 my $p = $param[$i+1];
3049                 return 1 if $p && ($p eq $pidfile);
3050                 return undef;
3051             }
3052         }
3053     }
3054     return undef;
3055 }
3056
3057 sub check_running {
3058     my ($vmid, $nocheck, $node) = @_;
3059
3060     my $filename = PVE::QemuConfig->config_file($vmid, $node);
3061
3062     die "unable to find configuration file for VM $vmid - no such machine\n"
3063         if !$nocheck && ! -f $filename;
3064
3065     my $pidfile = pidfile_name($vmid);
3066
3067     if (my $fd = IO::File->new("<$pidfile")) {
3068         my $st = stat($fd);
3069         my $line = <$fd>;
3070         close($fd);
3071
3072         my $mtime = $st->mtime;
3073         if ($mtime > time()) {
3074             warn "file '$filename' modified in future\n";
3075         }
3076
3077         if ($line =~ m/^(\d+)$/) {
3078             my $pid = $1;
3079             if (check_cmdline($pidfile, $pid)) {
3080                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3081                     return $pid;
3082                 }
3083             }
3084         }
3085     }
3086
3087     return undef;
3088 }
3089
3090 sub vzlist {
3091
3092     my $vzlist = config_list();
3093
3094     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3095
3096     while (defined(my $de = $fd->read)) {
3097         next if $de !~ m/^(\d+)\.pid$/;
3098         my $vmid = $1;
3099         next if !defined($vzlist->{$vmid});
3100         if (my $pid = check_running($vmid)) {
3101             $vzlist->{$vmid}->{pid} = $pid;
3102         }
3103     }
3104
3105     return $vzlist;
3106 }
3107
3108 sub disksize {
3109     my ($storecfg, $conf) = @_;
3110
3111     my $bootdisk = $conf->{bootdisk};
3112     return undef if !$bootdisk;
3113     return undef if !is_valid_drivename($bootdisk);
3114
3115     return undef if !$conf->{$bootdisk};
3116
3117     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3118     return undef if !defined($drive);
3119
3120     return undef if drive_is_cdrom($drive);
3121
3122     my $volid = $drive->{file};
3123     return undef if !$volid;
3124
3125     return $drive->{size};
3126 }
3127
3128 our $vmstatus_return_properties = {
3129     vmid => get_standard_option('pve-vmid'),
3130     status => {
3131         description => "Qemu process status.",
3132         type => 'string',
3133         enum => ['stopped', 'running'],
3134     },
3135     maxmem => {
3136         description => "Maximum memory in bytes.",
3137         type => 'integer',
3138         optional => 1,
3139         renderer => 'bytes',
3140     },
3141     maxdisk => {
3142         description => "Root disk size in bytes.",
3143         type => 'integer',
3144         optional => 1,
3145         renderer => 'bytes',
3146     },
3147     name => {
3148         description => "VM name.",
3149         type => 'string',
3150         optional => 1,
3151     },
3152     qmpstatus => {
3153         description => "Qemu QMP agent status.",
3154         type => 'string',
3155         optional => 1,
3156     },
3157     pid => {
3158         description => "PID of running qemu process.",
3159         type => 'integer',
3160         optional => 1,
3161     },
3162     uptime => {
3163         description => "Uptime.",
3164         type => 'integer',
3165         optional => 1,
3166         renderer => 'duration',
3167     },
3168     cpus => {
3169         description => "Maximum usable CPUs.",
3170         type => 'number',
3171         optional => 1,
3172     },
3173     lock => {
3174         description => "The current config lock, if any.",
3175         type => 'string',
3176         optional => 1,
3177     }
3178 };
3179
3180 my $last_proc_pid_stat;
3181
3182 # get VM status information
3183 # This must be fast and should not block ($full == false)
3184 # We only query KVM using QMP if $full == true (this can be slow)
3185 sub vmstatus {
3186     my ($opt_vmid, $full) = @_;
3187
3188     my $res = {};
3189
3190     my $storecfg = PVE::Storage::config();
3191
3192     my $list = vzlist();
3193     my $defaults = load_defaults();
3194
3195     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3196
3197     my $cpucount = $cpuinfo->{cpus} || 1;
3198
3199     foreach my $vmid (keys %$list) {
3200         next if $opt_vmid && ($vmid ne $opt_vmid);
3201
3202         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
3203         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
3204
3205         my $d = { vmid => $vmid };
3206         $d->{pid} = $list->{$vmid}->{pid};
3207
3208         # fixme: better status?
3209         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3210
3211         my $size = disksize($storecfg, $conf);
3212         if (defined($size)) {
3213             $d->{disk} = 0; # no info available
3214             $d->{maxdisk} = $size;
3215         } else {
3216             $d->{disk} = 0;
3217             $d->{maxdisk} = 0;
3218         }
3219
3220         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3221             * ($conf->{cores} || $defaults->{cores});
3222         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3223         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3224
3225         $d->{name} = $conf->{name} || "VM $vmid";
3226         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3227             : $defaults->{memory}*(1024*1024);
3228
3229         if ($conf->{balloon}) {
3230             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3231             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3232                 : $defaults->{shares};
3233         }
3234
3235         $d->{uptime} = 0;
3236         $d->{cpu} = 0;
3237         $d->{mem} = 0;
3238
3239         $d->{netout} = 0;
3240         $d->{netin} = 0;
3241
3242         $d->{diskread} = 0;
3243         $d->{diskwrite} = 0;
3244
3245         $d->{template} = PVE::QemuConfig->is_template($conf);
3246
3247         $d->{serial} = 1 if conf_has_serial($conf);
3248         $d->{lock} = $conf->{lock} if $conf->{lock};
3249
3250         $res->{$vmid} = $d;
3251     }
3252
3253     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3254     foreach my $dev (keys %$netdev) {
3255         next if $dev !~ m/^tap([1-9]\d*)i/;
3256         my $vmid = $1;
3257         my $d = $res->{$vmid};
3258         next if !$d;
3259
3260         $d->{netout} += $netdev->{$dev}->{receive};
3261         $d->{netin} += $netdev->{$dev}->{transmit};
3262
3263         if ($full) {
3264             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3265             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3266         }
3267
3268     }
3269
3270     my $ctime = gettimeofday;
3271
3272     foreach my $vmid (keys %$list) {
3273
3274         my $d = $res->{$vmid};
3275         my $pid = $d->{pid};
3276         next if !$pid;
3277
3278         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3279         next if !$pstat; # not running
3280
3281         my $used = $pstat->{utime} + $pstat->{stime};
3282
3283         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3284
3285         if ($pstat->{vsize}) {
3286             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3287         }
3288
3289         my $old = $last_proc_pid_stat->{$pid};
3290         if (!$old) {
3291             $last_proc_pid_stat->{$pid} = {
3292                 time => $ctime,
3293                 used => $used,
3294                 cpu => 0,
3295             };
3296             next;
3297         }
3298
3299         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3300
3301         if ($dtime > 1000) {
3302             my $dutime = $used -  $old->{used};
3303
3304             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3305             $last_proc_pid_stat->{$pid} = {
3306                 time => $ctime,
3307                 used => $used,
3308                 cpu => $d->{cpu},
3309             };
3310         } else {
3311             $d->{cpu} = $old->{cpu};
3312         }
3313     }
3314
3315     return $res if !$full;
3316
3317     my $qmpclient = PVE::QMPClient->new();
3318
3319     my $ballooncb = sub {
3320         my ($vmid, $resp) = @_;
3321
3322         my $info = $resp->{'return'};
3323         return if !$info->{max_mem};
3324
3325         my $d = $res->{$vmid};
3326
3327         # use memory assigned to VM
3328         $d->{maxmem} = $info->{max_mem};
3329         $d->{balloon} = $info->{actual};
3330
3331         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3332             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3333             $d->{freemem} = $info->{free_mem};
3334         }
3335
3336         $d->{ballooninfo} = $info;
3337     };
3338
3339     my $blockstatscb = sub {
3340         my ($vmid, $resp) = @_;
3341         my $data = $resp->{'return'} || [];
3342         my $totalrdbytes = 0;
3343         my $totalwrbytes = 0;
3344
3345         for my $blockstat (@$data) {
3346             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3347             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3348
3349             $blockstat->{device} =~ s/drive-//;
3350             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3351         }
3352         $res->{$vmid}->{diskread} = $totalrdbytes;
3353         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3354     };
3355
3356     my $statuscb = sub {
3357         my ($vmid, $resp) = @_;
3358
3359         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3360         # this fails if ballon driver is not loaded, so this must be
3361         # the last commnand (following command are aborted if this fails).
3362         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3363
3364         my $status = 'unknown';
3365         if (!defined($status = $resp->{'return'}->{status})) {
3366             warn "unable to get VM status\n";
3367             return;
3368         }
3369
3370         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3371     };
3372
3373     foreach my $vmid (keys %$list) {
3374         next if $opt_vmid && ($vmid ne $opt_vmid);
3375         next if !$res->{$vmid}->{pid}; # not running
3376         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3377     }
3378
3379     $qmpclient->queue_execute(undef, 2);
3380
3381     foreach my $vmid (keys %$list) {
3382         next if $opt_vmid && ($vmid ne $opt_vmid);
3383         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3384     }
3385
3386     return $res;
3387 }
3388
3389 sub foreach_drive {
3390     my ($conf, $func, @param) = @_;
3391
3392     foreach my $ds (valid_drive_names()) {
3393         next if !defined($conf->{$ds});
3394
3395         my $drive = parse_drive($ds, $conf->{$ds});
3396         next if !$drive;
3397
3398         &$func($ds, $drive, @param);
3399     }
3400 }
3401
3402 sub foreach_volid {
3403     my ($conf, $func, @param) = @_;
3404
3405     my $volhash = {};
3406
3407     my $test_volid = sub {
3408         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3409
3410         return if !$volid;
3411
3412         $volhash->{$volid}->{cdrom} //= 1;
3413         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3414
3415         $volhash->{$volid}->{replicate} //= 0;
3416         $volhash->{$volid}->{replicate} = 1 if $replicate;
3417
3418         $volhash->{$volid}->{shared} //= 0;
3419         $volhash->{$volid}->{shared} = 1 if $shared;
3420
3421         $volhash->{$volid}->{referenced_in_config} //= 0;
3422         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3423
3424         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3425             if defined($snapname);
3426         $volhash->{$volid}->{size} = $size if $size;
3427     };
3428
3429     foreach_drive($conf, sub {
3430         my ($ds, $drive) = @_;
3431         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3432     });
3433
3434     foreach my $snapname (keys %{$conf->{snapshots}}) {
3435         my $snap = $conf->{snapshots}->{$snapname};
3436         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3437         foreach_drive($snap, sub {
3438             my ($ds, $drive) = @_;
3439             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3440         });
3441     }
3442
3443     foreach my $volid (keys %$volhash) {
3444         &$func($volid, $volhash->{$volid}, @param);
3445     }
3446 }
3447
3448 sub conf_has_serial {
3449     my ($conf) = @_;
3450
3451     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3452         if ($conf->{"serial$i"}) {
3453             return 1;
3454         }
3455     }
3456
3457     return 0;
3458 }
3459
3460 sub conf_has_audio {
3461     my ($conf, $id) = @_;
3462
3463     $id //= 0;
3464     my $audio = $conf->{"audio$id"};
3465     return undef if !defined($audio);
3466
3467     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3468     my $audiodriver = $audioproperties->{driver} // 'spice';
3469
3470     return {
3471         dev => $audioproperties->{device},
3472         dev_id => "audiodev$id",
3473         backend => $audiodriver,
3474         backend_id => "$audiodriver-backend${id}",
3475     };
3476 }
3477
3478 sub vga_conf_has_spice {
3479     my ($vga) = @_;
3480
3481     my $vgaconf = parse_vga($vga);
3482     my $vgatype = $vgaconf->{type};
3483     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3484
3485     return $1 || 1;
3486 }
3487
3488 my $host_arch; # FIXME: fix PVE::Tools::get_host_arch
3489 sub get_host_arch() {
3490     $host_arch = (POSIX::uname())[4] if !$host_arch;
3491     return $host_arch;
3492 }
3493
3494 sub is_native($) {
3495     my ($arch) = @_;
3496     return get_host_arch() eq $arch;
3497 }
3498
3499 my $default_machines = {
3500     x86_64 => 'pc',
3501     aarch64 => 'virt',
3502 };
3503
3504 sub get_basic_machine_info {
3505     my ($conf, $forcemachine) = @_;
3506
3507     my $arch = $conf->{arch} // get_host_arch();
3508     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3509     return ($arch, $machine);
3510 }
3511
3512 sub get_ovmf_files($) {
3513     my ($arch) = @_;
3514
3515     my $ovmf = $OVMF->{$arch}
3516         or die "no OVMF images known for architecture '$arch'\n";
3517
3518     return @$ovmf;
3519 }
3520
3521 my $Arch2Qemu = {
3522     aarch64 => '/usr/bin/qemu-system-aarch64',
3523     x86_64 => '/usr/bin/qemu-system-x86_64',
3524 };
3525 sub get_command_for_arch($) {
3526     my ($arch) = @_;
3527     return '/usr/bin/kvm' if is_native($arch);
3528
3529     my $cmd = $Arch2Qemu->{$arch}
3530         or die "don't know how to emulate architecture '$arch'\n";
3531     return $cmd;
3532 }
3533
3534 sub get_cpu_options {
3535     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3536
3537     my $cpuFlags = [];
3538     my $ostype = $conf->{ostype};
3539
3540     my $cpu = $kvm ? "kvm64" : "qemu64";
3541     if ($arch eq 'aarch64') {
3542         $cpu = 'cortex-a57';
3543     }
3544     my $hv_vendor_id;
3545     if (my $cputype = $conf->{cpu}) {
3546         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3547             or die "Cannot parse cpu description: $cputype\n";
3548         $cpu = $cpuconf->{cputype};
3549         $kvm_off = 1 if $cpuconf->{hidden};
3550         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3551
3552         if (defined(my $flags = $cpuconf->{flags})) {
3553             push @$cpuFlags, split(";", $flags);
3554         }
3555     }
3556
3557     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3558
3559     push @$cpuFlags , '-x2apic'
3560         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3561
3562     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3563
3564     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3565
3566     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3567
3568         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3569         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3570     }
3571
3572     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3573
3574     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3575
3576     push @$cpuFlags, 'kvm=off' if $kvm_off;
3577
3578     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3579         push @$cpuFlags, "vendor=${cpu_vendor}"
3580             if $cpu_vendor ne 'default';
3581     } elsif ($arch ne 'aarch64') {
3582         die "internal error"; # should not happen
3583     }
3584
3585     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3586
3587     return ('-cpu', $cpu);
3588 }
3589
3590 sub config_to_command {
3591     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3592
3593     my $cmd = [];
3594     my $globalFlags = [];
3595     my $machineFlags = [];
3596     my $rtcFlags = [];
3597     my $devices = [];
3598     my $pciaddr = '';
3599     my $bridges = {};
3600     my $vernum = 0; # unknown
3601     my $ostype = $conf->{ostype};
3602     my $winversion = windows_version($ostype);
3603     my $kvm = $conf->{kvm};
3604
3605     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3606     my $kvm_binary = get_command_for_arch($arch);
3607     my $kvmver = kvm_user_version($kvm_binary);
3608     $kvm //= 1 if is_native($arch);
3609
3610     if ($kvm) {
3611         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3612             if !defined kvm_version();
3613     }
3614
3615     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3616         $vernum = $1*1000000+$2*1000;
3617     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3618         $vernum = $1*1000000+$2*1000+$3;
3619     }
3620
3621     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3622
3623     my $have_ovz = -f '/proc/vz/vestat';
3624
3625     my $q35 = machine_type_is_q35($conf);
3626     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3627     my $use_old_bios_files = undef;
3628     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3629
3630     my $cpuunits = defined($conf->{cpuunits}) ?
3631             $conf->{cpuunits} : $defaults->{cpuunits};
3632
3633     push @$cmd, $kvm_binary;
3634
3635     push @$cmd, '-id', $vmid;
3636
3637     my $vmname = $conf->{name} || "vm$vmid";
3638
3639     push @$cmd, '-name', $vmname;
3640
3641     my $use_virtio = 0;
3642
3643     my $qmpsocket = qmp_socket($vmid);
3644     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3645     push @$cmd, '-mon', "chardev=qmp,mode=control";
3646
3647     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3648         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3649         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3650     }
3651
3652     push @$cmd, '-pidfile' , pidfile_name($vmid);
3653
3654     push @$cmd, '-daemonize';
3655
3656     if ($conf->{smbios1}) {
3657         my $smbios_conf = parse_smbios1($conf->{smbios1});
3658         if ($smbios_conf->{base64}) {
3659             # Do not pass base64 flag to qemu
3660             delete $smbios_conf->{base64};
3661             my $smbios_string = "";
3662             foreach my $key (keys %$smbios_conf) {
3663                 my $value;
3664                 if ($key eq "uuid") {
3665                     $value = $smbios_conf->{uuid}
3666                 } else {
3667                     $value = decode_base64($smbios_conf->{$key});
3668                 }
3669                 # qemu accepts any binary data, only commas need escaping by double comma
3670                 $value =~ s/,/,,/g;
3671                 $smbios_string .= "," . $key . "=" . $value if $value;
3672             }
3673             push @$cmd, '-smbios', "type=1" . $smbios_string;
3674         } else {
3675             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3676         }
3677     }
3678
3679     if ($conf->{vmgenid}) {
3680         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3681     }
3682
3683     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3684     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3685         die "uefi base image not found\n" if ! -f $ovmf_code;
3686
3687         my $path;
3688         my $format;
3689         if (my $efidisk = $conf->{efidisk0}) {
3690             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3691             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3692             $format = $d->{format};
3693             if ($storeid) {
3694                 $path = PVE::Storage::path($storecfg, $d->{file});
3695                 if (!defined($format)) {
3696                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3697                     $format = qemu_img_format($scfg, $volname);
3698                 }
3699             } else {
3700                 $path = $d->{file};
3701                 die "efidisk format must be specified\n"
3702                     if !defined($format);
3703             }
3704         } else {
3705             warn "no efidisk configured! Using temporary efivars disk.\n";
3706             $path = "/tmp/$vmid-ovmf.fd";
3707             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3708             $format = 'raw';
3709         }
3710
3711         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3712         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3713     }
3714
3715     # load q35 config
3716     if ($q35) {
3717         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3718         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3719             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3720         } else {
3721             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3722         }
3723     }
3724
3725     # add usb controllers
3726     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3727     push @$devices, @usbcontrollers if @usbcontrollers;
3728     my $vga = parse_vga($conf->{vga});
3729
3730     my $qxlnum = vga_conf_has_spice($conf->{vga});
3731     $vga->{type} = 'qxl' if $qxlnum;
3732
3733     if (!$vga->{type}) {
3734         if ($arch eq 'aarch64') {
3735             $vga->{type} = 'virtio';
3736         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3737             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3738         } else {
3739             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3740         }
3741     }
3742
3743     # enable absolute mouse coordinates (needed by vnc)
3744     my $tablet;
3745     if (defined($conf->{tablet})) {
3746         $tablet = $conf->{tablet};
3747     } else {
3748         $tablet = $defaults->{tablet};
3749         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3750         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3751     }
3752
3753     if ($tablet) {
3754         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3755         my $kbd = print_keyboarddevice_full($conf, $arch);
3756         push @$devices, '-device', $kbd if defined($kbd);
3757     }
3758
3759     my $kvm_off = 0;
3760     my $gpu_passthrough;
3761
3762     # host pci devices
3763     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3764         my $d = parse_hostpci($conf->{"hostpci$i"});
3765         next if !$d;
3766
3767         my $pcie = $d->{pcie};
3768         if ($pcie) {
3769             die "q35 machine model is not enabled" if !$q35;
3770             # win7 wants to have the pcie devices directly on the pcie bus
3771             # instead of in the root port
3772             if ($winversion == 7) {
3773                 $pciaddr = print_pcie_addr("hostpci${i}bus0");
3774             } else {
3775                 $pciaddr = print_pcie_addr("hostpci$i");
3776             }
3777         } else {
3778             $pciaddr = print_pci_addr("hostpci$i", $bridges, $arch, $machine_type);
3779         }
3780
3781         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3782         my $romfile = $d->{romfile};
3783
3784         my $xvga = '';
3785         if ($d->{'x-vga'}) {
3786             $xvga = ',x-vga=on';
3787             $kvm_off = 1;
3788             $vga->{type} = 'none' if !defined($conf->{vga});
3789             $gpu_passthrough = 1;
3790
3791             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3792                 $xvga = "";
3793             }
3794         }
3795         my $pcidevices = $d->{pciid};
3796         my $multifunction = 1 if @$pcidevices > 1;
3797         my $sysfspath;
3798         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3799             my $id = $pcidevices->[0]->{id};
3800             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3801             $sysfspath = "/sys/bus/pci/devices/0000:$id/$uuid";
3802         } elsif ($d->{mdev}) {
3803             warn "ignoring mediated device with multifunction device\n";
3804         }
3805
3806         my $j=0;
3807         foreach my $pcidevice (@$pcidevices) {
3808
3809             my $id = "hostpci$i";
3810             $id .= ".$j" if $multifunction;
3811             my $addr = $pciaddr;
3812             $addr .= ".$j" if $multifunction;
3813             my $devicestr = "vfio-pci";
3814             if ($sysfspath) {
3815                 $devicestr .= ",sysfsdev=$sysfspath";
3816             } else {
3817                 $devicestr .= ",host=$pcidevice->{id}";
3818             }
3819             $devicestr .= ",id=$id$addr";
3820
3821             if($j == 0){
3822                 $devicestr .= "$rombar$xvga";
3823                 $devicestr .= ",multifunction=on" if $multifunction;
3824                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3825             }
3826
3827             push @$devices, '-device', $devicestr;
3828             $j++;
3829         }
3830     }
3831
3832     # usb devices
3833     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3834     push @$devices, @usbdevices if @usbdevices;
3835     # serial devices
3836     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3837         if (my $path = $conf->{"serial$i"}) {
3838             if ($path eq 'socket') {
3839                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3840                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3841                 # On aarch64, serial0 is the UART device. Qemu only allows
3842                 # connecting UART devices via the '-serial' command line, as
3843                 # the device has a fixed slot on the hardware...
3844                 if ($arch eq 'aarch64' && $i == 0) {
3845                     push @$devices, '-serial', "chardev:serial$i";
3846                 } else {
3847                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3848                 }
3849             } else {
3850                 die "no such serial device\n" if ! -c $path;
3851                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3852                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3853             }
3854         }
3855     }
3856
3857     # parallel devices
3858     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3859         if (my $path = $conf->{"parallel$i"}) {
3860             die "no such parallel device\n" if ! -c $path;
3861             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3862             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3863             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3864         }
3865     }
3866
3867     if (my $audio = conf_has_audio($conf)) {
3868
3869         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3870
3871         my $id = $audio->{dev_id};
3872         if ($audio->{dev} eq 'AC97') {
3873             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3874         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3875             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3876             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3877             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3878         } else {
3879             die "unkown audio device '$audio->{dev}', implement me!";
3880         }
3881
3882         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3883     }
3884
3885     my $sockets = 1;
3886     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3887     $sockets = $conf->{sockets} if  $conf->{sockets};
3888
3889     my $cores = $conf->{cores} || 1;
3890
3891     my $maxcpus = $sockets * $cores;
3892
3893     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3894
3895     my $allowed_vcpus = $cpuinfo->{cpus};
3896
3897     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3898         if ($allowed_vcpus < $maxcpus);
3899
3900     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3901
3902         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3903         for (my $i = 2; $i <= $vcpus; $i++)  {
3904             my $cpustr = print_cpu_device($conf,$i);
3905             push @$cmd, '-device', $cpustr;
3906         }
3907
3908     } else {
3909
3910         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3911     }
3912     push @$cmd, '-nodefaults';
3913
3914     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3915
3916     my $bootindex_hash = {};
3917     my $i = 1;
3918     foreach my $o (split(//, $bootorder)) {
3919         $bootindex_hash->{$o} = $i*100;
3920         $i++;
3921     }
3922
3923     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3924
3925     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3926
3927     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3928
3929     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3930         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3931         my $socket = vnc_socket($vmid);
3932         push @$cmd,  '-vnc', "unix:$socket,password";
3933     } else {
3934         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3935         push @$cmd, '-nographic';
3936     }
3937
3938     # time drift fix
3939     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3940
3941     my $useLocaltime = $conf->{localtime};
3942
3943     if ($winversion >= 5) { # windows
3944         $useLocaltime = 1 if !defined($conf->{localtime});
3945
3946         # use time drift fix when acpi is enabled
3947         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3948             $tdf = 1 if !defined($conf->{tdf});
3949         }
3950     }
3951
3952     if ($winversion >= 6) {
3953         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3954         push @$cmd, '-no-hpet';
3955     }
3956
3957     push @$rtcFlags, 'driftfix=slew' if $tdf;
3958
3959     if (!$kvm) {
3960         push @$machineFlags, 'accel=tcg';
3961     }
3962
3963     if ($machine_type) {
3964         push @$machineFlags, "type=${machine_type}";
3965     }
3966
3967     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3968         push @$rtcFlags, "base=$conf->{startdate}";
3969     } elsif ($useLocaltime) {
3970         push @$rtcFlags, 'base=localtime';
3971     }
3972
3973     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3974
3975     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3976
3977     push @$cmd, '-S' if $conf->{freeze};
3978
3979     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3980
3981     if (parse_guest_agent($conf)->{enabled}) {
3982         my $qgasocket = qmp_socket($vmid, 1);
3983         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3984         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3985         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3986         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3987     }
3988
3989     my $spice_port;
3990
3991     if ($qxlnum) {
3992         if ($qxlnum > 1) {
3993             if ($winversion){
3994                 for(my $i = 1; $i < $qxlnum; $i++){
3995                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3996                 }
3997             } else {
3998                 # assume other OS works like Linux
3999                 my ($ram, $vram) = ("134217728", "67108864");
4000                 if ($vga->{memory}) {
4001                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
4002                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
4003                 }
4004                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
4005                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
4006             }
4007         }
4008
4009         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
4010
4011         my $nodename = PVE::INotify::nodename();
4012         my $pfamily = PVE::Tools::get_host_address_family($nodename);
4013         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
4014         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
4015         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
4016         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
4017
4018         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
4019         if ($spice_enhancement->{foldersharing}) {
4020             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
4021             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
4022         }
4023
4024         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
4025         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
4026         push @$devices, '-spice', "$spice_opts";
4027
4028         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
4029         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
4030         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
4031
4032     }
4033
4034     # enable balloon by default, unless explicitly disabled
4035     if (!defined($conf->{balloon}) || $conf->{balloon}) {
4036         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
4037         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
4038     }
4039
4040     if ($conf->{watchdog}) {
4041         my $wdopts = parse_watchdog($conf->{watchdog});
4042         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
4043         my $watchdog = $wdopts->{model} || 'i6300esb';
4044         push @$devices, '-device', "$watchdog$pciaddr";
4045         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
4046     }
4047
4048     my $vollist = [];
4049     my $scsicontroller = {};
4050     my $ahcicontroller = {};
4051     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
4052
4053     # Add iscsi initiator name if available
4054     if (my $initiator = get_initiator_name()) {
4055         push @$devices, '-iscsi', "initiator-name=$initiator";
4056     }
4057
4058     foreach_drive($conf, sub {
4059         my ($ds, $drive) = @_;
4060
4061         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
4062             push @$vollist, $drive->{file};
4063         }
4064
4065         # ignore efidisk here, already added in bios/fw handling code above
4066         return if $drive->{interface} eq 'efidisk';
4067
4068         $use_virtio = 1 if $ds =~ m/^virtio/;
4069
4070         if (drive_is_cdrom ($drive)) {
4071             if ($bootindex_hash->{d}) {
4072                 $drive->{bootindex} = $bootindex_hash->{d};
4073                 $bootindex_hash->{d} += 1;
4074             }
4075         } else {
4076             if ($bootindex_hash->{c}) {
4077                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4078                 $bootindex_hash->{c} += 1;
4079             }
4080         }
4081
4082         if($drive->{interface} eq 'virtio'){
4083            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4084         }
4085
4086         if ($drive->{interface} eq 'scsi') {
4087
4088             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4089
4090             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4091             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4092
4093             my $iothread = '';
4094             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4095                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4096                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4097             } elsif ($drive->{iothread}) {
4098                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4099             }
4100
4101             my $queues = '';
4102             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4103                 $queues = ",num_queues=$drive->{queues}";
4104             }
4105
4106             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4107             $scsicontroller->{$controller}=1;
4108         }
4109
4110         if ($drive->{interface} eq 'sata') {
4111            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4112            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4113            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4114            $ahcicontroller->{$controller}=1;
4115         }
4116
4117         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4118         push @$devices, '-drive',$drive_cmd;
4119         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4120     });
4121
4122     for (my $i = 0; $i < $MAX_NETS; $i++) {
4123          next if !$conf->{"net$i"};
4124          my $d = parse_net($conf->{"net$i"});
4125          next if !$d;
4126
4127          $use_virtio = 1 if $d->{model} eq 'virtio';
4128
4129          if ($bootindex_hash->{n}) {
4130             $d->{bootindex} = $bootindex_hash->{n};
4131             $bootindex_hash->{n} += 1;
4132          }
4133
4134          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4135          push @$devices, '-netdev', $netdevfull;
4136
4137          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4138          push @$devices, '-device', $netdevicefull;
4139     }
4140
4141     if ($conf->{ivshmem}) {
4142         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4143
4144         my $bus;
4145         if ($q35) {
4146             $bus = print_pcie_addr("ivshmem");
4147         } else {
4148             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4149         }
4150
4151         my $ivshmem_name = $ivshmem->{name} // $vmid;
4152         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4153
4154         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4155         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4156     }
4157
4158     if (!$q35) {
4159         # add pci bridges
4160         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4161            $bridges->{1} = 1;
4162            $bridges->{2} = 1;
4163         }
4164
4165         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4166
4167         while (my ($k, $v) = each %$bridges) {
4168             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4169             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4170         }
4171     }
4172
4173     push @$cmd, @$devices;
4174     push @$cmd, '-rtc', join(',', @$rtcFlags)
4175         if scalar(@$rtcFlags);
4176     push @$cmd, '-machine', join(',', @$machineFlags)
4177         if scalar(@$machineFlags);
4178     push @$cmd, '-global', join(',', @$globalFlags)
4179         if scalar(@$globalFlags);
4180
4181     if (my $vmstate = $conf->{vmstate}) {
4182         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4183         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
4184         push @$cmd, '-loadstate', $statepath;
4185     }
4186
4187     # add custom args
4188     if ($conf->{args}) {
4189         my $aa = PVE::Tools::split_args($conf->{args});
4190         push @$cmd, @$aa;
4191     }
4192
4193     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4194 }
4195
4196 sub vnc_socket {
4197     my ($vmid) = @_;
4198     return "${var_run_tmpdir}/$vmid.vnc";
4199 }
4200
4201 sub spice_port {
4202     my ($vmid) = @_;
4203
4204     my $res = vm_mon_cmd($vmid, 'query-spice');
4205
4206     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4207 }
4208
4209 sub qmp_socket {
4210     my ($vmid, $qga, $name) = @_;
4211     my $sockettype = $qga ? 'qga' : 'qmp';
4212     my $ext = $name ? '-'.$name : '';
4213     return "${var_run_tmpdir}/$vmid$ext.$sockettype";
4214 }
4215
4216 sub pidfile_name {
4217     my ($vmid) = @_;
4218     return "${var_run_tmpdir}/$vmid.pid";
4219 }
4220
4221 sub vm_devices_list {
4222     my ($vmid) = @_;
4223
4224     my $res = vm_mon_cmd($vmid, 'query-pci');
4225     my $devices_to_check = [];
4226     my $devices = {};
4227     foreach my $pcibus (@$res) {
4228         push @$devices_to_check, @{$pcibus->{devices}},
4229     }
4230
4231     while (@$devices_to_check) {
4232         my $to_check = [];
4233         for my $d (@$devices_to_check) {
4234             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4235             next if !$d->{'pci_bridge'};
4236
4237             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4238             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4239         }
4240         $devices_to_check = $to_check;
4241     }
4242
4243     my $resblock = vm_mon_cmd($vmid, 'query-block');
4244     foreach my $block (@$resblock) {
4245         if($block->{device} =~ m/^drive-(\S+)/){
4246                 $devices->{$1} = 1;
4247         }
4248     }
4249
4250     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4251     foreach my $mice (@$resmice) {
4252         if ($mice->{name} eq 'QEMU HID Tablet') {
4253             $devices->{tablet} = 1;
4254             last;
4255         }
4256     }
4257
4258     # for usb devices there is no query-usb
4259     # but we can iterate over the entries in
4260     # qom-list path=/machine/peripheral
4261     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4262     foreach my $per (@$resperipheral) {
4263         if ($per->{name} =~ m/^usb\d+$/) {
4264             $devices->{$per->{name}} = 1;
4265         }
4266     }
4267
4268     return $devices;
4269 }
4270
4271 sub vm_deviceplug {
4272     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4273
4274     my $q35 = machine_type_is_q35($conf);
4275
4276     my $devices_list = vm_devices_list($vmid);
4277     return 1 if defined($devices_list->{$deviceid});
4278
4279     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4280
4281     if ($deviceid eq 'tablet') {
4282
4283         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4284
4285     } elsif ($deviceid eq 'keyboard') {
4286
4287         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4288
4289     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4290
4291         die "usb hotplug currently not reliable\n";
4292         # since we can't reliably hot unplug all added usb devices
4293         # and usb passthrough disables live migration
4294         # we disable usb hotplugging for now
4295         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4296
4297     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4298
4299         qemu_iothread_add($vmid, $deviceid, $device);
4300
4301         qemu_driveadd($storecfg, $vmid, $device);
4302         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4303
4304         qemu_deviceadd($vmid, $devicefull);
4305         eval { qemu_deviceaddverify($vmid, $deviceid); };
4306         if (my $err = $@) {
4307             eval { qemu_drivedel($vmid, $deviceid); };
4308             warn $@ if $@;
4309             die $err;
4310         }
4311
4312     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4313
4314
4315         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4316         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4317         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4318
4319         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4320
4321         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4322             qemu_iothread_add($vmid, $deviceid, $device);
4323             $devicefull .= ",iothread=iothread-$deviceid";
4324         }
4325
4326         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4327             $devicefull .= ",num_queues=$device->{queues}";
4328         }
4329
4330         qemu_deviceadd($vmid, $devicefull);
4331         qemu_deviceaddverify($vmid, $deviceid);
4332
4333     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4334
4335         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4336         qemu_driveadd($storecfg, $vmid, $device);
4337
4338         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4339         eval { qemu_deviceadd($vmid, $devicefull); };
4340         if (my $err = $@) {
4341             eval { qemu_drivedel($vmid, $deviceid); };
4342             warn $@ if $@;
4343             die $err;
4344         }
4345
4346     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4347
4348         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4349
4350         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4351         my $use_old_bios_files = undef;
4352         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4353
4354         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4355         qemu_deviceadd($vmid, $netdevicefull);
4356         eval {
4357             qemu_deviceaddverify($vmid, $deviceid);
4358             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4359         };
4360         if (my $err = $@) {
4361             eval { qemu_netdevdel($vmid, $deviceid); };
4362             warn $@ if $@;
4363             die $err;
4364         }
4365
4366     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4367
4368         my $bridgeid = $2;
4369         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4370         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4371
4372         qemu_deviceadd($vmid, $devicefull);
4373         qemu_deviceaddverify($vmid, $deviceid);
4374
4375     } else {
4376         die "can't hotplug device '$deviceid'\n";
4377     }
4378
4379     return 1;
4380 }
4381
4382 # fixme: this should raise exceptions on error!
4383 sub vm_deviceunplug {
4384     my ($vmid, $conf, $deviceid) = @_;
4385
4386     my $devices_list = vm_devices_list($vmid);
4387     return 1 if !defined($devices_list->{$deviceid});
4388
4389     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4390
4391     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4392
4393         qemu_devicedel($vmid, $deviceid);
4394
4395     } elsif ($deviceid =~ m/^usb\d+$/) {
4396
4397         die "usb hotplug currently not reliable\n";
4398         # when unplugging usb devices this way,
4399         # there may be remaining usb controllers/hubs
4400         # so we disable it for now
4401         qemu_devicedel($vmid, $deviceid);
4402         qemu_devicedelverify($vmid, $deviceid);
4403
4404     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4405
4406         qemu_devicedel($vmid, $deviceid);
4407         qemu_devicedelverify($vmid, $deviceid);
4408         qemu_drivedel($vmid, $deviceid);
4409         qemu_iothread_del($conf, $vmid, $deviceid);
4410
4411     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4412
4413         qemu_devicedel($vmid, $deviceid);
4414         qemu_devicedelverify($vmid, $deviceid);
4415         qemu_iothread_del($conf, $vmid, $deviceid);
4416
4417     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4418
4419         qemu_devicedel($vmid, $deviceid);
4420         qemu_drivedel($vmid, $deviceid);
4421         qemu_deletescsihw($conf, $vmid, $deviceid);
4422
4423     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4424
4425         qemu_devicedel($vmid, $deviceid);
4426         qemu_devicedelverify($vmid, $deviceid);
4427         qemu_netdevdel($vmid, $deviceid);
4428
4429     } else {
4430         die "can't unplug device '$deviceid'\n";
4431     }
4432
4433     return 1;
4434 }
4435
4436 sub qemu_deviceadd {
4437     my ($vmid, $devicefull) = @_;
4438
4439     $devicefull = "driver=".$devicefull;
4440     my %options =  split(/[=,]/, $devicefull);
4441
4442     vm_mon_cmd($vmid, "device_add" , %options);
4443 }
4444
4445 sub qemu_devicedel {
4446     my ($vmid, $deviceid) = @_;
4447
4448     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4449 }
4450
4451 sub qemu_iothread_add {
4452     my($vmid, $deviceid, $device) = @_;
4453
4454     if ($device->{iothread}) {
4455         my $iothreads = vm_iothreads_list($vmid);
4456         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4457     }
4458 }
4459
4460 sub qemu_iothread_del {
4461     my($conf, $vmid, $deviceid) = @_;
4462
4463     my $confid = $deviceid;
4464     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4465         $confid = 'scsi' . $1;
4466     }
4467     my $device = parse_drive($confid, $conf->{$confid});
4468     if ($device->{iothread}) {
4469         my $iothreads = vm_iothreads_list($vmid);
4470         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4471     }
4472 }
4473
4474 sub qemu_objectadd {
4475     my($vmid, $objectid, $qomtype) = @_;
4476
4477     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4478
4479     return 1;
4480 }
4481
4482 sub qemu_objectdel {
4483     my($vmid, $objectid) = @_;
4484
4485     vm_mon_cmd($vmid, "object-del", id => $objectid);
4486
4487     return 1;
4488 }
4489
4490 sub qemu_driveadd {
4491     my ($storecfg, $vmid, $device) = @_;
4492
4493     my $drive = print_drive_full($storecfg, $vmid, $device);
4494     $drive =~ s/\\/\\\\/g;
4495     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4496
4497     # If the command succeeds qemu prints: "OK"
4498     return 1 if $ret =~ m/OK/s;
4499
4500     die "adding drive failed: $ret\n";
4501 }
4502
4503 sub qemu_drivedel {
4504     my($vmid, $deviceid) = @_;
4505
4506     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4507     $ret =~ s/^\s+//;
4508
4509     return 1 if $ret eq "";
4510
4511     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4512     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4513
4514     die "deleting drive $deviceid failed : $ret\n";
4515 }
4516
4517 sub qemu_deviceaddverify {
4518     my ($vmid, $deviceid) = @_;
4519
4520     for (my $i = 0; $i <= 5; $i++) {
4521          my $devices_list = vm_devices_list($vmid);
4522          return 1 if defined($devices_list->{$deviceid});
4523          sleep 1;
4524     }
4525
4526     die "error on hotplug device '$deviceid'\n";
4527 }
4528
4529
4530 sub qemu_devicedelverify {
4531     my ($vmid, $deviceid) = @_;
4532
4533     # need to verify that the device is correctly removed as device_del
4534     # is async and empty return is not reliable
4535
4536     for (my $i = 0; $i <= 5; $i++) {
4537          my $devices_list = vm_devices_list($vmid);
4538          return 1 if !defined($devices_list->{$deviceid});
4539          sleep 1;
4540     }
4541
4542     die "error on hot-unplugging device '$deviceid'\n";
4543 }
4544
4545 sub qemu_findorcreatescsihw {
4546     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4547
4548     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4549
4550     my $scsihwid="$controller_prefix$controller";
4551     my $devices_list = vm_devices_list($vmid);
4552
4553     if(!defined($devices_list->{$scsihwid})) {
4554         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4555     }
4556
4557     return 1;
4558 }
4559
4560 sub qemu_deletescsihw {
4561     my ($conf, $vmid, $opt) = @_;
4562
4563     my $device = parse_drive($opt, $conf->{$opt});
4564
4565     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4566         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4567         return 1;
4568     }
4569
4570     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4571
4572     my $devices_list = vm_devices_list($vmid);
4573     foreach my $opt (keys %{$devices_list}) {
4574         if (PVE::QemuServer::is_valid_drivename($opt)) {
4575             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4576             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4577                 return 1;
4578             }
4579         }
4580     }
4581
4582     my $scsihwid="scsihw$controller";
4583
4584     vm_deviceunplug($vmid, $conf, $scsihwid);
4585
4586     return 1;
4587 }
4588
4589 sub qemu_add_pci_bridge {
4590     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4591
4592     my $bridges = {};
4593
4594     my $bridgeid;
4595
4596     print_pci_addr($device, $bridges, $arch, $machine_type);
4597
4598     while (my ($k, $v) = each %$bridges) {
4599         $bridgeid = $k;
4600     }
4601     return 1 if !defined($bridgeid) || $bridgeid < 1;
4602
4603     my $bridge = "pci.$bridgeid";
4604     my $devices_list = vm_devices_list($vmid);
4605
4606     if (!defined($devices_list->{$bridge})) {
4607         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4608     }
4609
4610     return 1;
4611 }
4612
4613 sub qemu_set_link_status {
4614     my ($vmid, $device, $up) = @_;
4615
4616     vm_mon_cmd($vmid, "set_link", name => $device,
4617                up => $up ? JSON::true : JSON::false);
4618 }
4619
4620 sub qemu_netdevadd {
4621     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4622
4623     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4624     my %options =  split(/[=,]/, $netdev);
4625
4626     vm_mon_cmd($vmid, "netdev_add",  %options);
4627     return 1;
4628 }
4629
4630 sub qemu_netdevdel {
4631     my ($vmid, $deviceid) = @_;
4632
4633     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4634 }
4635
4636 sub qemu_usb_hotplug {
4637     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4638
4639     return if !$device;
4640
4641     # remove the old one first
4642     vm_deviceunplug($vmid, $conf, $deviceid);
4643
4644     # check if xhci controller is necessary and available
4645     if ($device->{usb3}) {
4646
4647         my $devicelist = vm_devices_list($vmid);
4648
4649         if (!$devicelist->{xhci}) {
4650             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4651             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4652         }
4653     }
4654     my $d = parse_usb_device($device->{host});
4655     $d->{usb3} = $device->{usb3};
4656
4657     # add the new one
4658     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4659 }
4660
4661 sub qemu_cpu_hotplug {
4662     my ($vmid, $conf, $vcpus) = @_;
4663
4664     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4665
4666     my $sockets = 1;
4667     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4668     $sockets = $conf->{sockets} if  $conf->{sockets};
4669     my $cores = $conf->{cores} || 1;
4670     my $maxcpus = $sockets * $cores;
4671
4672     $vcpus = $maxcpus if !$vcpus;
4673
4674     die "you can't add more vcpus than maxcpus\n"
4675         if $vcpus > $maxcpus;
4676
4677     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4678
4679     if ($vcpus < $currentvcpus) {
4680
4681         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4682
4683             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4684                 qemu_devicedel($vmid, "cpu$i");
4685                 my $retry = 0;
4686                 my $currentrunningvcpus = undef;
4687                 while (1) {
4688                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4689                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4690                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4691                     $retry++;
4692                     sleep 1;
4693                 }
4694                 #update conf after each succesfull cpu unplug
4695                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4696                 PVE::QemuConfig->write_config($vmid, $conf);
4697             }
4698         } else {
4699             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4700         }
4701
4702         return;
4703     }
4704
4705     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4706     die "vcpus in running vm does not match its configuration\n"
4707         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4708
4709     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4710
4711         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4712             my $cpustr = print_cpu_device($conf, $i);
4713             qemu_deviceadd($vmid, $cpustr);
4714
4715             my $retry = 0;
4716             my $currentrunningvcpus = undef;
4717             while (1) {
4718                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4719                 last if scalar(@{$currentrunningvcpus}) == $i;
4720                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4721                 sleep 1;
4722                 $retry++;
4723             }
4724             #update conf after each succesfull cpu hotplug
4725             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4726             PVE::QemuConfig->write_config($vmid, $conf);
4727         }
4728     } else {
4729
4730         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4731             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4732         }
4733     }
4734 }
4735
4736 sub qemu_block_set_io_throttle {
4737     my ($vmid, $deviceid,
4738         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4739         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4740         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4741         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4742
4743     return if !check_running($vmid) ;
4744
4745     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4746         bps => int($bps),
4747         bps_rd => int($bps_rd),
4748         bps_wr => int($bps_wr),
4749         iops => int($iops),
4750         iops_rd => int($iops_rd),
4751         iops_wr => int($iops_wr),
4752         bps_max => int($bps_max),
4753         bps_rd_max => int($bps_rd_max),
4754         bps_wr_max => int($bps_wr_max),
4755         iops_max => int($iops_max),
4756         iops_rd_max => int($iops_rd_max),
4757         iops_wr_max => int($iops_wr_max),
4758         bps_max_length => int($bps_max_length),
4759         bps_rd_max_length => int($bps_rd_max_length),
4760         bps_wr_max_length => int($bps_wr_max_length),
4761         iops_max_length => int($iops_max_length),
4762         iops_rd_max_length => int($iops_rd_max_length),
4763         iops_wr_max_length => int($iops_wr_max_length),
4764     );
4765
4766 }
4767
4768 # old code, only used to shutdown old VM after update
4769 sub __read_avail {
4770     my ($fh, $timeout) = @_;
4771
4772     my $sel = new IO::Select;
4773     $sel->add($fh);
4774
4775     my $res = '';
4776     my $buf;
4777
4778     my @ready;
4779     while (scalar (@ready = $sel->can_read($timeout))) {
4780         my $count;
4781         if ($count = $fh->sysread($buf, 8192)) {
4782             if ($buf =~ /^(.*)\(qemu\) $/s) {
4783                 $res .= $1;
4784                 last;
4785             } else {
4786                 $res .= $buf;
4787             }
4788         } else {
4789             if (!defined($count)) {
4790                 die "$!\n";
4791             }
4792             last;
4793         }
4794     }
4795
4796     die "monitor read timeout\n" if !scalar(@ready);
4797
4798     return $res;
4799 }
4800
4801 sub qemu_block_resize {
4802     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4803
4804     my $running = check_running($vmid);
4805
4806     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4807
4808     return if !$running;
4809
4810     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4811
4812 }
4813
4814 sub qemu_volume_snapshot {
4815     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4816
4817     my $running = check_running($vmid);
4818
4819     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4820         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4821     } else {
4822         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4823     }
4824 }
4825
4826 sub qemu_volume_snapshot_delete {
4827     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4828
4829     my $running = check_running($vmid);
4830
4831     if($running) {
4832
4833         $running = undef;
4834         my $conf = PVE::QemuConfig->load_config($vmid);
4835         foreach_drive($conf, sub {
4836             my ($ds, $drive) = @_;
4837             $running = 1 if $drive->{file} eq $volid;
4838         });
4839     }
4840
4841     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4842         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4843     } else {
4844         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4845     }
4846 }
4847
4848 sub set_migration_caps {
4849     my ($vmid) = @_;
4850
4851     my $cap_ref = [];
4852
4853     my $enabled_cap = {
4854         "auto-converge" => 1,
4855         "xbzrle" => 1,
4856         "x-rdma-pin-all" => 0,
4857         "zero-blocks" => 0,
4858         "compress" => 0
4859     };
4860
4861     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4862
4863     for my $supported_capability (@$supported_capabilities) {
4864         push @$cap_ref, {
4865             capability => $supported_capability->{capability},
4866             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4867         };
4868     }
4869
4870     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4871 }
4872
4873 my $fast_plug_option = {
4874     'lock' => 1,
4875     'name' => 1,
4876     'onboot' => 1,
4877     'shares' => 1,
4878     'startup' => 1,
4879     'description' => 1,
4880     'protection' => 1,
4881     'vmstatestorage' => 1,
4882     'hookscript' => 1,
4883 };
4884
4885 # hotplug changes in [PENDING]
4886 # $selection hash can be used to only apply specified options, for
4887 # example: { cores => 1 } (only apply changed 'cores')
4888 # $errors ref is used to return error messages
4889 sub vmconfig_hotplug_pending {
4890     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4891
4892     my $defaults = load_defaults();
4893     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4894
4895     # commit values which do not have any impact on running VM first
4896     # Note: those option cannot raise errors, we we do not care about
4897     # $selection and always apply them.
4898
4899     my $add_error = sub {
4900         my ($opt, $msg) = @_;
4901         $errors->{$opt} = "hotplug problem - $msg";
4902     };
4903
4904     my $changes = 0;
4905     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4906         if ($fast_plug_option->{$opt}) {
4907             $conf->{$opt} = $conf->{pending}->{$opt};
4908             delete $conf->{pending}->{$opt};
4909             $changes = 1;
4910         }
4911     }
4912
4913     if ($changes) {
4914         PVE::QemuConfig->write_config($vmid, $conf);
4915         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4916     }
4917
4918     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4919
4920     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4921     while (my ($opt, $force) = each %$pending_delete_hash) {
4922         next if $selection && !$selection->{$opt};
4923         eval {
4924             if ($opt eq 'hotplug') {
4925                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4926             } elsif ($opt eq 'tablet') {
4927                 die "skip\n" if !$hotplug_features->{usb};
4928                 if ($defaults->{tablet}) {
4929                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4930                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4931                         if $arch eq 'aarch64';
4932                 } else {
4933                     vm_deviceunplug($vmid, $conf, 'tablet');
4934                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4935                 }
4936             } elsif ($opt =~ m/^usb\d+/) {
4937                 die "skip\n";
4938                 # since we cannot reliably hot unplug usb devices
4939                 # we are disabling it
4940                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4941                 vm_deviceunplug($vmid, $conf, $opt);
4942             } elsif ($opt eq 'vcpus') {
4943                 die "skip\n" if !$hotplug_features->{cpu};
4944                 qemu_cpu_hotplug($vmid, $conf, undef);
4945             } elsif ($opt eq 'balloon') {
4946                 # enable balloon device is not hotpluggable
4947                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4948                 # here we reset the ballooning value to memory
4949                 my $balloon = $conf->{memory} || $defaults->{memory};
4950                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4951             } elsif ($fast_plug_option->{$opt}) {
4952                 # do nothing
4953             } elsif ($opt =~ m/^net(\d+)$/) {
4954                 die "skip\n" if !$hotplug_features->{network};
4955                 vm_deviceunplug($vmid, $conf, $opt);
4956             } elsif (is_valid_drivename($opt)) {
4957                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4958                 vm_deviceunplug($vmid, $conf, $opt);
4959                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4960             } elsif ($opt =~ m/^memory$/) {
4961                 die "skip\n" if !$hotplug_features->{memory};
4962                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4963             } elsif ($opt eq 'cpuunits') {
4964                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4965             } elsif ($opt eq 'cpulimit') {
4966                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4967             } else {
4968                 die "skip\n";
4969             }
4970         };
4971         if (my $err = $@) {
4972             &$add_error($opt, $err) if $err ne "skip\n";
4973         } else {
4974             # save new config if hotplug was successful
4975             delete $conf->{$opt};
4976             vmconfig_undelete_pending_option($conf, $opt);
4977             PVE::QemuConfig->write_config($vmid, $conf);
4978             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4979         }
4980     }
4981
4982     my $apply_pending_cloudinit;
4983     $apply_pending_cloudinit = sub {
4984         my ($key, $value) = @_;
4985         $apply_pending_cloudinit = sub {}; # once is enough
4986
4987         my @cloudinit_opts = keys %$confdesc_cloudinit;
4988         foreach my $opt (keys %{$conf->{pending}}) {
4989             next if !grep { $_ eq $opt } @cloudinit_opts;
4990             $conf->{$opt} = delete $conf->{pending}->{$opt};
4991         }
4992
4993         my $new_conf = { %$conf };
4994         $new_conf->{$key} = $value;
4995         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4996     };
4997
4998     foreach my $opt (keys %{$conf->{pending}}) {
4999         next if $selection && !$selection->{$opt};
5000         my $value = $conf->{pending}->{$opt};
5001         eval {
5002             if ($opt eq 'hotplug') {
5003                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
5004             } elsif ($opt eq 'tablet') {
5005                 die "skip\n" if !$hotplug_features->{usb};
5006                 if ($value == 1) {
5007                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
5008                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
5009                         if $arch eq 'aarch64';
5010                 } elsif ($value == 0) {
5011                     vm_deviceunplug($vmid, $conf, 'tablet');
5012                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
5013                 }
5014             } elsif ($opt =~ m/^usb\d+$/) {
5015                 die "skip\n";
5016                 # since we cannot reliably hot unplug usb devices
5017                 # we are disabling it
5018                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
5019                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
5020                 die "skip\n" if !$d;
5021                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
5022             } elsif ($opt eq 'vcpus') {
5023                 die "skip\n" if !$hotplug_features->{cpu};
5024                 qemu_cpu_hotplug($vmid, $conf, $value);
5025             } elsif ($opt eq 'balloon') {
5026                 # enable/disable balloning device is not hotpluggable
5027                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
5028                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
5029                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
5030
5031                 # allow manual ballooning if shares is set to zero
5032                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
5033                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
5034                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
5035                 }
5036             } elsif ($opt =~ m/^net(\d+)$/) {
5037                 # some changes can be done without hotplug
5038                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
5039                                     $vmid, $opt, $value, $arch, $machine_type);
5040             } elsif (is_valid_drivename($opt)) {
5041                 # some changes can be done without hotplug
5042                 my $drive = parse_drive($opt, $value);
5043                 if (drive_is_cloudinit($drive)) {
5044                     &$apply_pending_cloudinit($opt, $value);
5045                 }
5046                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
5047                                      $vmid, $opt, $value, 1, $arch, $machine_type);
5048             } elsif ($opt =~ m/^memory$/) { #dimms
5049                 die "skip\n" if !$hotplug_features->{memory};
5050                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
5051             } elsif ($opt eq 'cpuunits') {
5052                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
5053             } elsif ($opt eq 'cpulimit') {
5054                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
5055                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
5056             } else {
5057                 die "skip\n";  # skip non-hot-pluggable options
5058             }
5059         };
5060         if (my $err = $@) {
5061             &$add_error($opt, $err) if $err ne "skip\n";
5062         } else {
5063             # save new config if hotplug was successful
5064             $conf->{$opt} = $value;
5065             delete $conf->{pending}->{$opt};
5066             PVE::QemuConfig->write_config($vmid, $conf);
5067             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5068         }
5069     }
5070 }
5071
5072 sub try_deallocate_drive {
5073     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5074
5075     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5076         my $volid = $drive->{file};
5077         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5078             my $sid = PVE::Storage::parse_volume_id($volid);
5079             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5080
5081             # check if the disk is really unused
5082             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5083                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5084             PVE::Storage::vdisk_free($storecfg, $volid);
5085             return 1;
5086         } else {
5087             # If vm is not owner of this disk remove from config
5088             return 1;
5089         }
5090     }
5091
5092     return undef;
5093 }
5094
5095 sub vmconfig_delete_or_detach_drive {
5096     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5097
5098     my $drive = parse_drive($opt, $conf->{$opt});
5099
5100     my $rpcenv = PVE::RPCEnvironment::get();
5101     my $authuser = $rpcenv->get_user();
5102
5103     if ($force) {
5104         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5105         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5106     } else {
5107         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5108     }
5109 }
5110
5111 sub vmconfig_apply_pending {
5112     my ($vmid, $conf, $storecfg) = @_;
5113
5114     # cold plug
5115
5116     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
5117     while (my ($opt, $force) = each %$pending_delete_hash) {
5118         die "internal error" if $opt =~ m/^unused/;
5119         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5120         if (!defined($conf->{$opt})) {
5121             vmconfig_undelete_pending_option($conf, $opt);
5122             PVE::QemuConfig->write_config($vmid, $conf);
5123         } elsif (is_valid_drivename($opt)) {
5124             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5125             vmconfig_undelete_pending_option($conf, $opt);
5126             delete $conf->{$opt};
5127             PVE::QemuConfig->write_config($vmid, $conf);
5128         } else {
5129             vmconfig_undelete_pending_option($conf, $opt);
5130             delete $conf->{$opt};
5131             PVE::QemuConfig->write_config($vmid, $conf);
5132         }
5133     }
5134
5135     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5136
5137     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5138         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5139
5140         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5141             # skip if nothing changed
5142         } elsif (is_valid_drivename($opt)) {
5143             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5144                 if defined($conf->{$opt});
5145             $conf->{$opt} = $conf->{pending}->{$opt};
5146         } else {
5147             $conf->{$opt} = $conf->{pending}->{$opt};
5148         }
5149
5150         delete $conf->{pending}->{$opt};
5151         PVE::QemuConfig->write_config($vmid, $conf);
5152     }
5153 }
5154
5155 my $safe_num_ne = sub {
5156     my ($a, $b) = @_;
5157
5158     return 0 if !defined($a) && !defined($b);
5159     return 1 if !defined($a);
5160     return 1 if !defined($b);
5161
5162     return $a != $b;
5163 };
5164
5165 my $safe_string_ne = sub {
5166     my ($a, $b) = @_;
5167
5168     return 0 if !defined($a) && !defined($b);
5169     return 1 if !defined($a);
5170     return 1 if !defined($b);
5171
5172     return $a ne $b;
5173 };
5174
5175 sub vmconfig_update_net {
5176     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5177
5178     my $newnet = parse_net($value);
5179
5180     if ($conf->{$opt}) {
5181         my $oldnet = parse_net($conf->{$opt});
5182
5183         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5184             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5185             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5186             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5187
5188             # for non online change, we try to hot-unplug
5189             die "skip\n" if !$hotplug;
5190             vm_deviceunplug($vmid, $conf, $opt);
5191         } else {
5192
5193             die "internal error" if $opt !~ m/net(\d+)/;
5194             my $iface = "tap${vmid}i$1";
5195
5196             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5197                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5198                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5199                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5200                 PVE::Network::tap_unplug($iface);
5201                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5202             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5203                 # Rate can be applied on its own but any change above needs to
5204                 # include the rate in tap_plug since OVS resets everything.
5205                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5206             }
5207
5208             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5209                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5210             }
5211
5212             return 1;
5213         }
5214     }
5215
5216     if ($hotplug) {
5217         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5218     } else {
5219         die "skip\n";
5220     }
5221 }
5222
5223 sub vmconfig_update_disk {
5224     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5225
5226     # fixme: do we need force?
5227
5228     my $drive = parse_drive($opt, $value);
5229
5230     if ($conf->{$opt}) {
5231
5232         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5233
5234             my $media = $drive->{media} || 'disk';
5235             my $oldmedia = $old_drive->{media} || 'disk';
5236             die "unable to change media type\n" if $media ne $oldmedia;
5237
5238             if (!drive_is_cdrom($old_drive)) {
5239
5240                 if ($drive->{file} ne $old_drive->{file}) {
5241
5242                     die "skip\n" if !$hotplug;
5243
5244                     # unplug and register as unused
5245                     vm_deviceunplug($vmid, $conf, $opt);
5246                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5247
5248                 } else {
5249                     # update existing disk
5250
5251                     # skip non hotpluggable value
5252                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5253                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5254                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5255                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5256                         die "skip\n";
5257                     }
5258
5259                     # apply throttle
5260                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5261                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5262                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5263                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5264                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5265                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5266                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5267                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5268                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5269                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5270                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5271                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5272                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5273                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5274                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5275                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5276                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5277                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5278
5279                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5280                                                    ($drive->{mbps} || 0)*1024*1024,
5281                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5282                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5283                                                    $drive->{iops} || 0,
5284                                                    $drive->{iops_rd} || 0,
5285                                                    $drive->{iops_wr} || 0,
5286                                                    ($drive->{mbps_max} || 0)*1024*1024,
5287                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5288                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5289                                                    $drive->{iops_max} || 0,
5290                                                    $drive->{iops_rd_max} || 0,
5291                                                    $drive->{iops_wr_max} || 0,
5292                                                    $drive->{bps_max_length} || 1,
5293                                                    $drive->{bps_rd_max_length} || 1,
5294                                                    $drive->{bps_wr_max_length} || 1,
5295                                                    $drive->{iops_max_length} || 1,
5296                                                    $drive->{iops_rd_max_length} || 1,
5297                                                    $drive->{iops_wr_max_length} || 1);
5298
5299                     }
5300
5301                     return 1;
5302                 }
5303
5304             } else { # cdrom
5305
5306                 if ($drive->{file} eq 'none') {
5307                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5308                     if (drive_is_cloudinit($old_drive)) {
5309                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5310                     }
5311                 } else {
5312                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5313                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5314                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5315                 }
5316
5317                 return 1;
5318             }
5319         }
5320     }
5321
5322     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5323     # hotplug new disks
5324     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5325     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5326 }
5327
5328 sub vm_start {
5329     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5330         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5331
5332     PVE::QemuConfig->lock_config($vmid, sub {
5333         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5334
5335         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5336
5337         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5338
5339         PVE::QemuConfig->check_lock($conf)
5340             if !($skiplock || $is_suspended);
5341
5342         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5343
5344         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5345             vmconfig_apply_pending($vmid, $conf, $storecfg);
5346             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5347         }
5348
5349         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5350
5351         my $defaults = load_defaults();
5352
5353         # set environment variable useful inside network script
5354         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5355
5356         my $local_volumes = {};
5357
5358         if ($targetstorage) {
5359             foreach_drive($conf, sub {
5360                 my ($ds, $drive) = @_;
5361
5362                 return if drive_is_cdrom($drive);
5363
5364                 my $volid = $drive->{file};
5365
5366                 return if !$volid;
5367
5368                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5369
5370                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5371                 return if $scfg->{shared};
5372                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5373             });
5374
5375             my $format = undef;
5376
5377             foreach my $opt (sort keys %$local_volumes) {
5378
5379                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5380                 my $drive = parse_drive($opt, $conf->{$opt});
5381
5382                 #if remote storage is specified, use default format
5383                 if ($targetstorage && $targetstorage ne "1") {
5384                     $storeid = $targetstorage;
5385                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5386                     $format = $defFormat;
5387                 } else {
5388                     #else we use same format than original
5389                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5390                     $format = qemu_img_format($scfg, $volid);
5391                 }
5392
5393                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5394                 my $newdrive = $drive;
5395                 $newdrive->{format} = $format;
5396                 $newdrive->{file} = $newvolid;
5397                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5398                 $local_volumes->{$opt} = $drivestr;
5399                 #pass drive to conf for command line
5400                 $conf->{$opt} = $drivestr;
5401             }
5402         }
5403
5404         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5405
5406         if ($is_suspended) {
5407             # enforce machine type on suspended vm to ensure HW compatibility
5408             $forcemachine = $conf->{runningmachine};
5409             print "Resuming suspended VM\n";
5410         }
5411
5412         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5413
5414         my $migrate_port = 0;
5415         my $migrate_uri;
5416         if ($statefile) {
5417             if ($statefile eq 'tcp') {
5418                 my $localip = "localhost";
5419                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5420                 my $nodename = PVE::INotify::nodename();
5421
5422                 if (!defined($migration_type)) {
5423                     if (defined($datacenterconf->{migration}->{type})) {
5424                         $migration_type = $datacenterconf->{migration}->{type};
5425                     } else {
5426                         $migration_type = 'secure';
5427                     }
5428                 }
5429
5430                 if ($migration_type eq 'insecure') {
5431                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5432                     if ($migrate_network_addr) {
5433                         $localip = $migrate_network_addr;
5434                     } else {
5435                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5436                     }
5437
5438                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5439                 }
5440
5441                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5442                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5443                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5444                 push @$cmd, '-incoming', $migrate_uri;
5445                 push @$cmd, '-S';
5446
5447             } elsif ($statefile eq 'unix') {
5448                 # should be default for secure migrations as a ssh TCP forward
5449                 # tunnel is not deterministic reliable ready and fails regurarly
5450                 # to set up in time, so use UNIX socket forwards
5451                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5452                 unlink $socket_addr;
5453
5454                 $migrate_uri = "unix:$socket_addr";
5455
5456                 push @$cmd, '-incoming', $migrate_uri;
5457                 push @$cmd, '-S';
5458
5459             } else {
5460                 push @$cmd, '-loadstate', $statefile;
5461             }
5462         } elsif ($paused) {
5463             push @$cmd, '-S';
5464         }
5465
5466         # host pci devices
5467         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5468           my $d = parse_hostpci($conf->{"hostpci$i"});
5469           next if !$d;
5470           my $pcidevices = $d->{pciid};
5471           foreach my $pcidevice (@$pcidevices) {
5472                 my $pciid = $pcidevice->{id};
5473
5474                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5475                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5476                 die "no pci device info for device '$pciid'\n" if !$info;
5477
5478                 if ($d->{mdev}) {
5479                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5480                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5481                 } else {
5482                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5483                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5484                     die "can't reset pci device '$pciid'\n"
5485                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5486                 }
5487           }
5488         }
5489
5490         PVE::Storage::activate_volumes($storecfg, $vollist);
5491
5492         eval {
5493             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5494                 outfunc => sub {}, errfunc => sub {});
5495         };
5496         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5497         # timeout should be more than enough here...
5498         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5499
5500         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5501                                                   : $defaults->{cpuunits};
5502
5503         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5504         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5505
5506         my %properties = (
5507             Slice => 'qemu.slice',
5508             KillMode => 'none',
5509             CPUShares => $cpuunits
5510         );
5511
5512         if (my $cpulimit = $conf->{cpulimit}) {
5513             $properties{CPUQuota} = int($cpulimit * 100);
5514         }
5515         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5516
5517         my $run_qemu = sub {
5518             PVE::Tools::run_fork sub {
5519                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5520                 run_command($cmd, %run_params);
5521             };
5522         };
5523
5524         if ($conf->{hugepages}) {
5525
5526             my $code = sub {
5527                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5528                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5529
5530                 PVE::QemuServer::Memory::hugepages_mount();
5531                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5532
5533                 eval { $run_qemu->() };
5534                 if (my $err = $@) {
5535                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5536                     die $err;
5537                 }
5538
5539                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5540             };
5541             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5542
5543         } else {
5544             eval { $run_qemu->() };
5545         }
5546
5547         if (my $err = $@) {
5548             # deactivate volumes if start fails
5549             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5550             die "start failed: $err";
5551         }
5552
5553         print "migration listens on $migrate_uri\n" if $migrate_uri;
5554
5555         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5556             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5557             warn $@ if $@;
5558         }
5559
5560         #start nbd server for storage migration
5561         if ($targetstorage) {
5562             my $nodename = PVE::INotify::nodename();
5563             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5564             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5565             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5566             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5567
5568             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
5569
5570             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5571
5572             foreach my $opt (sort keys %$local_volumes) {
5573                 my $volid = $local_volumes->{$opt};
5574                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5575                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
5576                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5577             }
5578         }
5579
5580         if ($migratedfrom) {
5581             eval {
5582                 set_migration_caps($vmid);
5583             };
5584             warn $@ if $@;
5585
5586             if ($spice_port) {
5587                 print "spice listens on port $spice_port\n";
5588                 if ($spice_ticket) {
5589                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5590                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5591                 }
5592             }
5593
5594         } else {
5595             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5596                 if !$statefile && $conf->{balloon};
5597
5598             foreach my $opt (keys %$conf) {
5599                 next if $opt !~  m/^net\d+$/;
5600                 my $nicconf = parse_net($conf->{$opt});
5601                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5602             }
5603         }
5604
5605         vm_mon_cmd_nocheck($vmid, 'qom-set',
5606                     path => "machine/peripheral/balloon0",
5607                     property => "guest-stats-polling-interval",
5608                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5609
5610         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5611             print "Resumed VM, removing state\n";
5612             delete $conf->@{qw(lock vmstate runningmachine)};
5613             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5614             PVE::Storage::vdisk_free($storecfg, $vmstate);
5615             PVE::QemuConfig->write_config($vmid, $conf);
5616         }
5617
5618         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5619     });
5620 }
5621
5622 sub vm_mon_cmd {
5623     my ($vmid, $execute, %params) = @_;
5624
5625     my $cmd = { execute => $execute, arguments => \%params };
5626     vm_qmp_command($vmid, $cmd);
5627 }
5628
5629 sub vm_mon_cmd_nocheck {
5630     my ($vmid, $execute, %params) = @_;
5631
5632     my $cmd = { execute => $execute, arguments => \%params };
5633     vm_qmp_command($vmid, $cmd, 1);
5634 }
5635
5636 sub vm_qmp_command {
5637     my ($vmid, $cmd, $nocheck) = @_;
5638
5639     my $res;
5640
5641     my $timeout;
5642     if ($cmd->{arguments}) {
5643         $timeout = delete $cmd->{arguments}->{timeout};
5644     }
5645
5646     eval {
5647         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5648         my $sname = qmp_socket($vmid);
5649         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5650             my $qmpclient = PVE::QMPClient->new();
5651
5652             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5653         } else {
5654             die "unable to open monitor socket\n";
5655         }
5656     };
5657     if (my $err = $@) {
5658         syslog("err", "VM $vmid qmp command failed - $err");
5659         die $err;
5660     }
5661
5662     return $res;
5663 }
5664
5665 sub vm_human_monitor_command {
5666     my ($vmid, $cmdline) = @_;
5667
5668     my $cmd = {
5669         execute => 'human-monitor-command',
5670         arguments => { 'command-line' => $cmdline},
5671     };
5672
5673     return vm_qmp_command($vmid, $cmd);
5674 }
5675
5676 sub vm_commandline {
5677     my ($storecfg, $vmid, $snapname) = @_;
5678
5679     my $conf = PVE::QemuConfig->load_config($vmid);
5680
5681     if ($snapname) {
5682         my $snapshot = $conf->{snapshots}->{$snapname};
5683         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5684
5685         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5686
5687         $conf = $snapshot;
5688     }
5689
5690     my $defaults = load_defaults();
5691
5692     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5693
5694     return PVE::Tools::cmd2string($cmd);
5695 }
5696
5697 sub vm_reset {
5698     my ($vmid, $skiplock) = @_;
5699
5700     PVE::QemuConfig->lock_config($vmid, sub {
5701
5702         my $conf = PVE::QemuConfig->load_config($vmid);
5703
5704         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5705
5706         vm_mon_cmd($vmid, "system_reset");
5707     });
5708 }
5709
5710 sub get_vm_volumes {
5711     my ($conf) = @_;
5712
5713     my $vollist = [];
5714     foreach_volid($conf, sub {
5715         my ($volid, $attr) = @_;
5716
5717         return if $volid =~ m|^/|;
5718
5719         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5720         return if !$sid;
5721
5722         push @$vollist, $volid;
5723     });
5724
5725     return $vollist;
5726 }
5727
5728 sub vm_stop_cleanup {
5729     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5730
5731     eval {
5732
5733         if (!$keepActive) {
5734             my $vollist = get_vm_volumes($conf);
5735             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5736         }
5737
5738         foreach my $ext (qw(mon qmp pid vnc qga)) {
5739             unlink "/var/run/qemu-server/${vmid}.$ext";
5740         }
5741
5742         if ($conf->{ivshmem}) {
5743             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5744             # just delete it for now, VMs which have this already open do not
5745             # are affected, but new VMs will get a separated one. If this
5746             # becomes an issue we either add some sort of ref-counting or just
5747             # add a "don't delete on stop" flag to the ivshmem format.
5748             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5749         }
5750
5751         foreach my $key (keys %$conf) {
5752             next if $key !~ m/^hostpci(\d+)$/;
5753             my $hostpciindex = $1;
5754             my $d = parse_hostpci($conf->{$key});
5755             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5756
5757             foreach my $pci (@{$d->{pciid}}) {
5758                 my $pciid = $pci->{id};
5759                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5760             }
5761         }
5762
5763         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5764     };
5765     warn $@ if $@; # avoid errors - just warn
5766 }
5767
5768 # Note: use $nockeck to skip tests if VM configuration file exists.
5769 # We need that when migration VMs to other nodes (files already moved)
5770 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5771 sub vm_stop {
5772     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5773
5774     $force = 1 if !defined($force) && !$shutdown;
5775
5776     if ($migratedfrom){
5777         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5778         kill 15, $pid if $pid;
5779         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5780         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5781         return;
5782     }
5783
5784     PVE::QemuConfig->lock_config($vmid, sub {
5785
5786         my $pid = check_running($vmid, $nocheck);
5787         return if !$pid;
5788
5789         my $conf;
5790         if (!$nocheck) {
5791             $conf = PVE::QemuConfig->load_config($vmid);
5792             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5793             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5794                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5795                 $timeout = $opts->{down} if $opts->{down};
5796             }
5797             PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5798         }
5799
5800         eval {
5801             if ($shutdown) {
5802                 if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5803                     vm_qmp_command($vmid, {
5804                         execute => "guest-shutdown",
5805                         arguments => { timeout => $timeout }
5806                     }, $nocheck);
5807                 } else {
5808                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5809                 }
5810             } else {
5811                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5812             }
5813         };
5814         my $err = $@;
5815
5816         if (!$err) {
5817             $timeout = 60 if !defined($timeout);
5818
5819             my $count = 0;
5820             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5821                 $count++;
5822                 sleep 1;
5823             }
5824
5825             if ($count >= $timeout) {
5826                 if ($force) {
5827                     warn "VM still running - terminating now with SIGTERM\n";
5828                     kill 15, $pid;
5829                 } else {
5830                     die "VM quit/powerdown failed - got timeout\n";
5831                 }
5832             } else {
5833                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5834                 return;
5835             }
5836         } else {
5837             if ($force) {
5838                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5839                 kill 15, $pid;
5840             } else {
5841                 die "VM quit/powerdown failed\n";
5842             }
5843         }
5844
5845         # wait again
5846         $timeout = 10;
5847
5848         my $count = 0;
5849         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5850             $count++;
5851             sleep 1;
5852         }
5853
5854         if ($count >= $timeout) {
5855             warn "VM still running - terminating now with SIGKILL\n";
5856             kill 9, $pid;
5857             sleep 1;
5858         }
5859
5860         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5861    });
5862 }
5863
5864 sub vm_suspend {
5865     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5866
5867     my $conf;
5868     my $path;
5869     my $storecfg;
5870     my $vmstate;
5871
5872     PVE::QemuConfig->lock_config($vmid, sub {
5873
5874         $conf = PVE::QemuConfig->load_config($vmid);
5875
5876         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5877         PVE::QemuConfig->check_lock($conf)
5878             if !($skiplock || $is_backing_up);
5879
5880         die "cannot suspend to disk during backup\n"
5881             if $is_backing_up && $includestate;
5882
5883         if ($includestate) {
5884             $conf->{lock} = 'suspending';
5885             my $date = strftime("%Y-%m-%d", localtime(time()));
5886             $storecfg = PVE::Storage::config();
5887             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5888             $path = PVE::Storage::path($storecfg, $vmstate);
5889             PVE::QemuConfig->write_config($vmid, $conf);
5890         } else {
5891             vm_mon_cmd($vmid, "stop");
5892         }
5893     });
5894
5895     if ($includestate) {
5896         # save vm state
5897         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5898
5899         eval {
5900             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5901             for(;;) {
5902                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5903                 if (!$state->{status}) {
5904                     die "savevm not active\n";
5905                 } elsif ($state->{status} eq 'active') {
5906                     sleep(1);
5907                     next;
5908                 } elsif ($state->{status} eq 'completed') {
5909                     print "State saved, quitting\n";
5910                     last;
5911                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5912                     die "query-savevm failed with error '$state->{error}'\n"
5913                 } else {
5914                     die "query-savevm returned status '$state->{status}'\n";
5915                 }
5916             }
5917         };
5918         my $err = $@;
5919
5920         PVE::QemuConfig->lock_config($vmid, sub {
5921             $conf = PVE::QemuConfig->load_config($vmid);
5922             if ($err) {
5923                 # cleanup, but leave suspending lock, to indicate something went wrong
5924                 eval {
5925                     vm_mon_cmd($vmid, "savevm-end");
5926                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5927                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5928                     delete $conf->@{qw(vmstate runningmachine)};
5929                     PVE::QemuConfig->write_config($vmid, $conf);
5930                 };
5931                 warn $@ if $@;
5932                 die $err;
5933             }
5934
5935             die "lock changed unexpectedly\n"
5936                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5937
5938             vm_qmp_command($vmid, { execute => "quit" });
5939             $conf->{lock} = 'suspended';
5940             PVE::QemuConfig->write_config($vmid, $conf);
5941         });
5942     }
5943 }
5944
5945 sub vm_resume {
5946     my ($vmid, $skiplock, $nocheck) = @_;
5947
5948     PVE::QemuConfig->lock_config($vmid, sub {
5949         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5950         my $res = $vm_mon_cmd->($vmid, 'query-status');
5951         my $resume_cmd = 'cont';
5952
5953         if ($res->{status} && $res->{status} eq 'suspended') {
5954             $resume_cmd = 'system_wakeup';
5955         }
5956
5957         if (!$nocheck) {
5958
5959             my $conf = PVE::QemuConfig->load_config($vmid);
5960
5961             PVE::QemuConfig->check_lock($conf)
5962                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5963         }
5964
5965         $vm_mon_cmd->($vmid, $resume_cmd);
5966     });
5967 }
5968
5969 sub vm_sendkey {
5970     my ($vmid, $skiplock, $key) = @_;
5971
5972     PVE::QemuConfig->lock_config($vmid, sub {
5973
5974         my $conf = PVE::QemuConfig->load_config($vmid);
5975
5976         # there is no qmp command, so we use the human monitor command
5977         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5978         die $res if $res ne '';
5979     });
5980 }
5981
5982 sub vm_destroy {
5983     my ($storecfg, $vmid, $skiplock) = @_;
5984
5985     PVE::QemuConfig->lock_config($vmid, sub {
5986
5987         my $conf = PVE::QemuConfig->load_config($vmid);
5988
5989         if (!check_running($vmid)) {
5990             destroy_vm($storecfg, $vmid, undef, $skiplock);
5991         } else {
5992             die "VM $vmid is running - destroy failed\n";
5993         }
5994     });
5995 }
5996
5997 # vzdump restore implementaion
5998
5999 sub tar_archive_read_firstfile {
6000     my $archive = shift;
6001
6002     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
6003
6004     # try to detect archive type first
6005     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
6006         die "unable to open file '$archive'\n";
6007     my $firstfile = <$fh>;
6008     kill 15, $pid;
6009     close $fh;
6010
6011     die "ERROR: archive contaions no data\n" if !$firstfile;
6012     chomp $firstfile;
6013
6014     return $firstfile;
6015 }
6016
6017 sub tar_restore_cleanup {
6018     my ($storecfg, $statfile) = @_;
6019
6020     print STDERR "starting cleanup\n";
6021
6022     if (my $fd = IO::File->new($statfile, "r")) {
6023         while (defined(my $line = <$fd>)) {
6024             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6025                 my $volid = $2;
6026                 eval {
6027                     if ($volid =~ m|^/|) {
6028                         unlink $volid || die 'unlink failed\n';
6029                     } else {
6030                         PVE::Storage::vdisk_free($storecfg, $volid);
6031                     }
6032                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
6033                 };
6034                 print STDERR "unable to cleanup '$volid' - $@" if $@;
6035             } else {
6036                 print STDERR "unable to parse line in statfile - $line";
6037             }
6038         }
6039         $fd->close();
6040     }
6041 }
6042
6043 sub restore_archive {
6044     my ($archive, $vmid, $user, $opts) = @_;
6045
6046     my $format = $opts->{format};
6047     my $comp;
6048
6049     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
6050         $format = 'tar' if !$format;
6051         $comp = 'gzip';
6052     } elsif ($archive =~ m/\.tar$/) {
6053         $format = 'tar' if !$format;
6054     } elsif ($archive =~ m/.tar.lzo$/) {
6055         $format = 'tar' if !$format;
6056         $comp = 'lzop';
6057     } elsif ($archive =~ m/\.vma$/) {
6058         $format = 'vma' if !$format;
6059     } elsif ($archive =~ m/\.vma\.gz$/) {
6060         $format = 'vma' if !$format;
6061         $comp = 'gzip';
6062     } elsif ($archive =~ m/\.vma\.lzo$/) {
6063         $format = 'vma' if !$format;
6064         $comp = 'lzop';
6065     } else {
6066         $format = 'vma' if !$format; # default
6067     }
6068
6069     # try to detect archive format
6070     if ($format eq 'tar') {
6071         return restore_tar_archive($archive, $vmid, $user, $opts);
6072     } else {
6073         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6074     }
6075 }
6076
6077 sub restore_update_config_line {
6078     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6079
6080     return if $line =~ m/^\#qmdump\#/;
6081     return if $line =~ m/^\#vzdump\#/;
6082     return if $line =~ m/^lock:/;
6083     return if $line =~ m/^unused\d+:/;
6084     return if $line =~ m/^parent:/;
6085
6086     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6087     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6088         # try to convert old 1.X settings
6089         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6090         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6091             my ($model, $macaddr) = split(/\=/, $devconfig);
6092             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6093             my $net = {
6094                 model => $model,
6095                 bridge => "vmbr$ind",
6096                 macaddr => $macaddr,
6097             };
6098             my $netstr = print_net($net);
6099
6100             print $outfd "net$cookie->{netcount}: $netstr\n";
6101             $cookie->{netcount}++;
6102         }
6103     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6104         my ($id, $netstr) = ($1, $2);
6105         my $net = parse_net($netstr);
6106         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6107         $netstr = print_net($net);
6108         print $outfd "$id: $netstr\n";
6109     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6110         my $virtdev = $1;
6111         my $value = $3;
6112         my $di = parse_drive($virtdev, $value);
6113         if (defined($di->{backup}) && !$di->{backup}) {
6114             print $outfd "#$line";
6115         } elsif ($map->{$virtdev}) {
6116             delete $di->{format}; # format can change on restore
6117             $di->{file} = $map->{$virtdev};
6118             $value = print_drive($vmid, $di);
6119             print $outfd "$virtdev: $value\n";
6120         } else {
6121             print $outfd $line;
6122         }
6123     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6124         my $vmgenid = $1;
6125         if ($vmgenid ne '0') {
6126             # always generate a new vmgenid if there was a valid one setup
6127             $vmgenid = generate_uuid();
6128         }
6129         print $outfd "vmgenid: $vmgenid\n";
6130     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6131         my ($uuid, $uuid_str);
6132         UUID::generate($uuid);
6133         UUID::unparse($uuid, $uuid_str);
6134         my $smbios1 = parse_smbios1($2);
6135         $smbios1->{uuid} = $uuid_str;
6136         print $outfd $1.print_smbios1($smbios1)."\n";
6137     } else {
6138         print $outfd $line;
6139     }
6140 }
6141
6142 sub scan_volids {
6143     my ($cfg, $vmid) = @_;
6144
6145     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6146
6147     my $volid_hash = {};
6148     foreach my $storeid (keys %$info) {
6149         foreach my $item (@{$info->{$storeid}}) {
6150             next if !($item->{volid} && $item->{size});
6151             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6152             $volid_hash->{$item->{volid}} = $item;
6153         }
6154     }
6155
6156     return $volid_hash;
6157 }
6158
6159 sub is_volume_in_use {
6160     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6161
6162     my $path = PVE::Storage::path($storecfg, $volid);
6163
6164     my $scan_config = sub {
6165         my ($cref, $snapname) = @_;
6166
6167         foreach my $key (keys %$cref) {
6168             my $value = $cref->{$key};
6169             if (is_valid_drivename($key)) {
6170                 next if $skip_drive && $key eq $skip_drive;
6171                 my $drive = parse_drive($key, $value);
6172                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6173                 return 1 if $volid eq $drive->{file};
6174                 if ($drive->{file} =~ m!^/!) {
6175                     return 1 if $drive->{file} eq $path;
6176                 } else {
6177                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6178                     next if !$storeid;
6179                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6180                     next if !$scfg;
6181                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6182                 }
6183             }
6184         }
6185
6186         return 0;
6187     };
6188
6189     return 1 if &$scan_config($conf);
6190
6191     undef $skip_drive;
6192
6193     foreach my $snapname (keys %{$conf->{snapshots}}) {
6194         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6195     }
6196
6197     return 0;
6198 }
6199
6200 sub update_disksize {
6201     my ($vmid, $conf, $volid_hash) = @_;
6202
6203     my $changes;
6204     my $prefix = "VM $vmid:";
6205
6206     # used and unused disks
6207     my $referenced = {};
6208
6209     # Note: it is allowed to define multiple storages with same path (alias), so
6210     # we need to check both 'volid' and real 'path' (two different volid can point
6211     # to the same path).
6212
6213     my $referencedpath = {};
6214
6215     # update size info
6216     foreach my $opt (keys %$conf) {
6217         if (is_valid_drivename($opt)) {
6218             my $drive = parse_drive($opt, $conf->{$opt});
6219             my $volid = $drive->{file};
6220             next if !$volid;
6221
6222             $referenced->{$volid} = 1;
6223             if ($volid_hash->{$volid} &&
6224                 (my $path = $volid_hash->{$volid}->{path})) {
6225                 $referencedpath->{$path} = 1;
6226             }
6227
6228             next if drive_is_cdrom($drive);
6229             next if !$volid_hash->{$volid};
6230
6231             $drive->{size} = $volid_hash->{$volid}->{size};
6232             my $new = print_drive($vmid, $drive);
6233             if ($new ne $conf->{$opt}) {
6234                 $changes = 1;
6235                 $conf->{$opt} = $new;
6236                 print "$prefix update disk '$opt' information.\n";
6237             }
6238         }
6239     }
6240
6241     # remove 'unusedX' entry if volume is used
6242     foreach my $opt (keys %$conf) {
6243         next if $opt !~ m/^unused\d+$/;
6244         my $volid = $conf->{$opt};
6245         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6246         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6247             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6248             $changes = 1;
6249             delete $conf->{$opt};
6250         }
6251
6252         $referenced->{$volid} = 1;
6253         $referencedpath->{$path} = 1 if $path;
6254     }
6255
6256     foreach my $volid (sort keys %$volid_hash) {
6257         next if $volid =~ m/vm-$vmid-state-/;
6258         next if $referenced->{$volid};
6259         my $path = $volid_hash->{$volid}->{path};
6260         next if !$path; # just to be sure
6261         next if $referencedpath->{$path};
6262         $changes = 1;
6263         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6264         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6265         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6266     }
6267
6268     return $changes;
6269 }
6270
6271 sub rescan {
6272     my ($vmid, $nolock, $dryrun) = @_;
6273
6274     my $cfg = PVE::Storage::config();
6275
6276     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6277     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6278     foreach my $stor (keys %{$cfg->{ids}}) {
6279         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6280     }
6281
6282     print "rescan volumes...\n";
6283     my $volid_hash = scan_volids($cfg, $vmid);
6284
6285     my $updatefn =  sub {
6286         my ($vmid) = @_;
6287
6288         my $conf = PVE::QemuConfig->load_config($vmid);
6289
6290         PVE::QemuConfig->check_lock($conf);
6291
6292         my $vm_volids = {};
6293         foreach my $volid (keys %$volid_hash) {
6294             my $info = $volid_hash->{$volid};
6295             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6296         }
6297
6298         my $changes = update_disksize($vmid, $conf, $vm_volids);
6299
6300         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6301     };
6302
6303     if (defined($vmid)) {
6304         if ($nolock) {
6305             &$updatefn($vmid);
6306         } else {
6307             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6308         }
6309     } else {
6310         my $vmlist = config_list();
6311         foreach my $vmid (keys %$vmlist) {
6312             if ($nolock) {
6313                 &$updatefn($vmid);
6314             } else {
6315                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6316             }
6317         }
6318     }
6319 }
6320
6321 sub restore_vma_archive {
6322     my ($archive, $vmid, $user, $opts, $comp) = @_;
6323
6324     my $readfrom = $archive;
6325
6326     my $cfg = PVE::Storage::config();
6327     my $commands = [];
6328     my $bwlimit = $opts->{bwlimit};
6329
6330     my $dbg_cmdstring = '';
6331     my $add_pipe = sub {
6332         my ($cmd) = @_;
6333         push @$commands, $cmd;
6334         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6335         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6336         $readfrom = '-';
6337     };
6338
6339     my $input = undef;
6340     if ($archive eq '-') {
6341         $input = '<&STDIN';
6342     } else {
6343         # If we use a backup from a PVE defined storage we also consider that
6344         # storage's rate limit:
6345         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6346         if (defined($volid)) {
6347             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6348             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6349             if ($readlimit) {
6350                 print STDERR "applying read rate limit: $readlimit\n";
6351                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6352                 $add_pipe->($cstream);
6353             }
6354         }
6355     }
6356
6357     if ($comp) {
6358         my $cmd;
6359         if ($comp eq 'gzip') {
6360             $cmd = ['zcat', $readfrom];
6361         } elsif ($comp eq 'lzop') {
6362             $cmd = ['lzop', '-d', '-c', $readfrom];
6363         } else {
6364             die "unknown compression method '$comp'\n";
6365         }
6366         $add_pipe->($cmd);
6367     }
6368
6369     my $tmpdir = "/var/tmp/vzdumptmp$$";
6370     rmtree $tmpdir;
6371
6372     # disable interrupts (always do cleanups)
6373     local $SIG{INT} =
6374         local $SIG{TERM} =
6375         local $SIG{QUIT} =
6376         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6377
6378     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6379     POSIX::mkfifo($mapfifo, 0600);
6380     my $fifofh;
6381
6382     my $openfifo = sub {
6383         open($fifofh, '>', $mapfifo) || die $!;
6384     };
6385
6386     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6387
6388     my $oldtimeout;
6389     my $timeout = 5;
6390
6391     my $devinfo = {};
6392
6393     my $rpcenv = PVE::RPCEnvironment::get();
6394
6395     my $conffile = PVE::QemuConfig->config_file($vmid);
6396     my $tmpfn = "$conffile.$$.tmp";
6397
6398     # Note: $oldconf is undef if VM does not exists
6399     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6400     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6401
6402     my %storage_limits;
6403
6404     my $print_devmap = sub {
6405         my $virtdev_hash = {};
6406
6407         my $cfgfn = "$tmpdir/qemu-server.conf";
6408
6409         # we can read the config - that is already extracted
6410         my $fh = IO::File->new($cfgfn, "r") ||
6411             "unable to read qemu-server.conf - $!\n";
6412
6413         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6414         if (-f $fwcfgfn) {
6415             my $pve_firewall_dir = '/etc/pve/firewall';
6416             mkdir $pve_firewall_dir; # make sure the dir exists
6417             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6418         }
6419
6420         while (defined(my $line = <$fh>)) {
6421             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6422                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6423                 die "archive does not contain data for drive '$virtdev'\n"
6424                     if !$devinfo->{$devname};
6425                 if (defined($opts->{storage})) {
6426                     $storeid = $opts->{storage} || 'local';
6427                 } elsif (!$storeid) {
6428                     $storeid = 'local';
6429                 }
6430                 $format = 'raw' if !$format;
6431                 $devinfo->{$devname}->{devname} = $devname;
6432                 $devinfo->{$devname}->{virtdev} = $virtdev;
6433                 $devinfo->{$devname}->{format} = $format;
6434                 $devinfo->{$devname}->{storeid} = $storeid;
6435
6436                 # check permission on storage
6437                 my $pool = $opts->{pool}; # todo: do we need that?
6438                 if ($user ne 'root@pam') {
6439                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6440                 }
6441
6442                 $storage_limits{$storeid} = $bwlimit;
6443
6444                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6445             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6446                 my $virtdev = $1;
6447                 my $drive = parse_drive($virtdev, $2);
6448                 if (drive_is_cloudinit($drive)) {
6449                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6450                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6451                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6452
6453                     my $d = {
6454                         format => $format,
6455                         storeid => $opts->{storage} // $storeid,
6456                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6457                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6458                         name => "vm-$vmid-cloudinit",
6459                         is_cloudinit => 1,
6460                     };
6461                     $virtdev_hash->{$virtdev} = $d;
6462                 }
6463             }
6464         }
6465
6466         foreach my $key (keys %storage_limits) {
6467             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6468             next if !$limit;
6469             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6470             $storage_limits{$key} = $limit * 1024;
6471         }
6472
6473         foreach my $devname (keys %$devinfo) {
6474             die "found no device mapping information for device '$devname'\n"
6475                 if !$devinfo->{$devname}->{virtdev};
6476         }
6477
6478         # create empty/temp config
6479         if ($oldconf) {
6480             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6481             foreach_drive($oldconf, sub {
6482                 my ($ds, $drive) = @_;
6483
6484                 return if !$drive->{is_cloudinit} && drive_is_cdrom($drive);
6485
6486                 my $volid = $drive->{file};
6487                 return if !$volid || $volid =~ m|^/|;
6488
6489                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6490                 return if !$path || !$owner || ($owner != $vmid);
6491
6492                 # Note: only delete disk we want to restore
6493                 # other volumes will become unused
6494                 if ($virtdev_hash->{$ds}) {
6495                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6496                     if (my $err = $@) {
6497                         warn $err;
6498                     }
6499                 }
6500             });
6501
6502             # delete vmstate files, after the restore we have no snapshots anymore
6503             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6504                 my $snap = $oldconf->{snapshots}->{$snapname};
6505                 if ($snap->{vmstate}) {
6506                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6507                     if (my $err = $@) {
6508                         warn $err;
6509                     }
6510                 }
6511             }
6512         }
6513
6514         my $map = {};
6515         foreach my $virtdev (sort keys %$virtdev_hash) {
6516             my $d = $virtdev_hash->{$virtdev};
6517             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6518             my $storeid = $d->{storeid};
6519             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6520
6521             my $map_opts = '';
6522             if (my $limit = $storage_limits{$storeid}) {
6523                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6524             }
6525
6526             # test if requested format is supported
6527             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6528             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6529             $d->{format} = $defFormat if !$supported;
6530
6531             my $name;
6532             if ($d->{is_cloudinit}) {
6533                 $name = $d->{name};
6534                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6535             }
6536
6537             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6538             print STDERR "new volume ID is '$volid'\n";
6539             $d->{volid} = $volid;
6540
6541             PVE::Storage::activate_volumes($cfg, [$volid]);
6542
6543             my $write_zeros = 1;
6544             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6545                 $write_zeros = 0;
6546             }
6547
6548             if (!$d->{is_cloudinit}) {
6549                 my $path = PVE::Storage::path($cfg, $volid);
6550
6551                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6552
6553                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6554             }
6555             $map->{$virtdev} = $volid;
6556         }
6557
6558         $fh->seek(0, 0) || die "seek failed - $!\n";
6559
6560         my $outfd = new IO::File ($tmpfn, "w") ||
6561             die "unable to write config for VM $vmid\n";
6562
6563         my $cookie = { netcount => 0 };
6564         while (defined(my $line = <$fh>)) {
6565             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6566         }
6567
6568         $fh->close();
6569         $outfd->close();
6570     };
6571
6572     eval {
6573         # enable interrupts
6574         local $SIG{INT} =
6575             local $SIG{TERM} =
6576             local $SIG{QUIT} =
6577             local $SIG{HUP} =
6578             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6579         local $SIG{ALRM} = sub { die "got timeout\n"; };
6580
6581         $oldtimeout = alarm($timeout);
6582
6583         my $parser = sub {
6584             my $line = shift;
6585
6586             print "$line\n";
6587
6588             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6589                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6590                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6591             } elsif ($line =~ m/^CTIME: /) {
6592                 # we correctly received the vma config, so we can disable
6593                 # the timeout now for disk allocation (set to 10 minutes, so
6594                 # that we always timeout if something goes wrong)
6595                 alarm(600);
6596                 &$print_devmap();
6597                 print $fifofh "done\n";
6598                 my $tmp = $oldtimeout || 0;
6599                 $oldtimeout = undef;
6600                 alarm($tmp);
6601                 close($fifofh);
6602             }
6603         };
6604
6605         print "restore vma archive: $dbg_cmdstring\n";
6606         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6607     };
6608     my $err = $@;
6609
6610     alarm($oldtimeout) if $oldtimeout;
6611
6612     my $vollist = [];
6613     foreach my $devname (keys %$devinfo) {
6614         my $volid = $devinfo->{$devname}->{volid};
6615         push @$vollist, $volid if $volid;
6616     }
6617
6618     PVE::Storage::deactivate_volumes($cfg, $vollist);
6619
6620     unlink $mapfifo;
6621
6622     if ($err) {
6623         rmtree $tmpdir;
6624         unlink $tmpfn;
6625
6626         foreach my $devname (keys %$devinfo) {
6627             my $volid = $devinfo->{$devname}->{volid};
6628             next if !$volid;
6629             eval {
6630                 if ($volid =~ m|^/|) {
6631                     unlink $volid || die 'unlink failed\n';
6632                 } else {
6633                     PVE::Storage::vdisk_free($cfg, $volid);
6634                 }
6635                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6636             };
6637             print STDERR "unable to cleanup '$volid' - $@" if $@;
6638         }
6639         die $err;
6640     }
6641
6642     rmtree $tmpdir;
6643
6644     rename($tmpfn, $conffile) ||
6645         die "unable to commit configuration file '$conffile'\n";
6646
6647     PVE::Cluster::cfs_update(); # make sure we read new file
6648
6649     eval { rescan($vmid, 1); };
6650     warn $@ if $@;
6651 }
6652
6653 sub restore_tar_archive {
6654     my ($archive, $vmid, $user, $opts) = @_;
6655
6656     if ($archive ne '-') {
6657         my $firstfile = tar_archive_read_firstfile($archive);
6658         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6659             if $firstfile ne 'qemu-server.conf';
6660     }
6661
6662     my $storecfg = PVE::Storage::config();
6663
6664     # destroy existing data - keep empty config
6665     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6666     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6667
6668     my $tocmd = "/usr/lib/qemu-server/qmextract";
6669
6670     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6671     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6672     $tocmd .= ' --prealloc' if $opts->{prealloc};
6673     $tocmd .= ' --info' if $opts->{info};
6674
6675     # tar option "xf" does not autodetect compression when read from STDIN,
6676     # so we pipe to zcat
6677     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6678         PVE::Tools::shellquote("--to-command=$tocmd");
6679
6680     my $tmpdir = "/var/tmp/vzdumptmp$$";
6681     mkpath $tmpdir;
6682
6683     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6684     local $ENV{VZDUMP_VMID} = $vmid;
6685     local $ENV{VZDUMP_USER} = $user;
6686
6687     my $conffile = PVE::QemuConfig->config_file($vmid);
6688     my $tmpfn = "$conffile.$$.tmp";
6689
6690     # disable interrupts (always do cleanups)
6691     local $SIG{INT} =
6692         local $SIG{TERM} =
6693         local $SIG{QUIT} =
6694         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6695
6696     eval {
6697         # enable interrupts
6698         local $SIG{INT} =
6699             local $SIG{TERM} =
6700             local $SIG{QUIT} =
6701             local $SIG{HUP} =
6702             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6703
6704         if ($archive eq '-') {
6705             print "extracting archive from STDIN\n";
6706             run_command($cmd, input => "<&STDIN");
6707         } else {
6708             print "extracting archive '$archive'\n";
6709             run_command($cmd);
6710         }
6711
6712         return if $opts->{info};
6713
6714         # read new mapping
6715         my $map = {};
6716         my $statfile = "$tmpdir/qmrestore.stat";
6717         if (my $fd = IO::File->new($statfile, "r")) {
6718             while (defined (my $line = <$fd>)) {
6719                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6720                     $map->{$1} = $2 if $1;
6721                 } else {
6722                     print STDERR "unable to parse line in statfile - $line\n";
6723                 }
6724             }
6725             $fd->close();
6726         }
6727
6728         my $confsrc = "$tmpdir/qemu-server.conf";
6729
6730         my $srcfd = new IO::File($confsrc, "r") ||
6731             die "unable to open file '$confsrc'\n";
6732
6733         my $outfd = new IO::File ($tmpfn, "w") ||
6734             die "unable to write config for VM $vmid\n";
6735
6736         my $cookie = { netcount => 0 };
6737         while (defined (my $line = <$srcfd>)) {
6738             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6739         }
6740
6741         $srcfd->close();
6742         $outfd->close();
6743     };
6744     my $err = $@;
6745
6746     if ($err) {
6747
6748         unlink $tmpfn;
6749
6750         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6751
6752         die $err;
6753     }
6754
6755     rmtree $tmpdir;
6756
6757     rename $tmpfn, $conffile ||
6758         die "unable to commit configuration file '$conffile'\n";
6759
6760     PVE::Cluster::cfs_update(); # make sure we read new file
6761
6762     eval { rescan($vmid, 1); };
6763     warn $@ if $@;
6764 };
6765
6766 sub foreach_storage_used_by_vm {
6767     my ($conf, $func) = @_;
6768
6769     my $sidhash = {};
6770
6771     foreach_drive($conf, sub {
6772         my ($ds, $drive) = @_;
6773         return if drive_is_cdrom($drive);
6774
6775         my $volid = $drive->{file};
6776
6777         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6778         $sidhash->{$sid} = $sid if $sid;
6779     });
6780
6781     foreach my $sid (sort keys %$sidhash) {
6782         &$func($sid);
6783     }
6784 }
6785
6786 sub do_snapshots_with_qemu {
6787     my ($storecfg, $volid) = @_;
6788
6789     my $storage_name = PVE::Storage::parse_volume_id($volid);
6790     my $scfg = $storecfg->{ids}->{$storage_name};
6791
6792     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6793         return 1;
6794     }
6795
6796     if ($volid =~ m/\.(qcow2|qed)$/){
6797         return 1;
6798     }
6799
6800     return undef;
6801 }
6802
6803 sub qga_check_running {
6804     my ($vmid, $nowarn) = @_;
6805
6806     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6807     if ($@) {
6808         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6809         return 0;
6810     }
6811     return 1;
6812 }
6813
6814 sub template_create {
6815     my ($vmid, $conf, $disk) = @_;
6816
6817     my $storecfg = PVE::Storage::config();
6818
6819     foreach_drive($conf, sub {
6820         my ($ds, $drive) = @_;
6821
6822         return if drive_is_cdrom($drive);
6823         return if $disk && $ds ne $disk;
6824
6825         my $volid = $drive->{file};
6826         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6827
6828         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6829         $drive->{file} = $voliddst;
6830         $conf->{$ds} = print_drive($vmid, $drive);
6831         PVE::QemuConfig->write_config($vmid, $conf);
6832     });
6833 }
6834
6835 sub convert_iscsi_path {
6836     my ($path) = @_;
6837
6838     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6839         my $portal = $1;
6840         my $target = $2;
6841         my $lun = $3;
6842
6843         my $initiator_name = get_initiator_name();
6844
6845         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6846                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6847     }
6848
6849     die "cannot convert iscsi path '$path', unkown format\n";
6850 }
6851
6852 sub qemu_img_convert {
6853     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6854
6855     my $storecfg = PVE::Storage::config();
6856     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6857     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6858
6859     if ($src_storeid && $dst_storeid) {
6860
6861         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6862
6863         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6864         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6865
6866         my $src_format = qemu_img_format($src_scfg, $src_volname);
6867         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6868
6869         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6870         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6871
6872         my $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6873         my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6874
6875         my $cmd = [];
6876         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6877         push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6878         push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6879         push @$cmd, '-T', 'none' if $src_scfg->{type} eq 'zfspool';
6880
6881         if ($src_is_iscsi) {
6882             push @$cmd, '--image-opts';
6883             $src_path = convert_iscsi_path($src_path);
6884         } else {
6885             push @$cmd, '-f', $src_format;
6886         }
6887
6888         if ($dst_is_iscsi) {
6889             push @$cmd, '--target-image-opts';
6890             $dst_path = convert_iscsi_path($dst_path);
6891         } else {
6892             push @$cmd, '-O', $dst_format;
6893         }
6894
6895         push @$cmd, $src_path;
6896
6897         if (!$dst_is_iscsi && $is_zero_initialized) {
6898             push @$cmd, "zeroinit:$dst_path";
6899         } else {
6900             push @$cmd, $dst_path;
6901         }
6902
6903         my $parser = sub {
6904             my $line = shift;
6905             if($line =~ m/\((\S+)\/100\%\)/){
6906                 my $percent = $1;
6907                 my $transferred = int($size * $percent / 100);
6908                 my $remaining = $size - $transferred;
6909
6910                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6911             }
6912
6913         };
6914
6915         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6916         my $err = $@;
6917         die "copy failed: $err" if $err;
6918     }
6919 }
6920
6921 sub qemu_img_format {
6922     my ($scfg, $volname) = @_;
6923
6924     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6925         return $1;
6926     } else {
6927         return "raw";
6928     }
6929 }
6930
6931 sub qemu_drive_mirror {
6932     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6933
6934     $jobs = {} if !$jobs;
6935
6936     my $qemu_target;
6937     my $format;
6938     $jobs->{"drive-$drive"} = {};
6939
6940     if ($dst_volid =~ /^nbd:/) {
6941         $qemu_target = $dst_volid;
6942         $format = "nbd";
6943     } else {
6944         my $storecfg = PVE::Storage::config();
6945         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6946
6947         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6948
6949         $format = qemu_img_format($dst_scfg, $dst_volname);
6950
6951         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6952
6953         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6954     }
6955
6956     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6957     $opts->{format} = $format if $format;
6958
6959     if (defined($bwlimit)) {
6960         $opts->{speed} = $bwlimit * 1024;
6961         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6962     } else {
6963         print "drive mirror is starting for drive-$drive\n";
6964     }
6965
6966     # if a job already runs for this device we get an error, catch it for cleanup
6967     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6968     if (my $err = $@) {
6969         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6970         warn "$@\n" if $@;
6971         die "mirroring error: $err\n";
6972     }
6973
6974     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6975 }
6976
6977 sub qemu_drive_mirror_monitor {
6978     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6979
6980     eval {
6981         my $err_complete = 0;
6982
6983         while (1) {
6984             die "storage migration timed out\n" if $err_complete > 300;
6985
6986             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6987
6988             my $running_mirror_jobs = {};
6989             foreach my $stat (@$stats) {
6990                 next if $stat->{type} ne 'mirror';
6991                 $running_mirror_jobs->{$stat->{device}} = $stat;
6992             }
6993
6994             my $readycounter = 0;
6995
6996             foreach my $job (keys %$jobs) {
6997
6998                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6999                     print "$job : finished\n";
7000                     delete $jobs->{$job};
7001                     next;
7002                 }
7003
7004                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
7005
7006                 my $busy = $running_mirror_jobs->{$job}->{busy};
7007                 my $ready = $running_mirror_jobs->{$job}->{ready};
7008                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
7009                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
7010                     my $remaining = $total - $transferred;
7011                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
7012
7013                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
7014                 }
7015
7016                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
7017             }
7018
7019             last if scalar(keys %$jobs) == 0;
7020
7021             if ($readycounter == scalar(keys %$jobs)) {
7022                 print "all mirroring jobs are ready \n";
7023                 last if $skipcomplete; #do the complete later
7024
7025                 if ($vmiddst && $vmiddst != $vmid) {
7026                     my $agent_running = $qga && qga_check_running($vmid);
7027                     if ($agent_running) {
7028                         print "freeze filesystem\n";
7029                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
7030                     } else {
7031                         print "suspend vm\n";
7032                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
7033                     }
7034
7035                     # if we clone a disk for a new target vm, we don't switch the disk
7036                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
7037
7038                     if ($agent_running) {
7039                         print "unfreeze filesystem\n";
7040                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
7041                     } else {
7042                         print "resume vm\n";
7043                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
7044                     }
7045
7046                     last;
7047                 } else {
7048
7049                     foreach my $job (keys %$jobs) {
7050                         # try to switch the disk if source and destination are on the same guest
7051                         print "$job: Completing block job...\n";
7052
7053                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7054                         if ($@ =~ m/cannot be completed/) {
7055                             print "$job: Block job cannot be completed, try again.\n";
7056                             $err_complete++;
7057                         }else {
7058                             print "$job: Completed successfully.\n";
7059                             $jobs->{$job}->{complete} = 1;
7060                         }
7061                     }
7062                 }
7063             }
7064             sleep 1;
7065         }
7066     };
7067     my $err = $@;
7068
7069     if ($err) {
7070         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7071         die "mirroring error: $err";
7072     }
7073
7074 }
7075
7076 sub qemu_blockjobs_cancel {
7077     my ($vmid, $jobs) = @_;
7078
7079     foreach my $job (keys %$jobs) {
7080         print "$job: Cancelling block job\n";
7081         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7082         $jobs->{$job}->{cancel} = 1;
7083     }
7084
7085     while (1) {
7086         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7087
7088         my $running_jobs = {};
7089         foreach my $stat (@$stats) {
7090             $running_jobs->{$stat->{device}} = $stat;
7091         }
7092
7093         foreach my $job (keys %$jobs) {
7094
7095             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7096                 print "$job: Done.\n";
7097                 delete $jobs->{$job};
7098             }
7099         }
7100
7101         last if scalar(keys %$jobs) == 0;
7102
7103         sleep 1;
7104     }
7105 }
7106
7107 sub clone_disk {
7108     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7109         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7110
7111     my $newvolid;
7112
7113     if (!$full) {
7114         print "create linked clone of drive $drivename ($drive->{file})\n";
7115         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7116         push @$newvollist, $newvolid;
7117     } else {
7118
7119         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7120         $storeid = $storage if $storage;
7121
7122         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7123         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7124
7125         print "create full clone of drive $drivename ($drive->{file})\n";
7126         my $name = undef;
7127         if (drive_is_cloudinit($drive)) {
7128             $name = "vm-$newvmid-cloudinit";
7129             $snapname = undef;
7130             # we only get here if it's supported by QEMU_FORMAT_RE, so just accept
7131             if ($dst_format ne 'raw') {
7132                 $name .= ".$dst_format";
7133             }
7134         }
7135         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7136         push @$newvollist, $newvolid;
7137
7138         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7139
7140         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7141         if (!$running || $snapname) {
7142             # TODO: handle bwlimits
7143             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7144         } else {
7145
7146             my $kvmver = get_running_qemu_version ($vmid);
7147             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7148                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7149                     if $drive->{iothread};
7150             }
7151
7152             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7153         }
7154     }
7155
7156     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7157
7158     my $disk = $drive;
7159     $disk->{format} = undef;
7160     $disk->{file} = $newvolid;
7161     $disk->{size} = $size;
7162
7163     return $disk;
7164 }
7165
7166 # this only works if VM is running
7167 sub get_current_qemu_machine {
7168     my ($vmid) = @_;
7169
7170     my $cmd = { execute => 'query-machines', arguments => {} };
7171     my $res = vm_qmp_command($vmid, $cmd);
7172
7173     my ($current, $default);
7174     foreach my $e (@$res) {
7175         $default = $e->{name} if $e->{'is-default'};
7176         $current = $e->{name} if $e->{'is-current'};
7177     }
7178
7179     # fallback to the default machine if current is not supported by qemu
7180     return $current || $default || 'pc';
7181 }
7182
7183 sub get_running_qemu_version {
7184     my ($vmid) = @_;
7185     my $cmd = { execute => 'query-version', arguments => {} };
7186     my $res = vm_qmp_command($vmid, $cmd);
7187     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7188 }
7189
7190 sub qemu_machine_feature_enabled {
7191     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7192
7193     my $current_major;
7194     my $current_minor;
7195
7196     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7197
7198         $current_major = $3;
7199         $current_minor = $4;
7200
7201     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7202
7203         $current_major = $1;
7204         $current_minor = $2;
7205     }
7206
7207     return 1 if $current_major > $version_major ||
7208                 ($current_major == $version_major &&
7209                  $current_minor >= $version_minor);
7210 }
7211
7212 sub qemu_machine_pxe {
7213     my ($vmid, $conf) = @_;
7214
7215     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7216
7217     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7218         $machine .= '.pxe';
7219     }
7220
7221     return $machine;
7222 }
7223
7224 sub qemu_use_old_bios_files {
7225     my ($machine_type) = @_;
7226
7227     return if !$machine_type;
7228
7229     my $use_old_bios_files = undef;
7230
7231     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7232         $machine_type = $1;
7233         $use_old_bios_files = 1;
7234     } else {
7235         my $kvmver = kvm_user_version();
7236         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7237         # load new efi bios files on migration. So this hack is required to allow
7238         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7239         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7240         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7241     }
7242
7243     return ($use_old_bios_files, $machine_type);
7244 }
7245
7246 sub create_efidisk($$$$$) {
7247     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7248
7249     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7250     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7251
7252     my $vars_size = PVE::Tools::convert_size(-s $ovmf_vars, 'b' => 'kb');
7253     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7254     PVE::Storage::activate_volumes($storecfg, [$volid]);
7255
7256     my $path = PVE::Storage::path($storecfg, $volid);
7257     eval {
7258         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $ovmf_vars, $path]);
7259     };
7260     die "Copying EFI vars image failed: $@" if $@;
7261
7262     return ($volid, $vars_size);
7263 }
7264
7265 sub vm_iothreads_list {
7266     my ($vmid) = @_;
7267
7268     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7269
7270     my $iothreads = {};
7271     foreach my $iothread (@$res) {
7272         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7273     }
7274
7275     return $iothreads;
7276 }
7277
7278 sub scsihw_infos {
7279     my ($conf, $drive) = @_;
7280
7281     my $maxdev = 0;
7282
7283     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7284         $maxdev = 7;
7285     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7286         $maxdev = 1;
7287     } else {
7288         $maxdev = 256;
7289     }
7290
7291     my $controller = int($drive->{index} / $maxdev);
7292     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7293
7294     return ($maxdev, $controller, $controller_prefix);
7295 }
7296
7297 sub add_hyperv_enlightenments {
7298     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7299
7300     return if $winversion < 6;
7301     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7302
7303     if ($gpu_passthrough || defined($hv_vendor_id)) {
7304         $hv_vendor_id //= 'proxmox';
7305         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7306     }
7307
7308     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7309         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7310         push @$cpuFlags , 'hv_vapic';
7311         push @$cpuFlags , 'hv_time';
7312     } else {
7313         push @$cpuFlags , 'hv_spinlocks=0xffff';
7314     }
7315
7316     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7317         push @$cpuFlags , 'hv_reset';
7318         push @$cpuFlags , 'hv_vpindex';
7319         push @$cpuFlags , 'hv_runtime';
7320     }
7321
7322     if ($winversion >= 7) {
7323         push @$cpuFlags , 'hv_relaxed';
7324
7325         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7326             push @$cpuFlags , 'hv_synic';
7327             push @$cpuFlags , 'hv_stimer';
7328         }
7329
7330         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7331             push @$cpuFlags , 'hv_ipi';
7332         }
7333     }
7334 }
7335
7336 sub windows_version {
7337     my ($ostype) = @_;
7338
7339     return 0 if !$ostype;
7340
7341     my $winversion = 0;
7342
7343     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7344         $winversion = 5;
7345     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7346         $winversion = 6;
7347     } elsif ($ostype =~ m/^win(\d+)$/) {
7348         $winversion = $1;
7349     }
7350
7351     return $winversion;
7352 }
7353
7354 sub resolve_dst_disk_format {
7355         my ($storecfg, $storeid, $src_volname, $format) = @_;
7356         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7357
7358         if (!$format) {
7359             # if no target format is specified, use the source disk format as hint
7360             if ($src_volname) {
7361                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7362                 $format = qemu_img_format($scfg, $src_volname);
7363             } else {
7364                 return $defFormat;
7365             }
7366         }
7367
7368         # test if requested format is supported - else use default
7369         my $supported = grep { $_ eq $format } @$validFormats;
7370         $format = $defFormat if !$supported;
7371         return $format;
7372 }
7373
7374 sub resolve_first_disk {
7375     my $conf = shift;
7376     my @disks = PVE::QemuServer::valid_drive_names();
7377     my $firstdisk;
7378     foreach my $ds (reverse @disks) {
7379         next if !$conf->{$ds};
7380         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7381         next if PVE::QemuServer::drive_is_cdrom($disk);
7382         $firstdisk = $ds;
7383     }
7384     return $firstdisk;
7385 }
7386
7387 sub generate_uuid {
7388     my ($uuid, $uuid_str);
7389     UUID::generate($uuid);
7390     UUID::unparse($uuid, $uuid_str);
7391     return $uuid_str;
7392 }
7393
7394 sub generate_smbios1_uuid {
7395     return "uuid=".generate_uuid();
7396 }
7397
7398 sub nbd_stop {
7399     my ($vmid) = @_;
7400
7401     vm_mon_cmd($vmid, 'nbd-server-stop');
7402 }
7403
7404 # bash completion helper
7405
7406 sub complete_backup_archives {
7407     my ($cmdname, $pname, $cvalue) = @_;
7408
7409     my $cfg = PVE::Storage::config();
7410
7411     my $storeid;
7412
7413     if ($cvalue =~ m/^([^:]+):/) {
7414         $storeid = $1;
7415     }
7416
7417     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7418
7419     my $res = [];
7420     foreach my $id (keys %$data) {
7421         foreach my $item (@{$data->{$id}}) {
7422             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7423             push @$res, $item->{volid} if defined($item->{volid});
7424         }
7425     }
7426
7427     return $res;
7428 }
7429
7430 my $complete_vmid_full = sub {
7431     my ($running) = @_;
7432
7433     my $idlist = vmstatus();
7434
7435     my $res = [];
7436
7437     foreach my $id (keys %$idlist) {
7438         my $d = $idlist->{$id};
7439         if (defined($running)) {
7440             next if $d->{template};
7441             next if $running && $d->{status} ne 'running';
7442             next if !$running && $d->{status} eq 'running';
7443         }
7444         push @$res, $id;
7445
7446     }
7447     return $res;
7448 };
7449
7450 sub complete_vmid {
7451     return &$complete_vmid_full();
7452 }
7453
7454 sub complete_vmid_stopped {
7455     return &$complete_vmid_full(0);
7456 }
7457
7458 sub complete_vmid_running {
7459     return &$complete_vmid_full(1);
7460 }
7461
7462 sub complete_storage {
7463
7464     my $cfg = PVE::Storage::config();
7465     my $ids = $cfg->{ids};
7466
7467     my $res = [];
7468     foreach my $sid (keys %$ids) {
7469         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7470         next if !$ids->{$sid}->{content}->{images};
7471         push @$res, $sid;
7472     }
7473
7474     return $res;
7475 }
7476
7477 1;