PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::Exception qw(raise raise_param_exc);
  31 use PVE::GuestHelpers;
  32 use PVE::INotify;
  33 use PVE::JSONSchema qw(get_standard_option);
  34 use PVE::ProcFSTools;
  35 use PVE::RPCEnvironment;
  36 use PVE::SafeSyslog;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Cloudinit;
  45 use PVE::QemuServer::Memory;
  46 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  47 use PVE::QemuServer::USB qw(parse_usb_device);
  48
  49 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  50 my $OVMF = {
  51     x86_64 => [
  52         "$EDK2_FW_BASE/OVMF_CODE.fd",
  53         "$EDK2_FW_BASE/OVMF_VARS.fd"
  54     ],
  55     aarch64 => [
  56         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  57         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  58     ],
  59 };
  60
  61 my $qemu_snap_storage = { rbd => 1 };
  62
  63 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  64
  65 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  66
  67 # Note about locking: we use flock on the config file protect
  68 # against concurent actions.
  69 # Aditionaly, we have a 'lock' setting in the config file. This
  70 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  71 # allowed when such lock is set. But you can ignore this kind of
  72 # lock with the --skiplock flag.
  73
  74 cfs_register_file('/qemu-server/',
  75                   \&parse_vm_config,
  76                   \&write_vm_config);
  77
  78 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  79     description => "Some command save/restore state from this location.",
  80     type => 'string',
  81     maxLength => 128,
  82     optional => 1,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $cpu_vendor_list = {
 123     # Intel CPUs
 124     486 => 'GenuineIntel',
 125     pentium => 'GenuineIntel',
 126     pentium2  => 'GenuineIntel',
 127     pentium3  => 'GenuineIntel',
 128     coreduo => 'GenuineIntel',
 129     core2duo => 'GenuineIntel',
 130     Conroe  => 'GenuineIntel',
 131     Penryn  => 'GenuineIntel',
 132     Nehalem  => 'GenuineIntel',
 133     'Nehalem-IBRS'  => 'GenuineIntel',
 134     Westmere => 'GenuineIntel',
 135     'Westmere-IBRS' => 'GenuineIntel',
 136     SandyBridge => 'GenuineIntel',
 137     'SandyBridge-IBRS' => 'GenuineIntel',
 138     IvyBridge => 'GenuineIntel',
 139     'IvyBridge-IBRS' => 'GenuineIntel',
 140     Haswell => 'GenuineIntel',
 141     'Haswell-IBRS' => 'GenuineIntel',
 142     'Haswell-noTSX' => 'GenuineIntel',
 143     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 144     Broadwell => 'GenuineIntel',
 145     'Broadwell-IBRS' => 'GenuineIntel',
 146     'Broadwell-noTSX' => 'GenuineIntel',
 147     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 148     'Skylake-Client' => 'GenuineIntel',
 149     'Skylake-Client-IBRS' => 'GenuineIntel',
 150     'Skylake-Server' => 'GenuineIntel',
 151     'Skylake-Server-IBRS' => 'GenuineIntel',
 152     'Cascadelake-Server' => 'GenuineIntel',
 153     KnightsMill => 'GenuineIntel',
 154
 155
 156     # AMD CPUs
 157     athlon => 'AuthenticAMD',
 158     phenom  => 'AuthenticAMD',
 159     Opteron_G1  => 'AuthenticAMD',
 160     Opteron_G2  => 'AuthenticAMD',
 161     Opteron_G3  => 'AuthenticAMD',
 162     Opteron_G4  => 'AuthenticAMD',
 163     Opteron_G5  => 'AuthenticAMD',
 164     EPYC => 'AuthenticAMD',
 165     'EPYC-IBPB' => 'AuthenticAMD',
 166
 167     # generic types, use vendor from host node
 168     host => 'default',
 169     kvm32 => 'default',
 170     kvm64 => 'default',
 171     qemu32 => 'default',
 172     qemu64 => 'default',
 173     max => 'default',
 174 };
 175
 176 my @supported_cpu_flags = (
 177     'pcid',
 178     'spec-ctrl',
 179     'ibpb',
 180     'ssbd',
 181     'virt-ssbd',
 182     'amd-ssbd',
 183     'amd-no-ssb',
 184     'pdpe1gb',
 185     'md-clear',
 186     'hv-tlbflush',
 187     'hv-evmcs',
 188     'aes'
 189 );
 190 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 191
 192 my $cpu_fmt = {
 193     cputype => {
 194         description => "Emulated CPU type.",
 195         type => 'string',
 196         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 197         default => 'kvm64',
 198         default_key => 1,
 199     },
 200     hidden => {
 201         description => "Do not identify as a KVM virtual machine.",
 202         type => 'boolean',
 203         optional => 1,
 204         default => 0
 205     },
 206     'hv-vendor-id' => {
 207         type => 'string',
 208         pattern => qr/[a-zA-Z0-9]{1,12}/,
 209         format_description => 'vendor-id',
 210         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 211         optional => 1,
 212     },
 213     flags => {
 214         description => "List of additional CPU flags separated by ';'."
 215                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 216                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 217         format_description => '+FLAG[;-FLAG...]',
 218         type => 'string',
 219         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 220         optional => 1,
 221     },
 222 };
 223
 224 my $watchdog_fmt = {
 225     model => {
 226         default_key => 1,
 227         type => 'string',
 228         enum => [qw(i6300esb ib700)],
 229         description => "Watchdog type to emulate.",
 230         default => 'i6300esb',
 231         optional => 1,
 232     },
 233     action => {
 234         type => 'string',
 235         enum => [qw(reset shutdown poweroff pause debug none)],
 236         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 237         optional => 1,
 238     },
 239 };
 240 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 241
 242 my $agent_fmt = {
 243     enabled => {
 244         description => "Enable/disable Qemu GuestAgent.",
 245         type => 'boolean',
 246         default => 0,
 247         default_key => 1,
 248     },
 249     fstrim_cloned_disks => {
 250         description => "Run fstrim after cloning/moving a disk.",
 251         type => 'boolean',
 252         optional => 1,
 253         default => 0
 254     },
 255 };
 256
 257 my $vga_fmt = {
 258     type => {
 259         description => "Select the VGA type.",
 260         type => 'string',
 261         default => 'std',
 262         optional => 1,
 263         default_key => 1,
 264         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 265     },
 266     memory => {
 267         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 268         type => 'integer',
 269         optional => 1,
 270         minimum => 4,
 271         maximum => 512,
 272     },
 273 };
 274
 275 my $ivshmem_fmt = {
 276     size => {
 277         type => 'integer',
 278         minimum => 1,
 279         description => "The size of the file in MB.",
 280     },
 281     name => {
 282         type => 'string',
 283         pattern => '[a-zA-Z0-9\-]+',
 284         optional => 1,
 285         format_description => 'string',
 286         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 287     },
 288 };
 289
 290 my $audio_fmt = {
 291     device => {
 292         type => 'string',
 293         enum => [qw(ich9-intel-hda intel-hda AC97)],
 294         description =>  "Configure an audio device."
 295     },
 296     driver =>  {
 297         type => 'string',
 298         enum => ['spice'],
 299         default => 'spice',
 300         optional => 1,
 301         description => "Driver backend for the audio device."
 302     },
 303 };
 304
 305 my $spice_enhancements_fmt = {
 306     foldersharing => {
 307         type => 'boolean',
 308         optional => 1,
 309         default => '0',
 310         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 311     },
 312     videostreaming =>  {
 313         type => 'string',
 314         enum => ['off', 'all', 'filter'],
 315         default => 'off',
 316         optional => 1,
 317         description => "Enable video streaming. Uses compression for detected video streams."
 318     },
 319 };
 320
 321 my $confdesc = {
 322     onboot => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Specifies whether a VM will be started during system bootup.",
 326         default => 0,
 327     },
 328     autostart => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Automatic restart after crash (currently ignored).",
 332         default => 0,
 333     },
 334     hotplug => {
 335         optional => 1,
 336         type => 'string', format => 'pve-hotplug-features',
 337         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 338         default => 'network,disk,usb',
 339     },
 340     reboot => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 344         default => 1,
 345     },
 346     lock => {
 347         optional => 1,
 348         type => 'string',
 349         description => "Lock/unlock the VM.",
 350         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 351     },
 352     cpulimit => {
 353         optional => 1,
 354         type => 'number',
 355         description => "Limit of CPU usage.",
 356         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 357         minimum => 0,
 358         maximum => 128,
 359         default => 0,
 360     },
 361     cpuunits => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "CPU weight for a VM.",
 365         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 366         minimum => 2,
 367         maximum => 262144,
 368         default => 1024,
 369     },
 370     memory => {
 371         optional => 1,
 372         type => 'integer',
 373         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 374         minimum => 16,
 375         default => 512,
 376     },
 377     balloon => {
 378         optional => 1,
 379         type => 'integer',
 380         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 381         minimum => 0,
 382     },
 383     shares => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 387         minimum => 0,
 388         maximum => 50000,
 389         default => 1000,
 390     },
 391     keyboard => {
 392         optional => 1,
 393         type => 'string',
 394         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 395                        "It should not be necessary to set it.",
 396         enum => PVE::Tools::kvmkeymaplist(),
 397         default => undef,
 398     },
 399     name => {
 400         optional => 1,
 401         type => 'string', format => 'dns-name',
 402         description => "Set a name for the VM. Only used on the configuration web interface.",
 403     },
 404     scsihw => {
 405         optional => 1,
 406         type => 'string',
 407         description => "SCSI controller model",
 408         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 409         default => 'lsi',
 410     },
 411     description => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 415     },
 416     ostype => {
 417         optional => 1,
 418         type => 'string',
 419         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 420         description => "Specify guest operating system.",
 421         verbose_description => <<EODESC,
 422 Specify guest operating system. This is used to enable special
 423 optimization/features for specific operating systems:
 424
 425 [horizontal]
 426 other;; unspecified OS
 427 wxp;; Microsoft Windows XP
 428 w2k;; Microsoft Windows 2000
 429 w2k3;; Microsoft Windows 2003
 430 w2k8;; Microsoft Windows 2008
 431 wvista;; Microsoft Windows Vista
 432 win7;; Microsoft Windows 7
 433 win8;; Microsoft Windows 8/2012/2012r2
 434 win10;; Microsoft Windows 10/2016
 435 l24;; Linux 2.4 Kernel
 436 l26;; Linux 2.6/3.X Kernel
 437 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 438 EODESC
 439     },
 440     boot => {
 441         optional => 1,
 442         type => 'string',
 443         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 444         pattern => '[acdn]{1,4}',
 445         default => 'cdn',
 446     },
 447     bootdisk => {
 448         optional => 1,
 449         type => 'string', format => 'pve-qm-bootdisk',
 450         description => "Enable booting from specified disk.",
 451         pattern => '(ide|sata|scsi|virtio)\d+',
 452     },
 453     smp => {
 454         optional => 1,
 455         type => 'integer',
 456         description => "The number of CPUs. Please use option -sockets instead.",
 457         minimum => 1,
 458         default => 1,
 459     },
 460     sockets => {
 461         optional => 1,
 462         type => 'integer',
 463         description => "The number of CPU sockets.",
 464         minimum => 1,
 465         default => 1,
 466     },
 467     cores => {
 468         optional => 1,
 469         type => 'integer',
 470         description => "The number of cores per socket.",
 471         minimum => 1,
 472         default => 1,
 473     },
 474     numa => {
 475         optional => 1,
 476         type => 'boolean',
 477         description => "Enable/disable NUMA.",
 478         default => 0,
 479     },
 480     hugepages => {
 481         optional => 1,
 482         type => 'string',
 483         description => "Enable/disable hugepages memory.",
 484         enum => [qw(any 2 1024)],
 485     },
 486     vcpus => {
 487         optional => 1,
 488         type => 'integer',
 489         description => "Number of hotplugged vcpus.",
 490         minimum => 1,
 491         default => 0,
 492     },
 493     acpi => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Enable/disable ACPI.",
 497         default => 1,
 498     },
 499     agent => {
 500         optional => 1,
 501         description => "Enable/disable Qemu GuestAgent and its properties.",
 502         type => 'string',
 503         format => $agent_fmt,
 504     },
 505     kvm => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable KVM hardware virtualization.",
 509         default => 1,
 510     },
 511     tdf => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable time drift fix.",
 515         default => 0,
 516     },
 517     localtime => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 521     },
 522     freeze => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 526     },
 527     vga => {
 528         optional => 1,
 529         type => 'string', format => $vga_fmt,
 530         description => "Configure the VGA hardware.",
 531         verbose_description => "Configure the VGA Hardware. If you want to use ".
 532             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 533             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 534             "is 'std' for all OS types besides some Windows versions (XP and " .
 535             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 536             "display server. For win* OS you can select how many independent " .
 537             "displays you want, Linux guests can add displays them self.\n".
 538             "You can also run without any graphic card, using a serial device as terminal.",
 539     },
 540     watchdog => {
 541         optional => 1,
 542         type => 'string', format => 'pve-qm-watchdog',
 543         description => "Create a virtual hardware watchdog device.",
 544         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 545             " (by a guest action), the watchdog must be periodically polled " .
 546             "by an agent inside the guest or else the watchdog will reset " .
 547             "the guest (or execute the respective action specified)",
 548     },
 549     startdate => {
 550         optional => 1,
 551         type => 'string',
 552         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 553         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 554         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 555         default => 'now',
 556     },
 557     startup =>  get_standard_option('pve-startup-order'),
 558     template => {
 559         optional => 1,
 560         type => 'boolean',
 561         description => "Enable/disable Template.",
 562         default => 0,
 563     },
 564     args => {
 565         optional => 1,
 566         type => 'string',
 567         description => "Arbitrary arguments passed to kvm.",
 568         verbose_description => <<EODESCR,
 569 Arbitrary arguments passed to kvm, for example:
 570
 571 args: -no-reboot -no-hpet
 572
 573 NOTE: this option is for experts only.
 574 EODESCR
 575     },
 576     tablet => {
 577         optional => 1,
 578         type => 'boolean',
 579         default => 1,
 580         description => "Enable/disable the USB tablet device.",
 581         verbose_description => "Enable/disable the USB tablet device. This device is " .
 582             "usually needed to allow absolute mouse positioning with VNC. " .
 583             "Else the mouse runs out of sync with normal VNC clients. " .
 584             "If you're running lots of console-only guests on one host, " .
 585             "you may consider disabling this to save some context switches. " .
 586             "This is turned off by default if you use spice (-vga=qxl).",
 587     },
 588     migrate_speed => {
 589         optional => 1,
 590         type => 'integer',
 591         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 592         minimum => 0,
 593         default => 0,
 594     },
 595     migrate_downtime => {
 596         optional => 1,
 597         type => 'number',
 598         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 599         minimum => 0,
 600         default => 0.1,
 601     },
 602     cdrom => {
 603         optional => 1,
 604         type => 'string', format => 'pve-qm-ide',
 605         typetext => '<volume>',
 606         description => "This is an alias for option -ide2",
 607     },
 608     cpu => {
 609         optional => 1,
 610         description => "Emulated CPU type.",
 611         type => 'string',
 612         format => $cpu_fmt,
 613     },
 614     parent => get_standard_option('pve-snapshot-name', {
 615         optional => 1,
 616         description => "Parent snapshot name. This is used internally, and should not be modified.",
 617     }),
 618     snaptime => {
 619         optional => 1,
 620         description => "Timestamp for snapshots.",
 621         type => 'integer',
 622         minimum => 0,
 623     },
 624     vmstate => {
 625         optional => 1,
 626         type => 'string', format => 'pve-volume-id',
 627         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 628     },
 629     vmstatestorage => get_standard_option('pve-storage-id', {
 630         description => "Default storage for VM state volumes/files.",
 631         optional => 1,
 632     }),
 633     runningmachine => get_standard_option('pve-qemu-machine', {
 634         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 635     }),
 636     machine => get_standard_option('pve-qemu-machine'),
 637     arch => {
 638         description => "Virtual processor architecture. Defaults to the host.",
 639         optional => 1,
 640         type => 'string',
 641         enum => [qw(x86_64 aarch64)],
 642     },
 643     smbios1 => {
 644         description => "Specify SMBIOS type 1 fields.",
 645         type => 'string', format => 'pve-qm-smbios1',
 646         maxLength => 512,
 647         optional => 1,
 648     },
 649     protection => {
 650         optional => 1,
 651         type => 'boolean',
 652         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 667         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 668             " 128-bit integer value identifier to the guest OS. This allows to".
 669             " notify the guest operating system when the virtual machine is".
 670             " executed with a different configuration (e.g. snapshot execution".
 671             " or creation from a template). The guest operating system notices".
 672             " the change, and is then able to react as appropriate by marking".
 673             " its copies of distributed databases as dirty, re-initializing its".
 674             " random number generator, etc.\n".
 675             "Note that auto-creation only works when done throug API/CLI create".
 676             " or update methods, but not when manually editing the config file.",
 677         default => "1 (autogenerated)",
 678         optional => 1,
 679     },
 680     hookscript => {
 681         type => 'string',
 682         format => 'pve-volume-id',
 683         optional => 1,
 684         description => "Script that will be executed during various steps in the vms lifetime.",
 685     },
 686     ivshmem => {
 687         type => 'string',
 688         format => $ivshmem_fmt,
 689         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704 };
 705
 706 my $cicustom_fmt = {
 707     meta => {
 708         type => 'string',
 709         optional => 1,
 710         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714     network => {
 715         type => 'string',
 716         optional => 1,
 717         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     user => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728 };
 729 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 730
 731 my $confdesc_cloudinit = {
 732     citype => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 736         enum => ['configdrive2', 'nocloud'],
 737     },
 738     ciuser => {
 739         optional => 1,
 740         type => 'string',
 741         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 742     },
 743     cipassword => {
 744         optional => 1,
 745         type => 'string',
 746         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 747     },
 748     cicustom => {
 749         optional => 1,
 750         type => 'string',
 751         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 752         format => 'pve-qm-cicustom',
 753     },
 754     searchdomain => {
 755         optional => 1,
 756         type => 'string',
 757         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 758     },
 759     nameserver => {
 760         optional => 1,
 761         type => 'string', format => 'address-list',
 762         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 763     },
 764     sshkeys => {
 765         optional => 1,
 766         type => 'string',
 767         format => 'urlencoded',
 768         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 769     },
 770 };
 771
 772 # what about other qemu settings ?
 773 #cpu => 'string',
 774 #machine => 'string',
 775 #fda => 'file',
 776 #fdb => 'file',
 777 #mtdblock => 'file',
 778 #sd => 'file',
 779 #pflash => 'file',
 780 #snapshot => 'bool',
 781 #bootp => 'file',
 782 ##tftp => 'dir',
 783 ##smb => 'dir',
 784 #kernel => 'file',
 785 #append => 'string',
 786 #initrd => 'file',
 787 ##soundhw => 'string',
 788
 789 while (my ($k, $v) = each %$confdesc) {
 790     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 791 }
 792
 793 my $MAX_IDE_DISKS = 4;
 794 my $MAX_SCSI_DISKS = 14;
 795 my $MAX_VIRTIO_DISKS = 16;
 796 my $MAX_SATA_DISKS = 6;
 797 my $MAX_USB_DEVICES = 5;
 798 my $MAX_NETS = 32;
 799 my $MAX_UNUSED_DISKS = 256;
 800 my $MAX_HOSTPCI_DEVICES = 16;
 801 my $MAX_SERIAL_PORTS = 4;
 802 my $MAX_PARALLEL_PORTS = 3;
 803 my $MAX_NUMA = 8;
 804
 805 my $numa_fmt = {
 806     cpus => {
 807         type => "string",
 808         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 809         description => "CPUs accessing this NUMA node.",
 810         format_description => "id[-id];...",
 811     },
 812     memory => {
 813         type => "number",
 814         description => "Amount of memory this NUMA node provides.",
 815         optional => 1,
 816     },
 817     hostnodes => {
 818         type => "string",
 819         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 820         description => "Host NUMA nodes to use.",
 821         format_description => "id[-id];...",
 822         optional => 1,
 823     },
 824     policy => {
 825         type => 'string',
 826         enum => [qw(preferred bind interleave)],
 827         description => "NUMA allocation policy.",
 828         optional => 1,
 829     },
 830 };
 831 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 832 my $numadesc = {
 833     optional => 1,
 834     type => 'string', format => $numa_fmt,
 835     description => "NUMA topology.",
 836 };
 837 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 838
 839 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 840     $confdesc->{"numa$i"} = $numadesc;
 841 }
 842
 843 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 844                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 845                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 846 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 847
 848 my $net_fmt_bridge_descr = <<__EOD__;
 849 Bridge to attach the network device to. The Proxmox VE standard bridge
 850 is called 'vmbr0'.
 851
 852 If you do not specify a bridge, we create a kvm user (NATed) network
 853 device, which provides DHCP and DNS services. The following addresses
 854 are used:
 855
 856  10.0.2.2   Gateway
 857  10.0.2.3   DNS Server
 858  10.0.2.4   SMB Server
 859
 860 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 861 __EOD__
 862
 863 my $net_fmt = {
 864     macaddr  => get_standard_option('mac-addr', {
 865         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 866     }),
 867     model => {
 868         type => 'string',
 869         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 870         enum => $nic_model_list,
 871         default_key => 1,
 872     },
 873     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 874     bridge => {
 875         type => 'string',
 876         description => $net_fmt_bridge_descr,
 877         format_description => 'bridge',
 878         optional => 1,
 879     },
 880     queues => {
 881         type => 'integer',
 882         minimum => 0, maximum => 16,
 883         description => 'Number of packet queues to be used on the device.',
 884         optional => 1,
 885     },
 886     rate => {
 887         type => 'number',
 888         minimum => 0,
 889         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 890         optional => 1,
 891     },
 892     tag => {
 893         type => 'integer',
 894         minimum => 1, maximum => 4094,
 895         description => 'VLAN tag to apply to packets on this interface.',
 896         optional => 1,
 897     },
 898     trunks => {
 899         type => 'string',
 900         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 901         description => 'VLAN trunks to pass through this interface.',
 902         format_description => 'vlanid[;vlanid...]',
 903         optional => 1,
 904     },
 905     firewall => {
 906         type => 'boolean',
 907         description => 'Whether this interface should be protected by the firewall.',
 908         optional => 1,
 909     },
 910     link_down => {
 911         type => 'boolean',
 912         description => 'Whether this interface should be disconnected (like pulling the plug).',
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 969 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 970
 971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 972 EODESCR
 973 };
 974 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 975
 976 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 977     $confdesc->{"net$i"} = $netdesc;
 978     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 979 }
 980
 981 foreach my $key (keys %$confdesc_cloudinit) {
 982     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 983 }
 984
 985 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 986 sub verify_volume_id_or_qm_path {
 987     my ($volid, $noerr) = @_;
 988
 989     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 990         return $volid;
 991     }
 992
 993     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 994     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 995     if ($@) {
 996         return undef if $noerr;
 997         die $@;
 998     }
 999     return $volid;
1000 }
1001
1002 my $drivename_hash;
1003
1004 my %drivedesc_base = (
1005     volume => { alias => 'file' },
1006     file => {
1007         type => 'string',
1008         format => 'pve-volume-id-or-qm-path',
1009         default_key => 1,
1010         format_description => 'volume',
1011         description => "The drive's backing volume.",
1012     },
1013     media => {
1014         type => 'string',
1015         enum => [qw(cdrom disk)],
1016         description => "The drive's media type.",
1017         default => 'disk',
1018         optional => 1
1019     },
1020     cyls => {
1021         type => 'integer',
1022         description => "Force the drive's physical geometry to have a specific cylinder count.",
1023         optional => 1
1024     },
1025     heads => {
1026         type => 'integer',
1027         description => "Force the drive's physical geometry to have a specific head count.",
1028         optional => 1
1029     },
1030     secs => {
1031         type => 'integer',
1032         description => "Force the drive's physical geometry to have a specific sector count.",
1033         optional => 1
1034     },
1035     trans => {
1036         type => 'string',
1037         enum => [qw(none lba auto)],
1038         description => "Force disk geometry bios translation mode.",
1039         optional => 1,
1040     },
1041     snapshot => {
1042         type => 'boolean',
1043         description => "Controls qemu's snapshot mode feature."
1044             . " If activated, changes made to the disk are temporary and will"
1045             . " be discarded when the VM is shutdown.",
1046         optional => 1,
1047     },
1048     cache => {
1049         type => 'string',
1050         enum => [qw(none writethrough writeback unsafe directsync)],
1051         description => "The drive's cache mode",
1052         optional => 1,
1053     },
1054     format => get_standard_option('pve-qm-image-format'),
1055     size => {
1056         type => 'string',
1057         format => 'disk-size',
1058         format_description => 'DiskSize',
1059         description => "Disk size. This is purely informational and has no effect.",
1060         optional => 1,
1061     },
1062     backup => {
1063         type => 'boolean',
1064         description => "Whether the drive should be included when making backups.",
1065         optional => 1,
1066     },
1067     replicate => {
1068         type => 'boolean',
1069         description => 'Whether the drive should considered for replication jobs.',
1070         optional => 1,
1071         default => 1,
1072     },
1073     rerror => {
1074         type => 'string',
1075         enum => [qw(ignore report stop)],
1076         description => 'Read error action.',
1077         optional => 1,
1078     },
1079     werror => {
1080         type => 'string',
1081         enum => [qw(enospc ignore report stop)],
1082         description => 'Write error action.',
1083         optional => 1,
1084     },
1085     aio => {
1086         type => 'string',
1087         enum => [qw(native threads)],
1088         description => 'AIO type to use.',
1089         optional => 1,
1090     },
1091     discard => {
1092         type => 'string',
1093         enum => [qw(ignore on)],
1094         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1095         optional => 1,
1096     },
1097     detect_zeroes => {
1098         type => 'boolean',
1099         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1100         optional => 1,
1101     },
1102     serial => {
1103         type => 'string',
1104         format => 'urlencoded',
1105         format_description => 'serial',
1106         maxLength => 20*3, # *3 since it's %xx url enoded
1107         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1108         optional => 1,
1109     },
1110     shared => {
1111         type => 'boolean',
1112         description => 'Mark this locally-managed volume as available on all nodes',
1113         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1114         optional => 1,
1115         default => 0,
1116     }
1117 );
1118
1119 my %iothread_fmt = ( iothread => {
1120         type => 'boolean',
1121         description => "Whether to use iothreads for this drive",
1122         optional => 1,
1123 });
1124
1125 my %model_fmt = (
1126     model => {
1127         type => 'string',
1128         format => 'urlencoded',
1129         format_description => 'model',
1130         maxLength => 40*3, # *3 since it's %xx url enoded
1131         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1132         optional => 1,
1133     },
1134 );
1135
1136 my %queues_fmt = (
1137     queues => {
1138         type => 'integer',
1139         description => "Number of queues.",
1140         minimum => 2,
1141         optional => 1
1142     }
1143 );
1144
1145 my %scsiblock_fmt = (
1146     scsiblock => {
1147         type => 'boolean',
1148         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1149         optional => 1,
1150         default => 0,
1151     },
1152 );
1153
1154 my %ssd_fmt = (
1155     ssd => {
1156         type => 'boolean',
1157         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1158         optional => 1,
1159     },
1160 );
1161
1162 my %wwn_fmt = (
1163     wwn => {
1164         type => 'string',
1165         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1166         format_description => 'wwn',
1167         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1168         optional => 1,
1169     },
1170 );
1171
1172 my $add_throttle_desc = sub {
1173     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1174     my $d = {
1175         type => $type,
1176         format_description => $unit,
1177         description => "Maximum $what in $longunit.",
1178         optional => 1,
1179     };
1180     $d->{minimum} = $minimum if defined($minimum);
1181     $drivedesc_base{$key} = $d;
1182 };
1183 # throughput: (leaky bucket)
1184 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1186 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1187 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1189 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1190 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1191 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1192 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1193
1194 # pools: (pool of IO before throttling starts taking effect)
1195 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1197 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1198 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1199 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1200 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1201
1202 # burst lengths
1203 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1205 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209
1210 # legacy support
1211 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1212 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1213 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1214 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1215
1216 my $ide_fmt = {
1217     %drivedesc_base,
1218     %model_fmt,
1219     %ssd_fmt,
1220     %wwn_fmt,
1221 };
1222 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1223
1224 my $idedesc = {
1225     optional => 1,
1226     type => 'string', format => $ide_fmt,
1227     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1228 };
1229 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1230
1231 my $scsi_fmt = {
1232     %drivedesc_base,
1233     %iothread_fmt,
1234     %queues_fmt,
1235     %scsiblock_fmt,
1236     %ssd_fmt,
1237     %wwn_fmt,
1238 };
1239 my $scsidesc = {
1240     optional => 1,
1241     type => 'string', format => $scsi_fmt,
1242     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1243 };
1244 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1245
1246 my $sata_fmt = {
1247     %drivedesc_base,
1248     %ssd_fmt,
1249     %wwn_fmt,
1250 };
1251 my $satadesc = {
1252     optional => 1,
1253     type => 'string', format => $sata_fmt,
1254     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1255 };
1256 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1257
1258 my $virtio_fmt = {
1259     %drivedesc_base,
1260     %iothread_fmt,
1261 };
1262 my $virtiodesc = {
1263     optional => 1,
1264     type => 'string', format => $virtio_fmt,
1265     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1266 };
1267 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1268
1269 my $alldrive_fmt = {
1270     %drivedesc_base,
1271     %iothread_fmt,
1272     %model_fmt,
1273     %queues_fmt,
1274     %scsiblock_fmt,
1275     %ssd_fmt,
1276     %wwn_fmt,
1277 };
1278
1279 my $efidisk_fmt = {
1280     volume => { alias => 'file' },
1281     file => {
1282         type => 'string',
1283         format => 'pve-volume-id-or-qm-path',
1284         default_key => 1,
1285         format_description => 'volume',
1286         description => "The drive's backing volume.",
1287     },
1288     format => get_standard_option('pve-qm-image-format'),
1289     size => {
1290         type => 'string',
1291         format => 'disk-size',
1292         format_description => 'DiskSize',
1293         description => "Disk size. This is purely informational and has no effect.",
1294         optional => 1,
1295     },
1296 };
1297
1298 my $efidisk_desc = {
1299     optional => 1,
1300     type => 'string', format => $efidisk_fmt,
1301     description => "Configure a Disk for storing EFI vars",
1302 };
1303
1304 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1305
1306 my $usb_fmt = {
1307     host => {
1308         default_key => 1,
1309         type => 'string', format => 'pve-qm-usb-device',
1310         format_description => 'HOSTUSBDEVICE|spice',
1311         description => <<EODESCR,
1312 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1313
1314  'bus-port(.port)*' (decimal numbers) or
1315  'vendor_id:product_id' (hexadeciaml numbers) or
1316  'spice'
1317
1318 You can use the 'lsusb -t' command to list existing usb devices.
1319
1320 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1321
1322 The value 'spice' can be used to add a usb redirection devices for spice.
1323 EODESCR
1324     },
1325     usb3 => {
1326         optional => 1,
1327         type => 'boolean',
1328         description => "Specifies whether if given host option is a USB3 device or port.",
1329         default => 0,
1330     },
1331 };
1332
1333 my $usbdesc = {
1334     optional => 1,
1335     type => 'string', format => $usb_fmt,
1336     description => "Configure an USB device (n is 0 to 4).",
1337 };
1338 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1339
1340 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1341 my $hostpci_fmt = {
1342     host => {
1343         default_key => 1,
1344         type => 'string',
1345         pattern => qr/$PCIRE(;$PCIRE)*/,
1346         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1347         description => <<EODESCR,
1348 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1349 of PCI virtual functions of the host. HOSTPCIID syntax is:
1350
1351 'bus:dev.func' (hexadecimal numbers)
1352
1353 You can us the 'lspci' command to list existing PCI devices.
1354 EODESCR
1355     },
1356     rombar => {
1357         type => 'boolean',
1358         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1359         optional => 1,
1360         default => 1,
1361     },
1362     romfile => {
1363         type => 'string',
1364         pattern => '[^,;]+',
1365         format_description => 'string',
1366         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1367         optional => 1,
1368     },
1369     pcie => {
1370         type => 'boolean',
1371         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1372         optional => 1,
1373         default => 0,
1374     },
1375     'x-vga' => {
1376         type => 'boolean',
1377         description =>  "Enable vfio-vga device support.",
1378         optional => 1,
1379         default => 0,
1380     },
1381     'mdev' => {
1382         type => 'string',
1383         format_description => 'string',
1384         pattern => '[^/\.:]+',
1385         optional => 1,
1386         description => <<EODESCR
1387 The type of mediated device to use.
1388 An instance of this type will be created on startup of the VM and
1389 will be cleaned up when the VM stops.
1390 EODESCR
1391     }
1392 };
1393 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1394
1395 my $hostpcidesc = {
1396         optional => 1,
1397         type => 'string', format => 'pve-qm-hostpci',
1398         description => "Map host PCI devices into guest.",
1399         verbose_description =>  <<EODESCR,
1400 Map host PCI devices into guest.
1401
1402 NOTE: This option allows direct access to host hardware. So it is no longer
1403 possible to migrate such machines - use with special care.
1404
1405 CAUTION: Experimental! User reported problems with this option.
1406 EODESCR
1407 };
1408 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1409
1410 my $serialdesc = {
1411         optional => 1,
1412         type => 'string',
1413         pattern => '(/dev/.+|socket)',
1414         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1415         verbose_description =>  <<EODESCR,
1416 Create a serial device inside the VM (n is 0 to 3), and pass through a
1417 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1418 host side (use 'qm terminal' to open a terminal connection).
1419
1420 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1421
1422 CAUTION: Experimental! User reported problems with this option.
1423 EODESCR
1424 };
1425
1426 my $paralleldesc= {
1427         optional => 1,
1428         type => 'string',
1429         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1430         description =>  "Map host parallel devices (n is 0 to 2).",
1431         verbose_description =>  <<EODESCR,
1432 Map host parallel devices (n is 0 to 2).
1433
1434 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1435
1436 CAUTION: Experimental! User reported problems with this option.
1437 EODESCR
1438 };
1439
1440 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1441     $confdesc->{"parallel$i"} = $paralleldesc;
1442 }
1443
1444 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1445     $confdesc->{"serial$i"} = $serialdesc;
1446 }
1447
1448 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1449     $confdesc->{"hostpci$i"} = $hostpcidesc;
1450 }
1451
1452 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1453     $drivename_hash->{"ide$i"} = 1;
1454     $confdesc->{"ide$i"} = $idedesc;
1455 }
1456
1457 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1458     $drivename_hash->{"sata$i"} = 1;
1459     $confdesc->{"sata$i"} = $satadesc;
1460 }
1461
1462 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1463     $drivename_hash->{"scsi$i"} = 1;
1464     $confdesc->{"scsi$i"} = $scsidesc ;
1465 }
1466
1467 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1468     $drivename_hash->{"virtio$i"} = 1;
1469     $confdesc->{"virtio$i"} = $virtiodesc;
1470 }
1471
1472 $drivename_hash->{efidisk0} = 1;
1473 $confdesc->{efidisk0} = $efidisk_desc;
1474
1475 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1476     $confdesc->{"usb$i"} = $usbdesc;
1477 }
1478
1479 my $unuseddesc = {
1480     optional => 1,
1481     type => 'string', format => 'pve-volume-id',
1482     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1483 };
1484
1485 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1486     $confdesc->{"unused$i"} = $unuseddesc;
1487 }
1488
1489 my $kvm_api_version = 0;
1490
1491 sub kvm_version {
1492     return $kvm_api_version if $kvm_api_version;
1493
1494     open my $fh, '<', '/dev/kvm'
1495         or return undef;
1496
1497     # 0xae00 => KVM_GET_API_VERSION
1498     $kvm_api_version = ioctl($fh, 0xae00, 0);
1499
1500     return $kvm_api_version;
1501 }
1502
1503 my $kvm_user_version = {};
1504 my $kvm_mtime = {};
1505
1506 sub kvm_user_version {
1507     my ($binary) = @_;
1508
1509     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1510     my $st = stat($binary);
1511
1512     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1513     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1514         $cachedmtime == $st->mtime;
1515
1516     $kvm_user_version->{$binary} = 'unknown';
1517     $kvm_mtime->{$binary} = $st->mtime;
1518
1519     my $code = sub {
1520         my $line = shift;
1521         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1522             $kvm_user_version->{$binary} = $2;
1523         }
1524     };
1525
1526     eval { run_command([$binary, '--version'], outfunc => $code); };
1527     warn $@ if $@;
1528
1529     return $kvm_user_version->{$binary};
1530
1531 }
1532
1533 sub kernel_has_vhost_net {
1534     return -c '/dev/vhost-net';
1535 }
1536
1537 sub valid_drive_names {
1538     # order is important - used to autoselect boot disk
1539     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1540             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1541             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1542             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1543             'efidisk0');
1544 }
1545
1546 sub is_valid_drivename {
1547     my $dev = shift;
1548
1549     return defined($drivename_hash->{$dev});
1550 }
1551
1552 sub option_exists {
1553     my $key = shift;
1554     return defined($confdesc->{$key});
1555 }
1556
1557 sub nic_models {
1558     return $nic_model_list;
1559 }
1560
1561 sub os_list_description {
1562
1563     return {
1564         other => 'Other',
1565         wxp => 'Windows XP',
1566         w2k => 'Windows 2000',
1567         w2k3 =>, 'Windows 2003',
1568         w2k8 => 'Windows 2008',
1569         wvista => 'Windows Vista',
1570         win7 => 'Windows 7',
1571         win8 => 'Windows 8/2012',
1572         win10 => 'Windows 10/2016',
1573         l24 => 'Linux 2.4',
1574         l26 => 'Linux 2.6',
1575     };
1576 }
1577
1578 my $cdrom_path;
1579
1580 sub get_cdrom_path {
1581
1582     return  $cdrom_path if $cdrom_path;
1583
1584     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1585     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1586     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1587 }
1588
1589 sub get_iso_path {
1590     my ($storecfg, $vmid, $cdrom) = @_;
1591
1592     if ($cdrom eq 'cdrom') {
1593         return get_cdrom_path();
1594     } elsif ($cdrom eq 'none') {
1595         return '';
1596     } elsif ($cdrom =~ m|^/|) {
1597         return $cdrom;
1598     } else {
1599         return PVE::Storage::path($storecfg, $cdrom);
1600     }
1601 }
1602
1603 # try to convert old style file names to volume IDs
1604 sub filename_to_volume_id {
1605     my ($vmid, $file, $media) = @_;
1606
1607      if (!($file eq 'none' || $file eq 'cdrom' ||
1608           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1609
1610         return undef if $file =~ m|/|;
1611
1612         if ($media && $media eq 'cdrom') {
1613             $file = "local:iso/$file";
1614         } else {
1615             $file = "local:$vmid/$file";
1616         }
1617     }
1618
1619     return $file;
1620 }
1621
1622 sub verify_media_type {
1623     my ($opt, $vtype, $media) = @_;
1624
1625     return if !$media;
1626
1627     my $etype;
1628     if ($media eq 'disk') {
1629         $etype = 'images';
1630     } elsif ($media eq 'cdrom') {
1631         $etype = 'iso';
1632     } else {
1633         die "internal error";
1634     }
1635
1636     return if ($vtype eq $etype);
1637
1638     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1639 }
1640
1641 sub cleanup_drive_path {
1642     my ($opt, $storecfg, $drive) = @_;
1643
1644     # try to convert filesystem paths to volume IDs
1645
1646     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1647         ($drive->{file} !~ m|^/dev/.+|) &&
1648         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1649         ($drive->{file} !~ m/^\d+$/)) {
1650         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1651         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1652         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1653         verify_media_type($opt, $vtype, $drive->{media});
1654         $drive->{file} = $volid;
1655     }
1656
1657     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1658 }
1659
1660 sub parse_hotplug_features {
1661     my ($data) = @_;
1662
1663     my $res = {};
1664
1665     return $res if $data eq '0';
1666
1667     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1668
1669     foreach my $feature (PVE::Tools::split_list($data)) {
1670         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1671             $res->{$1} = 1;
1672         } else {
1673             die "invalid hotplug feature '$feature'\n";
1674         }
1675     }
1676     return $res;
1677 }
1678
1679 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1680 sub pve_verify_hotplug_features {
1681     my ($value, $noerr) = @_;
1682
1683     return $value if parse_hotplug_features($value);
1684
1685     return undef if $noerr;
1686
1687     die "unable to parse hotplug option\n";
1688 }
1689
1690 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1691 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1692 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1693 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1694 #        [,iothread=on][,serial=serial][,model=model]
1695
1696 sub parse_drive {
1697     my ($key, $data) = @_;
1698
1699     my ($interface, $index);
1700
1701     if ($key =~ m/^([^\d]+)(\d+)$/) {
1702         $interface = $1;
1703         $index = $2;
1704     } else {
1705         return undef;
1706     }
1707
1708     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1709                                      : $confdesc->{$key}->{format};
1710     if (!$desc) {
1711         warn "invalid drive key: $key\n";
1712         return undef;
1713     }
1714     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1715     return undef if !$res;
1716     $res->{interface} = $interface;
1717     $res->{index} = $index;
1718
1719     my $error = 0;
1720     foreach my $opt (qw(bps bps_rd bps_wr)) {
1721         if (my $bps = defined(delete $res->{$opt})) {
1722             if (defined($res->{"m$opt"})) {
1723                 warn "both $opt and m$opt specified\n";
1724                 ++$error;
1725                 next;
1726             }
1727             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1728         }
1729     }
1730
1731     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1732     for my $requirement (
1733         [mbps_max => 'mbps'],
1734         [mbps_rd_max => 'mbps_rd'],
1735         [mbps_wr_max => 'mbps_wr'],
1736         [miops_max => 'miops'],
1737         [miops_rd_max => 'miops_rd'],
1738         [miops_wr_max => 'miops_wr'],
1739         [bps_max_length => 'mbps_max'],
1740         [bps_rd_max_length => 'mbps_rd_max'],
1741         [bps_wr_max_length => 'mbps_wr_max'],
1742         [iops_max_length => 'iops_max'],
1743         [iops_rd_max_length => 'iops_rd_max'],
1744         [iops_wr_max_length => 'iops_wr_max']) {
1745         my ($option, $requires) = @$requirement;
1746         if ($res->{$option} && !$res->{$requires}) {
1747             warn "$option requires $requires\n";
1748             ++$error;
1749         }
1750     }
1751
1752     return undef if $error;
1753
1754     return undef if $res->{mbps_rd} && $res->{mbps};
1755     return undef if $res->{mbps_wr} && $res->{mbps};
1756     return undef if $res->{iops_rd} && $res->{iops};
1757     return undef if $res->{iops_wr} && $res->{iops};
1758
1759     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1760         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1761         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1762         return undef if $res->{interface} eq 'virtio';
1763     }
1764
1765     if (my $size = $res->{size}) {
1766         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1767     }
1768
1769     return $res;
1770 }
1771
1772 sub print_drive {
1773     my ($vmid, $drive) = @_;
1774     my $data = { %$drive };
1775     delete $data->{$_} for qw(index interface);
1776     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1777 }
1778
1779 sub scsi_inquiry {
1780     my($fh, $noerr) = @_;
1781
1782     my $SG_IO = 0x2285;
1783     my $SG_GET_VERSION_NUM = 0x2282;
1784
1785     my $versionbuf = "\x00" x 8;
1786     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1787     if (!$ret) {
1788         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1789         return undef;
1790     }
1791     my $version = unpack("I", $versionbuf);
1792     if ($version < 30000) {
1793         die "scsi generic interface too old\n"  if !$noerr;
1794         return undef;
1795     }
1796
1797     my $buf = "\x00" x 36;
1798     my $sensebuf = "\x00" x 8;
1799     my $cmd = pack("C x3 C x1", 0x12, 36);
1800
1801     # see /usr/include/scsi/sg.h
1802     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1803
1804     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1805                       length($sensebuf), 0, length($buf), $buf,
1806                       $cmd, $sensebuf, 6000);
1807
1808     $ret = ioctl($fh, $SG_IO, $packet);
1809     if (!$ret) {
1810         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1811         return undef;
1812     }
1813
1814     my @res = unpack($sg_io_hdr_t, $packet);
1815     if ($res[17] || $res[18]) {
1816         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1817         return undef;
1818     }
1819
1820     my $res = {};
1821     (my $byte0, my $byte1, $res->{vendor},
1822      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1823
1824     $res->{removable} = $byte1 & 128 ? 1 : 0;
1825     $res->{type} = $byte0 & 31;
1826
1827     return $res;
1828 }
1829
1830 sub path_is_scsi {
1831     my ($path) = @_;
1832
1833     my $fh = IO::File->new("+<$path") || return undef;
1834     my $res = scsi_inquiry($fh, 1);
1835     close($fh);
1836
1837     return $res;
1838 }
1839
1840 sub machine_type_is_q35 {
1841     my ($conf) = @_;
1842
1843     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1844 }
1845
1846 sub print_tabletdevice_full {
1847     my ($conf, $arch) = @_;
1848
1849     my $q35 = machine_type_is_q35($conf);
1850
1851     # we use uhci for old VMs because tablet driver was buggy in older qemu
1852     my $usbbus;
1853     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1854         $usbbus = 'ehci';
1855     } else {
1856         $usbbus = 'uhci';
1857     }
1858
1859     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1860 }
1861
1862 sub print_keyboarddevice_full {
1863     my ($conf, $arch, $machine) = @_;
1864
1865     return undef if $arch ne 'aarch64';
1866
1867     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1868 }
1869
1870 sub print_drivedevice_full {
1871     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1872
1873     my $device = '';
1874     my $maxdev = 0;
1875
1876     if ($drive->{interface} eq 'virtio') {
1877         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1878         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1879         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1880     } elsif ($drive->{interface} eq 'scsi') {
1881
1882         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1883         my $unit = $drive->{index} % $maxdev;
1884         my $devicetype = 'hd';
1885         my $path = '';
1886         if (drive_is_cdrom($drive)) {
1887             $devicetype = 'cd';
1888         } else {
1889             if ($drive->{file} =~ m|^/|) {
1890                 $path = $drive->{file};
1891                 if (my $info = path_is_scsi($path)) {
1892                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1893                         $devicetype = 'block';
1894                     } elsif ($info->{type} == 1) { # tape
1895                         $devicetype = 'generic';
1896                     }
1897                 }
1898             } else {
1899                  $path = PVE::Storage::path($storecfg, $drive->{file});
1900             }
1901
1902             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1903             if ($path =~ m/^iscsi\:\/\// &&
1904                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1905                 $devicetype = 'generic';
1906             }
1907         }
1908
1909         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1910             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1911         } else {
1912             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1913         }
1914
1915         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1916             $device .= ",rotation_rate=1";
1917         }
1918         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1919
1920     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1921         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1922         my $controller = int($drive->{index} / $maxdev);
1923         my $unit = $drive->{index} % $maxdev;
1924         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1925
1926         $device = "ide-$devicetype";
1927         if ($drive->{interface} eq 'ide') {
1928             $device .= ",bus=ide.$controller,unit=$unit";
1929         } else {
1930             $device .= ",bus=ahci$controller.$unit";
1931         }
1932         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1933
1934         if ($devicetype eq 'hd') {
1935             if (my $model = $drive->{model}) {
1936                 $model = URI::Escape::uri_unescape($model);
1937                 $device .= ",model=$model";
1938             }
1939             if ($drive->{ssd}) {
1940                 $device .= ",rotation_rate=1";
1941             }
1942         }
1943         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1944     } elsif ($drive->{interface} eq 'usb') {
1945         die "implement me";
1946         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1947     } else {
1948         die "unsupported interface type";
1949     }
1950
1951     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1952
1953     if (my $serial = $drive->{serial}) {
1954         $serial = URI::Escape::uri_unescape($serial);
1955         $device .= ",serial=$serial";
1956     }
1957
1958
1959     return $device;
1960 }
1961
1962 sub get_initiator_name {
1963     my $initiator;
1964
1965     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1966     while (defined(my $line = <$fh>)) {
1967         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1968         $initiator = $1;
1969         last;
1970     }
1971     $fh->close();
1972
1973     return $initiator;
1974 }
1975
1976 sub print_drive_full {
1977     my ($storecfg, $vmid, $drive) = @_;
1978
1979     my $path;
1980     my $volid = $drive->{file};
1981     my $format;
1982
1983     if (drive_is_cdrom($drive)) {
1984         $path = get_iso_path($storecfg, $vmid, $volid);
1985     } else {
1986         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1987         if ($storeid) {
1988             $path = PVE::Storage::path($storecfg, $volid);
1989             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1990             $format = qemu_img_format($scfg, $volname);
1991         } else {
1992             $path = $volid;
1993             $format = "raw";
1994         }
1995    }
1996
1997     my $opts = '';
1998     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1999     foreach my $o (@qemu_drive_options) {
2000         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
2001     }
2002
2003     # snapshot only accepts on|off
2004     if (defined($drive->{snapshot})) {
2005         my $v = $drive->{snapshot} ? 'on' : 'off';
2006         $opts .= ",snapshot=$v";
2007     }
2008
2009     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2010         my ($dir, $qmpname) = @$type;
2011         if (my $v = $drive->{"mbps$dir"}) {
2012             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2013         }
2014         if (my $v = $drive->{"mbps${dir}_max"}) {
2015             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2016         }
2017         if (my $v = $drive->{"bps${dir}_max_length"}) {
2018             $opts .= ",throttling.bps$qmpname-max-length=$v";
2019         }
2020         if (my $v = $drive->{"iops${dir}"}) {
2021             $opts .= ",throttling.iops$qmpname=$v";
2022         }
2023         if (my $v = $drive->{"iops${dir}_max"}) {
2024             $opts .= ",throttling.iops$qmpname-max=$v";
2025         }
2026         if (my $v = $drive->{"iops${dir}_max_length"}) {
2027             $opts .= ",throttling.iops$qmpname-max-length=$v";
2028         }
2029     }
2030
2031     $opts .= ",format=$format" if $format && !$drive->{format};
2032
2033     my $cache_direct = 0;
2034
2035     if (my $cache = $drive->{cache}) {
2036         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2037     } elsif (!drive_is_cdrom($drive)) {
2038         $opts .= ",cache=none";
2039         $cache_direct = 1;
2040     }
2041
2042     # aio native works only with O_DIRECT
2043     if (!$drive->{aio}) {
2044         if($cache_direct) {
2045             $opts .= ",aio=native";
2046         } else {
2047             $opts .= ",aio=threads";
2048         }
2049     }
2050
2051     if (!drive_is_cdrom($drive)) {
2052         my $detectzeroes;
2053         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2054             $detectzeroes = 'off';
2055         } elsif ($drive->{discard}) {
2056             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2057         } else {
2058             # This used to be our default with discard not being specified:
2059             $detectzeroes = 'on';
2060         }
2061         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2062     }
2063
2064     my $pathinfo = $path ? "file=$path," : '';
2065
2066     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2067 }
2068
2069 sub print_netdevice_full {
2070     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2071
2072     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2073
2074     my $device = $net->{model};
2075     if ($net->{model} eq 'virtio') {
2076          $device = 'virtio-net-pci';
2077      };
2078
2079     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2080     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2081     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2082         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2083         my $vectors = $net->{queues} * 2 + 2;
2084         $tmpstr .= ",vectors=$vectors,mq=on";
2085     }
2086     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2087
2088     if ($use_old_bios_files) {
2089         my $romfile;
2090         if ($device eq 'virtio-net-pci') {
2091             $romfile = 'pxe-virtio.rom';
2092         } elsif ($device eq 'e1000') {
2093             $romfile = 'pxe-e1000.rom';
2094         } elsif ($device eq 'ne2k') {
2095             $romfile = 'pxe-ne2k_pci.rom';
2096         } elsif ($device eq 'pcnet') {
2097             $romfile = 'pxe-pcnet.rom';
2098         } elsif ($device eq 'rtl8139') {
2099             $romfile = 'pxe-rtl8139.rom';
2100         }
2101         $tmpstr .= ",romfile=$romfile" if $romfile;
2102     }
2103
2104     return $tmpstr;
2105 }
2106
2107 sub print_netdev_full {
2108     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2109
2110     my $i = '';
2111     if ($netid =~ m/^net(\d+)$/) {
2112         $i = int($1);
2113     }
2114
2115     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2116
2117     my $ifname = "tap${vmid}i$i";
2118
2119     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2120     die "interface name '$ifname' is too long (max 15 character)\n"
2121         if length($ifname) >= 16;
2122
2123     my $vhostparam = '';
2124     if (is_native($arch)) {
2125         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2126     }
2127
2128     my $vmname = $conf->{name} || "vm$vmid";
2129
2130     my $netdev = "";
2131     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2132
2133     if ($net->{bridge}) {
2134         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2135     } else {
2136         $netdev = "type=user,id=$netid,hostname=$vmname";
2137     }
2138
2139     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2140
2141     return $netdev;
2142 }
2143
2144
2145 sub print_cpu_device {
2146     my ($conf, $id) = @_;
2147
2148     my $kvm = $conf->{kvm} // 1;
2149     my $cpu = $kvm ? "kvm64" : "qemu64";
2150     if (my $cputype = $conf->{cpu}) {
2151         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2152             or die "Cannot parse cpu description: $cputype\n";
2153         $cpu = $cpuconf->{cputype};
2154     }
2155
2156     my $cores = $conf->{cores} || 1;
2157
2158     my $current_core = ($id - 1) % $cores;
2159     my $current_socket = int(($id - 1 - $current_core)/$cores);
2160
2161     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2162 }
2163
2164 my $vga_map = {
2165     'cirrus' => 'cirrus-vga',
2166     'std' => 'VGA',
2167     'vmware' => 'vmware-svga',
2168     'virtio' => 'virtio-vga',
2169 };
2170
2171 sub print_vga_device {
2172     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2173
2174     my $type = $vga_map->{$vga->{type}};
2175     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2176         $type = 'virtio-gpu';
2177     }
2178     my $vgamem_mb = $vga->{memory};
2179     if ($qxlnum) {
2180         $type = $id ? 'qxl' : 'qxl-vga';
2181     }
2182     die "no devicetype for $vga->{type}\n" if !$type;
2183
2184     my $memory = "";
2185     if ($vgamem_mb) {
2186         if ($vga->{type} eq 'virtio') {
2187             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2188             $memory = ",max_hostmem=$bytes";
2189         } elsif ($qxlnum) {
2190             # from https://www.spice-space.org/multiple-monitors.html
2191             $memory = ",vgamem_mb=$vga->{memory}";
2192             my $ram = $vgamem_mb * 4;
2193             my $vram = $vgamem_mb * 2;
2194             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2195         } else {
2196             $memory = ",vgamem_mb=$vga->{memory}";
2197         }
2198     } elsif ($qxlnum && $id) {
2199         $memory = ",ram_size=67108864,vram_size=33554432";
2200     }
2201
2202     my $q35 = machine_type_is_q35($conf);
2203     my $vgaid = "vga" . ($id // '');
2204     my $pciaddr;
2205
2206     if ($q35 && $vgaid eq 'vga') {
2207         # the first display uses pcie.0 bus on q35 machines
2208         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2209     } else {
2210         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2211     }
2212
2213     return "$type,id=${vgaid}${memory}${pciaddr}";
2214 }
2215
2216 sub drive_is_cloudinit {
2217     my ($drive) = @_;
2218     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2219 }
2220
2221 sub drive_is_cdrom {
2222     my ($drive, $exclude_cloudinit) = @_;
2223
2224     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2225
2226     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2227
2228 }
2229
2230 sub parse_number_sets {
2231     my ($set) = @_;
2232     my $res = [];
2233     foreach my $part (split(/;/, $set)) {
2234         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2235             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2236             push @$res, [ $1, $2 ];
2237         } else {
2238             die "invalid range: $part\n";
2239         }
2240     }
2241     return $res;
2242 }
2243
2244 sub parse_numa {
2245     my ($data) = @_;
2246
2247     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2248     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2249     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2250     return $res;
2251 }
2252
2253 sub parse_hostpci {
2254     my ($value) = @_;
2255
2256     return undef if !$value;
2257
2258     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2259
2260     my @idlist = split(/;/, $res->{host});
2261     delete $res->{host};
2262     foreach my $id (@idlist) {
2263         if ($id =~ m/\./) { # full id 00:00.1
2264             push @{$res->{pciid}}, {
2265                 id => $id,
2266             };
2267         } else { # partial id 00:00
2268             $res->{pciid} = PVE::SysFSTools::lspci($id);
2269         }
2270     }
2271     return $res;
2272 }
2273
2274 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2275 sub parse_net {
2276     my ($data) = @_;
2277
2278     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2279     if ($@) {
2280         warn $@;
2281         return undef;
2282     }
2283     if (!defined($res->{macaddr})) {
2284         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2285         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2286     }
2287     return $res;
2288 }
2289
2290 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2291 sub parse_ipconfig {
2292     my ($data) = @_;
2293
2294     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2295     if ($@) {
2296         warn $@;
2297         return undef;
2298     }
2299
2300     if ($res->{gw} && !$res->{ip}) {
2301         warn 'gateway specified without specifying an IP address';
2302         return undef;
2303     }
2304     if ($res->{gw6} && !$res->{ip6}) {
2305         warn 'IPv6 gateway specified without specifying an IPv6 address';
2306         return undef;
2307     }
2308     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2309         warn 'gateway specified together with DHCP';
2310         return undef;
2311     }
2312     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2313         # gw6 + auto/dhcp
2314         warn "IPv6 gateway specified together with $res->{ip6} address";
2315         return undef;
2316     }
2317
2318     if (!$res->{ip} && !$res->{ip6}) {
2319         return { ip => 'dhcp', ip6 => 'dhcp' };
2320     }
2321
2322     return $res;
2323 }
2324
2325 sub print_net {
2326     my $net = shift;
2327
2328     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2329 }
2330
2331 sub add_random_macs {
2332     my ($settings) = @_;
2333
2334     foreach my $opt (keys %$settings) {
2335         next if $opt !~ m/^net(\d+)$/;
2336         my $net = parse_net($settings->{$opt});
2337         next if !$net;
2338         $settings->{$opt} = print_net($net);
2339     }
2340 }
2341
2342 sub vm_is_volid_owner {
2343     my ($storecfg, $vmid, $volid) = @_;
2344
2345     if ($volid !~  m|^/|) {
2346         my ($path, $owner);
2347         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2348         if ($owner && ($owner == $vmid)) {
2349             return 1;
2350         }
2351     }
2352
2353     return undef;
2354 }
2355
2356 sub vmconfig_register_unused_drive {
2357     my ($storecfg, $vmid, $conf, $drive) = @_;
2358
2359     if (drive_is_cloudinit($drive)) {
2360         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2361         warn $@ if $@;
2362     } elsif (!drive_is_cdrom($drive)) {
2363         my $volid = $drive->{file};
2364         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2365             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2366         }
2367     }
2368 }
2369
2370 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2371 my $smbios1_fmt = {
2372     uuid => {
2373         type => 'string',
2374         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2375         format_description => 'UUID',
2376         description => "Set SMBIOS1 UUID.",
2377         optional => 1,
2378     },
2379     version => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 version.",
2384         optional => 1,
2385     },
2386     serial => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 serial number.",
2391         optional => 1,
2392     },
2393     manufacturer => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 manufacturer.",
2398         optional => 1,
2399     },
2400     product => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 product ID.",
2405         optional => 1,
2406     },
2407     sku => {
2408         type => 'string',
2409         pattern => '[A-Za-z0-9+\/]+={0,2}',
2410         format_description => 'Base64 encoded string',
2411         description => "Set SMBIOS1 SKU string.",
2412         optional => 1,
2413     },
2414     family => {
2415         type => 'string',
2416         pattern => '[A-Za-z0-9+\/]+={0,2}',
2417         format_description => 'Base64 encoded string',
2418         description => "Set SMBIOS1 family string.",
2419         optional => 1,
2420     },
2421     base64 => {
2422         type => 'boolean',
2423         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2424         optional => 1,
2425     },
2426 };
2427
2428 sub parse_smbios1 {
2429     my ($data) = @_;
2430
2431     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2432     warn $@ if $@;
2433     return $res;
2434 }
2435
2436 sub print_smbios1 {
2437     my ($smbios1) = @_;
2438     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2439 }
2440
2441 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2442
2443 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2444 sub verify_bootdisk {
2445     my ($value, $noerr) = @_;
2446
2447     return $value if is_valid_drivename($value);
2448
2449     return undef if $noerr;
2450
2451     die "invalid boot disk '$value'\n";
2452 }
2453
2454 sub parse_watchdog {
2455     my ($value) = @_;
2456
2457     return undef if !$value;
2458
2459     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2460     warn $@ if $@;
2461     return $res;
2462 }
2463
2464 sub parse_guest_agent {
2465     my ($value) = @_;
2466
2467     return {} if !defined($value->{agent});
2468
2469     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2470     warn $@ if $@;
2471
2472     # if the agent is disabled ignore the other potentially set properties
2473     return {} if !$res->{enabled};
2474     return $res;
2475 }
2476
2477 sub parse_vga {
2478     my ($value) = @_;
2479
2480     return {} if !$value;
2481     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2482     warn $@ if $@;
2483     return $res;
2484 }
2485
2486 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2487 sub verify_usb_device {
2488     my ($value, $noerr) = @_;
2489
2490     return $value if parse_usb_device($value);
2491
2492     return undef if $noerr;
2493
2494     die "unable to parse usb device\n";
2495 }
2496
2497 # add JSON properties for create and set function
2498 sub json_config_properties {
2499     my $prop = shift;
2500
2501     foreach my $opt (keys %$confdesc) {
2502         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2503         $prop->{$opt} = $confdesc->{$opt};
2504     }
2505
2506     return $prop;
2507 }
2508
2509 # return copy of $confdesc_cloudinit to generate documentation
2510 sub cloudinit_config_properties {
2511
2512     return dclone($confdesc_cloudinit);
2513 }
2514
2515 sub check_type {
2516     my ($key, $value) = @_;
2517
2518     die "unknown setting '$key'\n" if !$confdesc->{$key};
2519
2520     my $type = $confdesc->{$key}->{type};
2521
2522     if (!defined($value)) {
2523         die "got undefined value\n";
2524     }
2525
2526     if ($value =~ m/[\n\r]/) {
2527         die "property contains a line feed\n";
2528     }
2529
2530     if ($type eq 'boolean') {
2531         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2532         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2533         die "type check ('boolean') failed - got '$value'\n";
2534     } elsif ($type eq 'integer') {
2535         return int($1) if $value =~ m/^(\d+)$/;
2536         die "type check ('integer') failed - got '$value'\n";
2537     } elsif ($type eq 'number') {
2538         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2539         die "type check ('number') failed - got '$value'\n";
2540     } elsif ($type eq 'string') {
2541         if (my $fmt = $confdesc->{$key}->{format}) {
2542             PVE::JSONSchema::check_format($fmt, $value);
2543             return $value;
2544         }
2545         $value =~ s/^\"(.*)\"$/$1/;
2546         return $value;
2547     } else {
2548         die "internal error"
2549     }
2550 }
2551
2552 sub destroy_vm {
2553     my ($storecfg, $vmid, $skiplock, $replacement_conf) = @_;
2554
2555     my $conf = PVE::QemuConfig->load_config($vmid);
2556
2557     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2558
2559     if ($conf->{template}) {
2560         # check if any base image is still used by a linked clone
2561         foreach_drive($conf, sub {
2562                 my ($ds, $drive) = @_;
2563                 return if drive_is_cdrom($drive);
2564
2565                 my $volid = $drive->{file};
2566                 return if !$volid || $volid =~ m|^/|;
2567
2568                 die "base volume '$volid' is still in use by linked cloned\n"
2569                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2570
2571         });
2572     }
2573
2574     # only remove disks owned by this VM
2575     foreach_drive($conf, sub {
2576         my ($ds, $drive) = @_;
2577         return if drive_is_cdrom($drive, 1);
2578
2579         my $volid = $drive->{file};
2580         return if !$volid || $volid =~ m|^/|;
2581
2582         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2583         return if !$path || !$owner || ($owner != $vmid);
2584
2585         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2586         warn "Could not remove disk '$volid', check manually: $@" if $@;
2587     });
2588
2589     # also remove unused disk
2590     my $vmdisks = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2591     PVE::Storage::foreach_volid($vmdisks, sub {
2592         my ($volid, $sid, $volname, $d) = @_;
2593         eval { PVE::Storage::vdisk_free($storecfg, $volid) };
2594         warn $@ if $@;
2595     });
2596
2597     if (defined $replacement_conf) {
2598         PVE::LXC::Config->write_config($vmid, $replacement_conf);
2599     } else {
2600         PVE::QemuConfig->destroy_config($vmid);
2601     }
2602 }
2603
2604 sub parse_vm_config {
2605     my ($filename, $raw) = @_;
2606
2607     return undef if !defined($raw);
2608
2609     my $res = {
2610         digest => Digest::SHA::sha1_hex($raw),
2611         snapshots => {},
2612         pending => {},
2613     };
2614
2615     $filename =~ m|/qemu-server/(\d+)\.conf$|
2616         || die "got strange filename '$filename'";
2617
2618     my $vmid = $1;
2619
2620     my $conf = $res;
2621     my $descr;
2622     my $section = '';
2623
2624     my @lines = split(/\n/, $raw);
2625     foreach my $line (@lines) {
2626         next if $line =~ m/^\s*$/;
2627
2628         if ($line =~ m/^\[PENDING\]\s*$/i) {
2629             $section = 'pending';
2630             if (defined($descr)) {
2631                 $descr =~ s/\s+$//;
2632                 $conf->{description} = $descr;
2633             }
2634             $descr = undef;
2635             $conf = $res->{$section} = {};
2636             next;
2637
2638         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2639             $section = $1;
2640             if (defined($descr)) {
2641                 $descr =~ s/\s+$//;
2642                 $conf->{description} = $descr;
2643             }
2644             $descr = undef;
2645             $conf = $res->{snapshots}->{$section} = {};
2646             next;
2647         }
2648
2649         if ($line =~ m/^\#(.*)\s*$/) {
2650             $descr = '' if !defined($descr);
2651             $descr .= PVE::Tools::decode_text($1) . "\n";
2652             next;
2653         }
2654
2655         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2656             $descr = '' if !defined($descr);
2657             $descr .= PVE::Tools::decode_text($2);
2658         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2659             $conf->{snapstate} = $1;
2660         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2661             my $key = $1;
2662             my $value = $2;
2663             $conf->{$key} = $value;
2664         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2665             my $value = $1;
2666             if ($section eq 'pending') {
2667                 $conf->{delete} = $value; # we parse this later
2668             } else {
2669                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2670             }
2671         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2672             my $key = $1;
2673             my $value = $2;
2674             eval { $value = check_type($key, $value); };
2675             if ($@) {
2676                 warn "vm $vmid - unable to parse value of '$key' - $@";
2677             } else {
2678                 $key = 'ide2' if $key eq 'cdrom';
2679                 my $fmt = $confdesc->{$key}->{format};
2680                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2681                     my $v = parse_drive($key, $value);
2682                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2683                         $v->{file} = $volid;
2684                         $value = print_drive($vmid, $v);
2685                     } else {
2686                         warn "vm $vmid - unable to parse value of '$key'\n";
2687                         next;
2688                     }
2689                 }
2690
2691                 $conf->{$key} = $value;
2692             }
2693         }
2694     }
2695
2696     if (defined($descr)) {
2697         $descr =~ s/\s+$//;
2698         $conf->{description} = $descr;
2699     }
2700     delete $res->{snapstate}; # just to be sure
2701
2702     return $res;
2703 }
2704
2705 sub write_vm_config {
2706     my ($filename, $conf) = @_;
2707
2708     delete $conf->{snapstate}; # just to be sure
2709
2710     if ($conf->{cdrom}) {
2711         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2712         $conf->{ide2} = $conf->{cdrom};
2713         delete $conf->{cdrom};
2714     }
2715
2716     # we do not use 'smp' any longer
2717     if ($conf->{sockets}) {
2718         delete $conf->{smp};
2719     } elsif ($conf->{smp}) {
2720         $conf->{sockets} = $conf->{smp};
2721         delete $conf->{cores};
2722         delete $conf->{smp};
2723     }
2724
2725     my $used_volids = {};
2726
2727     my $cleanup_config = sub {
2728         my ($cref, $pending, $snapname) = @_;
2729
2730         foreach my $key (keys %$cref) {
2731             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2732                 $key eq 'snapstate' || $key eq 'pending';
2733             my $value = $cref->{$key};
2734             if ($key eq 'delete') {
2735                 die "propertry 'delete' is only allowed in [PENDING]\n"
2736                     if !$pending;
2737                 # fixme: check syntax?
2738                 next;
2739             }
2740             eval { $value = check_type($key, $value); };
2741             die "unable to parse value of '$key' - $@" if $@;
2742
2743             $cref->{$key} = $value;
2744
2745             if (!$snapname && is_valid_drivename($key)) {
2746                 my $drive = parse_drive($key, $value);
2747                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2748             }
2749         }
2750     };
2751
2752     &$cleanup_config($conf);
2753
2754     &$cleanup_config($conf->{pending}, 1);
2755
2756     foreach my $snapname (keys %{$conf->{snapshots}}) {
2757         die "internal error" if $snapname eq 'pending';
2758         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2759     }
2760
2761     # remove 'unusedX' settings if we re-add a volume
2762     foreach my $key (keys %$conf) {
2763         my $value = $conf->{$key};
2764         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2765             delete $conf->{$key};
2766         }
2767     }
2768
2769     my $generate_raw_config = sub {
2770         my ($conf, $pending) = @_;
2771
2772         my $raw = '';
2773
2774         # add description as comment to top of file
2775         if (defined(my $descr = $conf->{description})) {
2776             if ($descr) {
2777                 foreach my $cl (split(/\n/, $descr)) {
2778                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2779                 }
2780             } else {
2781                 $raw .= "#\n" if $pending;
2782             }
2783         }
2784
2785         foreach my $key (sort keys %$conf) {
2786             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2787             $raw .= "$key: $conf->{$key}\n";
2788         }
2789         return $raw;
2790     };
2791
2792     my $raw = &$generate_raw_config($conf);
2793
2794     if (scalar(keys %{$conf->{pending}})){
2795         $raw .= "\n[PENDING]\n";
2796         $raw .= &$generate_raw_config($conf->{pending}, 1);
2797     }
2798
2799     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2800         $raw .= "\n[$snapname]\n";
2801         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2802     }
2803
2804     return $raw;
2805 }
2806
2807 sub load_defaults {
2808
2809     my $res = {};
2810
2811     # we use static defaults from our JSON schema configuration
2812     foreach my $key (keys %$confdesc) {
2813         if (defined(my $default = $confdesc->{$key}->{default})) {
2814             $res->{$key} = $default;
2815         }
2816     }
2817
2818     return $res;
2819 }
2820
2821 sub config_list {
2822     my $vmlist = PVE::Cluster::get_vmlist();
2823     my $res = {};
2824     return $res if !$vmlist || !$vmlist->{ids};
2825     my $ids = $vmlist->{ids};
2826
2827     foreach my $vmid (keys %$ids) {
2828         my $d = $ids->{$vmid};
2829         next if !$d->{node} || $d->{node} ne $nodename;
2830         next if !$d->{type} || $d->{type} ne 'qemu';
2831         $res->{$vmid}->{exists} = 1;
2832     }
2833     return $res;
2834 }
2835
2836 # test if VM uses local resources (to prevent migration)
2837 sub check_local_resources {
2838     my ($conf, $noerr) = @_;
2839
2840     my @loc_res = ();
2841
2842     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2843     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2844
2845     push @loc_res, "ivshmem" if $conf->{ivshmem};
2846
2847     foreach my $k (keys %$conf) {
2848         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2849         # sockets are safe: they will recreated be on the target side post-migrate
2850         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2851         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2852     }
2853
2854     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2855
2856     return \@loc_res;
2857 }
2858
2859 # check if used storages are available on all nodes (use by migrate)
2860 sub check_storage_availability {
2861     my ($storecfg, $conf, $node) = @_;
2862
2863     foreach_drive($conf, sub {
2864         my ($ds, $drive) = @_;
2865
2866         my $volid = $drive->{file};
2867         return if !$volid;
2868
2869         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2870         return if !$sid;
2871
2872         # check if storage is available on both nodes
2873         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2874         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2875    });
2876 }
2877
2878 # list nodes where all VM images are available (used by has_feature API)
2879 sub shared_nodes {
2880     my ($conf, $storecfg) = @_;
2881
2882     my $nodelist = PVE::Cluster::get_nodelist();
2883     my $nodehash = { map { $_ => 1 } @$nodelist };
2884     my $nodename = PVE::INotify::nodename();
2885
2886     foreach_drive($conf, sub {
2887         my ($ds, $drive) = @_;
2888
2889         my $volid = $drive->{file};
2890         return if !$volid;
2891
2892         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2893         if ($storeid) {
2894             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2895             if ($scfg->{disable}) {
2896                 $nodehash = {};
2897             } elsif (my $avail = $scfg->{nodes}) {
2898                 foreach my $node (keys %$nodehash) {
2899                     delete $nodehash->{$node} if !$avail->{$node};
2900                 }
2901             } elsif (!$scfg->{shared}) {
2902                 foreach my $node (keys %$nodehash) {
2903                     delete $nodehash->{$node} if $node ne $nodename
2904                 }
2905             }
2906         }
2907     });
2908
2909     return $nodehash
2910 }
2911
2912 sub check_local_storage_availability {
2913     my ($conf, $storecfg) = @_;
2914
2915     my $nodelist = PVE::Cluster::get_nodelist();
2916     my $nodehash = { map { $_ => {} } @$nodelist };
2917
2918     foreach_drive($conf, sub {
2919         my ($ds, $drive) = @_;
2920
2921         my $volid = $drive->{file};
2922         return if !$volid;
2923
2924         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2925         if ($storeid) {
2926             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2927
2928             if ($scfg->{disable}) {
2929                 foreach my $node (keys %$nodehash) {
2930                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2931                 }
2932             } elsif (my $avail = $scfg->{nodes}) {
2933                 foreach my $node (keys %$nodehash) {
2934                     if (!$avail->{$node}) {
2935                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2936                     }
2937                 }
2938             }
2939         }
2940     });
2941
2942     foreach my $node (values %$nodehash) {
2943         if (my $unavail = $node->{unavailable_storages}) {
2944             $node->{unavailable_storages} = [ sort keys %$unavail ];
2945         }
2946     }
2947
2948     return $nodehash
2949 }
2950
2951 sub check_cmdline {
2952     my ($pidfile, $pid) = @_;
2953
2954     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2955     if (defined($fh)) {
2956         my $line = <$fh>;
2957         $fh->close;
2958         return undef if !$line;
2959         my @param = split(/\0/, $line);
2960
2961         my $cmd = $param[0];
2962         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2963
2964         for (my $i = 0; $i < scalar (@param); $i++) {
2965             my $p = $param[$i];
2966             next if !$p;
2967             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2968                 my $p = $param[$i+1];
2969                 return 1 if $p && ($p eq $pidfile);
2970                 return undef;
2971             }
2972         }
2973     }
2974     return undef;
2975 }
2976
2977 sub check_running {
2978     my ($vmid, $nocheck, $node) = @_;
2979
2980     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2981
2982     die "unable to find configuration file for VM $vmid - no such machine\n"
2983         if !$nocheck && ! -f $filename;
2984
2985     my $pidfile = pidfile_name($vmid);
2986
2987     if (my $fd = IO::File->new("<$pidfile")) {
2988         my $st = stat($fd);
2989         my $line = <$fd>;
2990         close($fd);
2991
2992         my $mtime = $st->mtime;
2993         if ($mtime > time()) {
2994             warn "file '$filename' modified in future\n";
2995         }
2996
2997         if ($line =~ m/^(\d+)$/) {
2998             my $pid = $1;
2999             if (check_cmdline($pidfile, $pid)) {
3000                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3001                     return $pid;
3002                 }
3003             }
3004         }
3005     }
3006
3007     return undef;
3008 }
3009
3010 sub vzlist {
3011
3012     my $vzlist = config_list();
3013
3014     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3015
3016     while (defined(my $de = $fd->read)) {
3017         next if $de !~ m/^(\d+)\.pid$/;
3018         my $vmid = $1;
3019         next if !defined($vzlist->{$vmid});
3020         if (my $pid = check_running($vmid)) {
3021             $vzlist->{$vmid}->{pid} = $pid;
3022         }
3023     }
3024
3025     return $vzlist;
3026 }
3027
3028 sub disksize {
3029     my ($storecfg, $conf) = @_;
3030
3031     my $bootdisk = $conf->{bootdisk};
3032     return undef if !$bootdisk;
3033     return undef if !is_valid_drivename($bootdisk);
3034
3035     return undef if !$conf->{$bootdisk};
3036
3037     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3038     return undef if !defined($drive);
3039
3040     return undef if drive_is_cdrom($drive);
3041
3042     my $volid = $drive->{file};
3043     return undef if !$volid;
3044
3045     return $drive->{size};
3046 }
3047
3048 our $vmstatus_return_properties = {
3049     vmid => get_standard_option('pve-vmid'),
3050     status => {
3051         description => "Qemu process status.",
3052         type => 'string',
3053         enum => ['stopped', 'running'],
3054     },
3055     maxmem => {
3056         description => "Maximum memory in bytes.",
3057         type => 'integer',
3058         optional => 1,
3059         renderer => 'bytes',
3060     },
3061     maxdisk => {
3062         description => "Root disk size in bytes.",
3063         type => 'integer',
3064         optional => 1,
3065         renderer => 'bytes',
3066     },
3067     name => {
3068         description => "VM name.",
3069         type => 'string',
3070         optional => 1,
3071     },
3072     qmpstatus => {
3073         description => "Qemu QMP agent status.",
3074         type => 'string',
3075         optional => 1,
3076     },
3077     pid => {
3078         description => "PID of running qemu process.",
3079         type => 'integer',
3080         optional => 1,
3081     },
3082     uptime => {
3083         description => "Uptime.",
3084         type => 'integer',
3085         optional => 1,
3086         renderer => 'duration',
3087     },
3088     cpus => {
3089         description => "Maximum usable CPUs.",
3090         type => 'number',
3091         optional => 1,
3092     },
3093     lock => {
3094         description => "The current config lock, if any.",
3095         type => 'string',
3096         optional => 1,
3097     }
3098 };
3099
3100 my $last_proc_pid_stat;
3101
3102 # get VM status information
3103 # This must be fast and should not block ($full == false)
3104 # We only query KVM using QMP if $full == true (this can be slow)
3105 sub vmstatus {
3106     my ($opt_vmid, $full) = @_;
3107
3108     my $res = {};
3109
3110     my $storecfg = PVE::Storage::config();
3111
3112     my $list = vzlist();
3113     my $defaults = load_defaults();
3114
3115     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3116
3117     my $cpucount = $cpuinfo->{cpus} || 1;
3118
3119     foreach my $vmid (keys %$list) {
3120         next if $opt_vmid && ($vmid ne $opt_vmid);
3121
3122         my $conf = PVE::QemuConfig->load_config($vmid);
3123
3124         my $d = { vmid => $vmid };
3125         $d->{pid} = $list->{$vmid}->{pid};
3126
3127         # fixme: better status?
3128         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3129
3130         my $size = disksize($storecfg, $conf);
3131         if (defined($size)) {
3132             $d->{disk} = 0; # no info available
3133             $d->{maxdisk} = $size;
3134         } else {
3135             $d->{disk} = 0;
3136             $d->{maxdisk} = 0;
3137         }
3138
3139         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3140             * ($conf->{cores} || $defaults->{cores});
3141         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3142         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3143
3144         $d->{name} = $conf->{name} || "VM $vmid";
3145         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3146             : $defaults->{memory}*(1024*1024);
3147
3148         if ($conf->{balloon}) {
3149             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3150             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3151                 : $defaults->{shares};
3152         }
3153
3154         $d->{uptime} = 0;
3155         $d->{cpu} = 0;
3156         $d->{mem} = 0;
3157
3158         $d->{netout} = 0;
3159         $d->{netin} = 0;
3160
3161         $d->{diskread} = 0;
3162         $d->{diskwrite} = 0;
3163
3164         $d->{template} = PVE::QemuConfig->is_template($conf);
3165
3166         $d->{serial} = 1 if conf_has_serial($conf);
3167         $d->{lock} = $conf->{lock} if $conf->{lock};
3168
3169         $res->{$vmid} = $d;
3170     }
3171
3172     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3173     foreach my $dev (keys %$netdev) {
3174         next if $dev !~ m/^tap([1-9]\d*)i/;
3175         my $vmid = $1;
3176         my $d = $res->{$vmid};
3177         next if !$d;
3178
3179         $d->{netout} += $netdev->{$dev}->{receive};
3180         $d->{netin} += $netdev->{$dev}->{transmit};
3181
3182         if ($full) {
3183             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3184             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3185         }
3186
3187     }
3188
3189     my $ctime = gettimeofday;
3190
3191     foreach my $vmid (keys %$list) {
3192
3193         my $d = $res->{$vmid};
3194         my $pid = $d->{pid};
3195         next if !$pid;
3196
3197         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3198         next if !$pstat; # not running
3199
3200         my $used = $pstat->{utime} + $pstat->{stime};
3201
3202         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3203
3204         if ($pstat->{vsize}) {
3205             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3206         }
3207
3208         my $old = $last_proc_pid_stat->{$pid};
3209         if (!$old) {
3210             $last_proc_pid_stat->{$pid} = {
3211                 time => $ctime,
3212                 used => $used,
3213                 cpu => 0,
3214             };
3215             next;
3216         }
3217
3218         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3219
3220         if ($dtime > 1000) {
3221             my $dutime = $used -  $old->{used};
3222
3223             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3224             $last_proc_pid_stat->{$pid} = {
3225                 time => $ctime,
3226                 used => $used,
3227                 cpu => $d->{cpu},
3228             };
3229         } else {
3230             $d->{cpu} = $old->{cpu};
3231         }
3232     }
3233
3234     return $res if !$full;
3235
3236     my $qmpclient = PVE::QMPClient->new();
3237
3238     my $ballooncb = sub {
3239         my ($vmid, $resp) = @_;
3240
3241         my $info = $resp->{'return'};
3242         return if !$info->{max_mem};
3243
3244         my $d = $res->{$vmid};
3245
3246         # use memory assigned to VM
3247         $d->{maxmem} = $info->{max_mem};
3248         $d->{balloon} = $info->{actual};
3249
3250         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3251             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3252             $d->{freemem} = $info->{free_mem};
3253         }
3254
3255         $d->{ballooninfo} = $info;
3256     };
3257
3258     my $blockstatscb = sub {
3259         my ($vmid, $resp) = @_;
3260         my $data = $resp->{'return'} || [];
3261         my $totalrdbytes = 0;
3262         my $totalwrbytes = 0;
3263
3264         for my $blockstat (@$data) {
3265             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3266             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3267
3268             $blockstat->{device} =~ s/drive-//;
3269             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3270         }
3271         $res->{$vmid}->{diskread} = $totalrdbytes;
3272         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3273     };
3274
3275     my $statuscb = sub {
3276         my ($vmid, $resp) = @_;
3277
3278         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3279         # this fails if ballon driver is not loaded, so this must be
3280         # the last commnand (following command are aborted if this fails).
3281         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3282
3283         my $status = 'unknown';
3284         if (!defined($status = $resp->{'return'}->{status})) {
3285             warn "unable to get VM status\n";
3286             return;
3287         }
3288
3289         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3290     };
3291
3292     foreach my $vmid (keys %$list) {
3293         next if $opt_vmid && ($vmid ne $opt_vmid);
3294         next if !$res->{$vmid}->{pid}; # not running
3295         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3296     }
3297
3298     $qmpclient->queue_execute(undef, 2);
3299
3300     foreach my $vmid (keys %$list) {
3301         next if $opt_vmid && ($vmid ne $opt_vmid);
3302         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3303     }
3304
3305     return $res;
3306 }
3307
3308 sub foreach_drive {
3309     my ($conf, $func, @param) = @_;
3310
3311     foreach my $ds (valid_drive_names()) {
3312         next if !defined($conf->{$ds});
3313
3314         my $drive = parse_drive($ds, $conf->{$ds});
3315         next if !$drive;
3316
3317         &$func($ds, $drive, @param);
3318     }
3319 }
3320
3321 sub foreach_volid {
3322     my ($conf, $func, @param) = @_;
3323
3324     my $volhash = {};
3325
3326     my $test_volid = sub {
3327         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3328
3329         return if !$volid;
3330
3331         $volhash->{$volid}->{cdrom} //= 1;
3332         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3333
3334         $volhash->{$volid}->{replicate} //= 0;
3335         $volhash->{$volid}->{replicate} = 1 if $replicate;
3336
3337         $volhash->{$volid}->{shared} //= 0;
3338         $volhash->{$volid}->{shared} = 1 if $shared;
3339
3340         $volhash->{$volid}->{referenced_in_config} //= 0;
3341         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3342
3343         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3344             if defined($snapname);
3345         $volhash->{$volid}->{size} = $size if $size;
3346     };
3347
3348     foreach_drive($conf, sub {
3349         my ($ds, $drive) = @_;
3350         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3351     });
3352
3353     foreach my $snapname (keys %{$conf->{snapshots}}) {
3354         my $snap = $conf->{snapshots}->{$snapname};
3355         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3356         foreach_drive($snap, sub {
3357             my ($ds, $drive) = @_;
3358             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3359         });
3360     }
3361
3362     foreach my $volid (keys %$volhash) {
3363         &$func($volid, $volhash->{$volid}, @param);
3364     }
3365 }
3366
3367 sub conf_has_serial {
3368     my ($conf) = @_;
3369
3370     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3371         if ($conf->{"serial$i"}) {
3372             return 1;
3373         }
3374     }
3375
3376     return 0;
3377 }
3378
3379 sub conf_has_audio {
3380     my ($conf, $id) = @_;
3381
3382     $id //= 0;
3383     my $audio = $conf->{"audio$id"};
3384     return undef if !defined($audio);
3385
3386     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3387     my $audiodriver = $audioproperties->{driver} // 'spice';
3388
3389     return {
3390         dev => $audioproperties->{device},
3391         dev_id => "audiodev$id",
3392         backend => $audiodriver,
3393         backend_id => "$audiodriver-backend${id}",
3394     };
3395 }
3396
3397 sub vga_conf_has_spice {
3398     my ($vga) = @_;
3399
3400     my $vgaconf = parse_vga($vga);
3401     my $vgatype = $vgaconf->{type};
3402     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3403
3404     return $1 || 1;
3405 }
3406
3407 sub is_native($) {
3408     my ($arch) = @_;
3409     return get_host_arch() eq $arch;
3410 }
3411
3412 my $default_machines = {
3413     x86_64 => 'pc',
3414     aarch64 => 'virt',
3415 };
3416
3417 sub get_basic_machine_info {
3418     my ($conf, $forcemachine) = @_;
3419
3420     my $arch = $conf->{arch} // get_host_arch();
3421     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3422     return ($arch, $machine);
3423 }
3424
3425 sub get_ovmf_files($) {
3426     my ($arch) = @_;
3427
3428     my $ovmf = $OVMF->{$arch}
3429         or die "no OVMF images known for architecture '$arch'\n";
3430
3431     return @$ovmf;
3432 }
3433
3434 my $Arch2Qemu = {
3435     aarch64 => '/usr/bin/qemu-system-aarch64',
3436     x86_64 => '/usr/bin/qemu-system-x86_64',
3437 };
3438 sub get_command_for_arch($) {
3439     my ($arch) = @_;
3440     return '/usr/bin/kvm' if is_native($arch);
3441
3442     my $cmd = $Arch2Qemu->{$arch}
3443         or die "don't know how to emulate architecture '$arch'\n";
3444     return $cmd;
3445 }
3446
3447 sub get_cpu_options {
3448     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3449
3450     my $cpuFlags = [];
3451     my $ostype = $conf->{ostype};
3452
3453     my $cpu = $kvm ? "kvm64" : "qemu64";
3454     if ($arch eq 'aarch64') {
3455         $cpu = 'cortex-a57';
3456     }
3457     my $hv_vendor_id;
3458     if (my $cputype = $conf->{cpu}) {
3459         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3460             or die "Cannot parse cpu description: $cputype\n";
3461         $cpu = $cpuconf->{cputype};
3462         $kvm_off = 1 if $cpuconf->{hidden};
3463         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3464
3465         if (defined(my $flags = $cpuconf->{flags})) {
3466             push @$cpuFlags, split(";", $flags);
3467         }
3468     }
3469
3470     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3471
3472     push @$cpuFlags , '-x2apic'
3473         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3474
3475     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3476
3477     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3478
3479     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3480
3481         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3482         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3483     }
3484
3485     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3486
3487     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3488
3489     push @$cpuFlags, 'kvm=off' if $kvm_off;
3490
3491     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3492         push @$cpuFlags, "vendor=${cpu_vendor}"
3493             if $cpu_vendor ne 'default';
3494     } elsif ($arch ne 'aarch64') {
3495         die "internal error"; # should not happen
3496     }
3497
3498     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3499
3500     return ('-cpu', $cpu);
3501 }
3502
3503 sub config_to_command {
3504     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3505
3506     my $cmd = [];
3507     my $globalFlags = [];
3508     my $machineFlags = [];
3509     my $rtcFlags = [];
3510     my $devices = [];
3511     my $pciaddr = '';
3512     my $bridges = {};
3513     my $vernum = 0; # unknown
3514     my $ostype = $conf->{ostype};
3515     my $winversion = windows_version($ostype);
3516     my $kvm = $conf->{kvm};
3517
3518     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3519     my $kvm_binary = get_command_for_arch($arch);
3520     my $kvmver = kvm_user_version($kvm_binary);
3521     $kvm //= 1 if is_native($arch);
3522
3523     if ($kvm) {
3524         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3525             if !defined kvm_version();
3526     }
3527
3528     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3529         $vernum = $1*1000000+$2*1000;
3530     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3531         $vernum = $1*1000000+$2*1000+$3;
3532     }
3533
3534     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3535
3536     my $q35 = machine_type_is_q35($conf);
3537     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3538     my $use_old_bios_files = undef;
3539     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3540
3541     my $cpuunits = defined($conf->{cpuunits}) ?
3542             $conf->{cpuunits} : $defaults->{cpuunits};
3543
3544     push @$cmd, $kvm_binary;
3545
3546     push @$cmd, '-id', $vmid;
3547
3548     my $vmname = $conf->{name} || "vm$vmid";
3549
3550     push @$cmd, '-name', $vmname;
3551
3552     my $use_virtio = 0;
3553
3554     my $qmpsocket = qmp_socket($vmid);
3555     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3556     push @$cmd, '-mon', "chardev=qmp,mode=control";
3557
3558     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3559         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3560         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3561     }
3562
3563     push @$cmd, '-pidfile' , pidfile_name($vmid);
3564
3565     push @$cmd, '-daemonize';
3566
3567     if ($conf->{smbios1}) {
3568         my $smbios_conf = parse_smbios1($conf->{smbios1});
3569         if ($smbios_conf->{base64}) {
3570             # Do not pass base64 flag to qemu
3571             delete $smbios_conf->{base64};
3572             my $smbios_string = "";
3573             foreach my $key (keys %$smbios_conf) {
3574                 my $value;
3575                 if ($key eq "uuid") {
3576                     $value = $smbios_conf->{uuid}
3577                 } else {
3578                     $value = decode_base64($smbios_conf->{$key});
3579                 }
3580                 # qemu accepts any binary data, only commas need escaping by double comma
3581                 $value =~ s/,/,,/g;
3582                 $smbios_string .= "," . $key . "=" . $value if $value;
3583             }
3584             push @$cmd, '-smbios', "type=1" . $smbios_string;
3585         } else {
3586             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3587         }
3588     }
3589
3590     if ($conf->{vmgenid}) {
3591         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3592     }
3593
3594     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3595     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3596         die "uefi base image not found\n" if ! -f $ovmf_code;
3597
3598         my $path;
3599         my $format;
3600         if (my $efidisk = $conf->{efidisk0}) {
3601             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3602             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3603             $format = $d->{format};
3604             if ($storeid) {
3605                 $path = PVE::Storage::path($storecfg, $d->{file});
3606                 if (!defined($format)) {
3607                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3608                     $format = qemu_img_format($scfg, $volname);
3609                 }
3610             } else {
3611                 $path = $d->{file};
3612                 die "efidisk format must be specified\n"
3613                     if !defined($format);
3614             }
3615         } else {
3616             warn "no efidisk configured! Using temporary efivars disk.\n";
3617             $path = "/tmp/$vmid-ovmf.fd";
3618             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3619             $format = 'raw';
3620         }
3621
3622         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3623         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3624     }
3625
3626     # load q35 config
3627     if ($q35) {
3628         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3629         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3630             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3631         } else {
3632             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3633         }
3634     }
3635
3636     # add usb controllers
3637     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3638     push @$devices, @usbcontrollers if @usbcontrollers;
3639     my $vga = parse_vga($conf->{vga});
3640
3641     my $qxlnum = vga_conf_has_spice($conf->{vga});
3642     $vga->{type} = 'qxl' if $qxlnum;
3643
3644     if (!$vga->{type}) {
3645         if ($arch eq 'aarch64') {
3646             $vga->{type} = 'virtio';
3647         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3648             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3649         } else {
3650             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3651         }
3652     }
3653
3654     # enable absolute mouse coordinates (needed by vnc)
3655     my $tablet;
3656     if (defined($conf->{tablet})) {
3657         $tablet = $conf->{tablet};
3658     } else {
3659         $tablet = $defaults->{tablet};
3660         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3661         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3662     }
3663
3664     if ($tablet) {
3665         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3666         my $kbd = print_keyboarddevice_full($conf, $arch);
3667         push @$devices, '-device', $kbd if defined($kbd);
3668     }
3669
3670     my $kvm_off = 0;
3671     my $gpu_passthrough;
3672
3673     # host pci devices
3674     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3675         my $id = "hostpci$i";
3676         my $d = parse_hostpci($conf->{$id});
3677         next if !$d;
3678
3679         if (my $pcie = $d->{pcie}) {
3680             die "q35 machine model is not enabled" if !$q35;
3681             # win7 wants to have the pcie devices directly on the pcie bus
3682             # instead of in the root port
3683             if ($winversion == 7) {
3684                 $pciaddr = print_pcie_addr("${id}bus0");
3685             } else {
3686                 # add more root ports if needed, 4 are present by default
3687                 # by pve-q35 cfgs, rest added here on demand.
3688                 if ($i > 3) {
3689                     push @$devices, '-device', print_pcie_root_port($i);
3690                 }
3691                 $pciaddr = print_pcie_addr($id);
3692             }
3693         } else {
3694             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3695         }
3696
3697         my $xvga = '';
3698         if ($d->{'x-vga'}) {
3699             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3700             $kvm_off = 1;
3701             $vga->{type} = 'none' if !defined($conf->{vga});
3702             $gpu_passthrough = 1;
3703         }
3704
3705         my $pcidevices = $d->{pciid};
3706         my $multifunction = 1 if @$pcidevices > 1;
3707
3708         my $sysfspath;
3709         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3710             my $pci_id = $pcidevices->[0]->{id};
3711             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3712             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3713         } elsif ($d->{mdev}) {
3714             warn "ignoring mediated device '$id' with multifunction device\n";
3715         }
3716
3717         my $j=0;
3718         foreach my $pcidevice (@$pcidevices) {
3719             my $devicestr = "vfio-pci";
3720
3721             if ($sysfspath) {
3722                 $devicestr .= ",sysfsdev=$sysfspath";
3723             } else {
3724                 $devicestr .= ",host=$pcidevice->{id}";
3725             }
3726
3727             my $mf_addr = $multifunction ? ".$j" : '';
3728             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3729
3730             if ($j == 0) {
3731                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3732                 $devicestr .= "$xvga";
3733                 $devicestr .= ",multifunction=on" if $multifunction;
3734                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3735             }
3736
3737             push @$devices, '-device', $devicestr;
3738             $j++;
3739         }
3740     }
3741
3742     # usb devices
3743     my $usb_dev_features = {};
3744     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3745
3746     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3747     push @$devices, @usbdevices if @usbdevices;
3748     # serial devices
3749     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3750         if (my $path = $conf->{"serial$i"}) {
3751             if ($path eq 'socket') {
3752                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3753                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3754                 # On aarch64, serial0 is the UART device. Qemu only allows
3755                 # connecting UART devices via the '-serial' command line, as
3756                 # the device has a fixed slot on the hardware...
3757                 if ($arch eq 'aarch64' && $i == 0) {
3758                     push @$devices, '-serial', "chardev:serial$i";
3759                 } else {
3760                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3761                 }
3762             } else {
3763                 die "no such serial device\n" if ! -c $path;
3764                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3765                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3766             }
3767         }
3768     }
3769
3770     # parallel devices
3771     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3772         if (my $path = $conf->{"parallel$i"}) {
3773             die "no such parallel device\n" if ! -c $path;
3774             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3775             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3776             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3777         }
3778     }
3779
3780     if (my $audio = conf_has_audio($conf)) {
3781
3782         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3783
3784         my $id = $audio->{dev_id};
3785         if ($audio->{dev} eq 'AC97') {
3786             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3787         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3788             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3789             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3790             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3791         } else {
3792             die "unkown audio device '$audio->{dev}', implement me!";
3793         }
3794
3795         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3796     }
3797
3798     my $sockets = 1;
3799     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3800     $sockets = $conf->{sockets} if  $conf->{sockets};
3801
3802     my $cores = $conf->{cores} || 1;
3803
3804     my $maxcpus = $sockets * $cores;
3805
3806     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3807
3808     my $allowed_vcpus = $cpuinfo->{cpus};
3809
3810     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3811         if ($allowed_vcpus < $maxcpus);
3812
3813     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3814
3815         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3816         for (my $i = 2; $i <= $vcpus; $i++)  {
3817             my $cpustr = print_cpu_device($conf,$i);
3818             push @$cmd, '-device', $cpustr;
3819         }
3820
3821     } else {
3822
3823         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3824     }
3825     push @$cmd, '-nodefaults';
3826
3827     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3828
3829     my $bootindex_hash = {};
3830     my $i = 1;
3831     foreach my $o (split(//, $bootorder)) {
3832         $bootindex_hash->{$o} = $i*100;
3833         $i++;
3834     }
3835
3836     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3837
3838     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3839
3840     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3841
3842     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3843         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3844         my $socket = vnc_socket($vmid);
3845         push @$cmd,  '-vnc', "unix:$socket,password";
3846     } else {
3847         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3848         push @$cmd, '-nographic';
3849     }
3850
3851     # time drift fix
3852     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3853
3854     my $useLocaltime = $conf->{localtime};
3855
3856     if ($winversion >= 5) { # windows
3857         $useLocaltime = 1 if !defined($conf->{localtime});
3858
3859         # use time drift fix when acpi is enabled
3860         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3861             $tdf = 1 if !defined($conf->{tdf});
3862         }
3863     }
3864
3865     if ($winversion >= 6) {
3866         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3867         push @$cmd, '-no-hpet';
3868     }
3869
3870     push @$rtcFlags, 'driftfix=slew' if $tdf;
3871
3872     if (!$kvm) {
3873         push @$machineFlags, 'accel=tcg';
3874     }
3875
3876     if ($machine_type) {
3877         push @$machineFlags, "type=${machine_type}";
3878     }
3879
3880     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3881         push @$rtcFlags, "base=$conf->{startdate}";
3882     } elsif ($useLocaltime) {
3883         push @$rtcFlags, 'base=localtime';
3884     }
3885
3886     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3887
3888     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3889
3890     push @$cmd, '-S' if $conf->{freeze};
3891
3892     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3893
3894     if (parse_guest_agent($conf)->{enabled}) {
3895         my $qgasocket = qmp_socket($vmid, 1);
3896         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3897         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3898         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3899         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3900     }
3901
3902     my $spice_port;
3903
3904     if ($qxlnum) {
3905         if ($qxlnum > 1) {
3906             if ($winversion){
3907                 for(my $i = 1; $i < $qxlnum; $i++){
3908                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3909                 }
3910             } else {
3911                 # assume other OS works like Linux
3912                 my ($ram, $vram) = ("134217728", "67108864");
3913                 if ($vga->{memory}) {
3914                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3915                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3916                 }
3917                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3918                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3919             }
3920         }
3921
3922         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3923
3924         my $nodename = PVE::INotify::nodename();
3925         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3926         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3927         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3928
3929         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3930         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3931         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3932
3933         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3934         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3935
3936         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3937         if ($spice_enhancement->{foldersharing}) {
3938             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3939             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3940         }
3941
3942         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3943         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3944         push @$devices, '-spice', "$spice_opts";
3945     }
3946
3947     # enable balloon by default, unless explicitly disabled
3948     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3949         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3950         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3951     }
3952
3953     if ($conf->{watchdog}) {
3954         my $wdopts = parse_watchdog($conf->{watchdog});
3955         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3956         my $watchdog = $wdopts->{model} || 'i6300esb';
3957         push @$devices, '-device', "$watchdog$pciaddr";
3958         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3959     }
3960
3961     my $vollist = [];
3962     my $scsicontroller = {};
3963     my $ahcicontroller = {};
3964     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3965
3966     # Add iscsi initiator name if available
3967     if (my $initiator = get_initiator_name()) {
3968         push @$devices, '-iscsi', "initiator-name=$initiator";
3969     }
3970
3971     foreach_drive($conf, sub {
3972         my ($ds, $drive) = @_;
3973
3974         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3975             push @$vollist, $drive->{file};
3976         }
3977
3978         # ignore efidisk here, already added in bios/fw handling code above
3979         return if $drive->{interface} eq 'efidisk';
3980
3981         $use_virtio = 1 if $ds =~ m/^virtio/;
3982
3983         if (drive_is_cdrom ($drive)) {
3984             if ($bootindex_hash->{d}) {
3985                 $drive->{bootindex} = $bootindex_hash->{d};
3986                 $bootindex_hash->{d} += 1;
3987             }
3988         } else {
3989             if ($bootindex_hash->{c}) {
3990                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3991                 $bootindex_hash->{c} += 1;
3992             }
3993         }
3994
3995         if($drive->{interface} eq 'virtio'){
3996            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3997         }
3998
3999         if ($drive->{interface} eq 'scsi') {
4000
4001             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4002
4003             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4004             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4005
4006             my $iothread = '';
4007             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4008                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4009                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4010             } elsif ($drive->{iothread}) {
4011                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4012             }
4013
4014             my $queues = '';
4015             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4016                 $queues = ",num_queues=$drive->{queues}";
4017             }
4018
4019             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4020             $scsicontroller->{$controller}=1;
4021         }
4022
4023         if ($drive->{interface} eq 'sata') {
4024            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4025            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4026            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4027            $ahcicontroller->{$controller}=1;
4028         }
4029
4030         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4031         push @$devices, '-drive',$drive_cmd;
4032         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4033     });
4034
4035     for (my $i = 0; $i < $MAX_NETS; $i++) {
4036          next if !$conf->{"net$i"};
4037          my $d = parse_net($conf->{"net$i"});
4038          next if !$d;
4039
4040          $use_virtio = 1 if $d->{model} eq 'virtio';
4041
4042          if ($bootindex_hash->{n}) {
4043             $d->{bootindex} = $bootindex_hash->{n};
4044             $bootindex_hash->{n} += 1;
4045          }
4046
4047          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4048          push @$devices, '-netdev', $netdevfull;
4049
4050          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4051          push @$devices, '-device', $netdevicefull;
4052     }
4053
4054     if ($conf->{ivshmem}) {
4055         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4056
4057         my $bus;
4058         if ($q35) {
4059             $bus = print_pcie_addr("ivshmem");
4060         } else {
4061             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4062         }
4063
4064         my $ivshmem_name = $ivshmem->{name} // $vmid;
4065         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4066
4067         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4068         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4069     }
4070
4071     if (!$q35) {
4072         # add pci bridges
4073         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4074            $bridges->{1} = 1;
4075            $bridges->{2} = 1;
4076         }
4077
4078         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4079
4080         for my $k (sort {$b cmp $a} keys %$bridges) {
4081             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4082             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4083         }
4084     }
4085
4086     push @$cmd, @$devices;
4087     push @$cmd, '-rtc', join(',', @$rtcFlags)
4088         if scalar(@$rtcFlags);
4089     push @$cmd, '-machine', join(',', @$machineFlags)
4090         if scalar(@$machineFlags);
4091     push @$cmd, '-global', join(',', @$globalFlags)
4092         if scalar(@$globalFlags);
4093
4094     if (my $vmstate = $conf->{vmstate}) {
4095         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4096         push @$vollist, $vmstate;
4097         push @$cmd, '-loadstate', $statepath;
4098     }
4099
4100     # add custom args
4101     if ($conf->{args}) {
4102         my $aa = PVE::Tools::split_args($conf->{args});
4103         push @$cmd, @$aa;
4104     }
4105
4106     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4107 }
4108
4109 sub vnc_socket {
4110     my ($vmid) = @_;
4111     return "${var_run_tmpdir}/$vmid.vnc";
4112 }
4113
4114 sub spice_port {
4115     my ($vmid) = @_;
4116
4117     my $res = vm_mon_cmd($vmid, 'query-spice');
4118
4119     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4120 }
4121
4122 sub qmp_socket {
4123     my ($vmid, $qga) = @_;
4124     my $sockettype = $qga ? 'qga' : 'qmp';
4125     return "${var_run_tmpdir}/$vmid.$sockettype";
4126 }
4127
4128 sub pidfile_name {
4129     my ($vmid) = @_;
4130     return "${var_run_tmpdir}/$vmid.pid";
4131 }
4132
4133 sub vm_devices_list {
4134     my ($vmid) = @_;
4135
4136     my $res = vm_mon_cmd($vmid, 'query-pci');
4137     my $devices_to_check = [];
4138     my $devices = {};
4139     foreach my $pcibus (@$res) {
4140         push @$devices_to_check, @{$pcibus->{devices}},
4141     }
4142
4143     while (@$devices_to_check) {
4144         my $to_check = [];
4145         for my $d (@$devices_to_check) {
4146             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4147             next if !$d->{'pci_bridge'};
4148
4149             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4150             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4151         }
4152         $devices_to_check = $to_check;
4153     }
4154
4155     my $resblock = vm_mon_cmd($vmid, 'query-block');
4156     foreach my $block (@$resblock) {
4157         if($block->{device} =~ m/^drive-(\S+)/){
4158                 $devices->{$1} = 1;
4159         }
4160     }
4161
4162     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4163     foreach my $mice (@$resmice) {
4164         if ($mice->{name} eq 'QEMU HID Tablet') {
4165             $devices->{tablet} = 1;
4166             last;
4167         }
4168     }
4169
4170     # for usb devices there is no query-usb
4171     # but we can iterate over the entries in
4172     # qom-list path=/machine/peripheral
4173     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4174     foreach my $per (@$resperipheral) {
4175         if ($per->{name} =~ m/^usb\d+$/) {
4176             $devices->{$per->{name}} = 1;
4177         }
4178     }
4179
4180     return $devices;
4181 }
4182
4183 sub vm_deviceplug {
4184     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4185
4186     my $q35 = machine_type_is_q35($conf);
4187
4188     my $devices_list = vm_devices_list($vmid);
4189     return 1 if defined($devices_list->{$deviceid});
4190
4191     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4192
4193     if ($deviceid eq 'tablet') {
4194
4195         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4196
4197     } elsif ($deviceid eq 'keyboard') {
4198
4199         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4200
4201     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4202
4203         die "usb hotplug currently not reliable\n";
4204         # since we can't reliably hot unplug all added usb devices
4205         # and usb passthrough disables live migration
4206         # we disable usb hotplugging for now
4207         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4208
4209     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4210
4211         qemu_iothread_add($vmid, $deviceid, $device);
4212
4213         qemu_driveadd($storecfg, $vmid, $device);
4214         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4215
4216         qemu_deviceadd($vmid, $devicefull);
4217         eval { qemu_deviceaddverify($vmid, $deviceid); };
4218         if (my $err = $@) {
4219             eval { qemu_drivedel($vmid, $deviceid); };
4220             warn $@ if $@;
4221             die $err;
4222         }
4223
4224     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4225
4226
4227         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4228         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4229         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4230
4231         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4232
4233         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4234             qemu_iothread_add($vmid, $deviceid, $device);
4235             $devicefull .= ",iothread=iothread-$deviceid";
4236         }
4237
4238         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4239             $devicefull .= ",num_queues=$device->{queues}";
4240         }
4241
4242         qemu_deviceadd($vmid, $devicefull);
4243         qemu_deviceaddverify($vmid, $deviceid);
4244
4245     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4246
4247         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4248         qemu_driveadd($storecfg, $vmid, $device);
4249
4250         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4251         eval { qemu_deviceadd($vmid, $devicefull); };
4252         if (my $err = $@) {
4253             eval { qemu_drivedel($vmid, $deviceid); };
4254             warn $@ if $@;
4255             die $err;
4256         }
4257
4258     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4259
4260         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4261
4262         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4263         my $use_old_bios_files = undef;
4264         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4265
4266         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4267         qemu_deviceadd($vmid, $netdevicefull);
4268         eval {
4269             qemu_deviceaddverify($vmid, $deviceid);
4270             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4271         };
4272         if (my $err = $@) {
4273             eval { qemu_netdevdel($vmid, $deviceid); };
4274             warn $@ if $@;
4275             die $err;
4276         }
4277
4278     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4279
4280         my $bridgeid = $2;
4281         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4282         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4283
4284         qemu_deviceadd($vmid, $devicefull);
4285         qemu_deviceaddverify($vmid, $deviceid);
4286
4287     } else {
4288         die "can't hotplug device '$deviceid'\n";
4289     }
4290
4291     return 1;
4292 }
4293
4294 # fixme: this should raise exceptions on error!
4295 sub vm_deviceunplug {
4296     my ($vmid, $conf, $deviceid) = @_;
4297
4298     my $devices_list = vm_devices_list($vmid);
4299     return 1 if !defined($devices_list->{$deviceid});
4300
4301     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4302
4303     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4304
4305         qemu_devicedel($vmid, $deviceid);
4306
4307     } elsif ($deviceid =~ m/^usb\d+$/) {
4308
4309         die "usb hotplug currently not reliable\n";
4310         # when unplugging usb devices this way,
4311         # there may be remaining usb controllers/hubs
4312         # so we disable it for now
4313         qemu_devicedel($vmid, $deviceid);
4314         qemu_devicedelverify($vmid, $deviceid);
4315
4316     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4317
4318         qemu_devicedel($vmid, $deviceid);
4319         qemu_devicedelverify($vmid, $deviceid);
4320         qemu_drivedel($vmid, $deviceid);
4321         qemu_iothread_del($conf, $vmid, $deviceid);
4322
4323     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4324
4325         qemu_devicedel($vmid, $deviceid);
4326         qemu_devicedelverify($vmid, $deviceid);
4327         qemu_iothread_del($conf, $vmid, $deviceid);
4328
4329     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4330
4331         qemu_devicedel($vmid, $deviceid);
4332         qemu_drivedel($vmid, $deviceid);
4333         qemu_deletescsihw($conf, $vmid, $deviceid);
4334
4335     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4336
4337         qemu_devicedel($vmid, $deviceid);
4338         qemu_devicedelverify($vmid, $deviceid);
4339         qemu_netdevdel($vmid, $deviceid);
4340
4341     } else {
4342         die "can't unplug device '$deviceid'\n";
4343     }
4344
4345     return 1;
4346 }
4347
4348 sub qemu_deviceadd {
4349     my ($vmid, $devicefull) = @_;
4350
4351     $devicefull = "driver=".$devicefull;
4352     my %options =  split(/[=,]/, $devicefull);
4353
4354     vm_mon_cmd($vmid, "device_add" , %options);
4355 }
4356
4357 sub qemu_devicedel {
4358     my ($vmid, $deviceid) = @_;
4359
4360     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4361 }
4362
4363 sub qemu_iothread_add {
4364     my($vmid, $deviceid, $device) = @_;
4365
4366     if ($device->{iothread}) {
4367         my $iothreads = vm_iothreads_list($vmid);
4368         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4369     }
4370 }
4371
4372 sub qemu_iothread_del {
4373     my($conf, $vmid, $deviceid) = @_;
4374
4375     my $confid = $deviceid;
4376     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4377         $confid = 'scsi' . $1;
4378     }
4379     my $device = parse_drive($confid, $conf->{$confid});
4380     if ($device->{iothread}) {
4381         my $iothreads = vm_iothreads_list($vmid);
4382         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4383     }
4384 }
4385
4386 sub qemu_objectadd {
4387     my($vmid, $objectid, $qomtype) = @_;
4388
4389     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4390
4391     return 1;
4392 }
4393
4394 sub qemu_objectdel {
4395     my($vmid, $objectid) = @_;
4396
4397     vm_mon_cmd($vmid, "object-del", id => $objectid);
4398
4399     return 1;
4400 }
4401
4402 sub qemu_driveadd {
4403     my ($storecfg, $vmid, $device) = @_;
4404
4405     my $drive = print_drive_full($storecfg, $vmid, $device);
4406     $drive =~ s/\\/\\\\/g;
4407     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4408
4409     # If the command succeeds qemu prints: "OK"
4410     return 1 if $ret =~ m/OK/s;
4411
4412     die "adding drive failed: $ret\n";
4413 }
4414
4415 sub qemu_drivedel {
4416     my($vmid, $deviceid) = @_;
4417
4418     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4419     $ret =~ s/^\s+//;
4420
4421     return 1 if $ret eq "";
4422
4423     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4424     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4425
4426     die "deleting drive $deviceid failed : $ret\n";
4427 }
4428
4429 sub qemu_deviceaddverify {
4430     my ($vmid, $deviceid) = @_;
4431
4432     for (my $i = 0; $i <= 5; $i++) {
4433          my $devices_list = vm_devices_list($vmid);
4434          return 1 if defined($devices_list->{$deviceid});
4435          sleep 1;
4436     }
4437
4438     die "error on hotplug device '$deviceid'\n";
4439 }
4440
4441
4442 sub qemu_devicedelverify {
4443     my ($vmid, $deviceid) = @_;
4444
4445     # need to verify that the device is correctly removed as device_del
4446     # is async and empty return is not reliable
4447
4448     for (my $i = 0; $i <= 5; $i++) {
4449          my $devices_list = vm_devices_list($vmid);
4450          return 1 if !defined($devices_list->{$deviceid});
4451          sleep 1;
4452     }
4453
4454     die "error on hot-unplugging device '$deviceid'\n";
4455 }
4456
4457 sub qemu_findorcreatescsihw {
4458     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4459
4460     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4461
4462     my $scsihwid="$controller_prefix$controller";
4463     my $devices_list = vm_devices_list($vmid);
4464
4465     if(!defined($devices_list->{$scsihwid})) {
4466         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4467     }
4468
4469     return 1;
4470 }
4471
4472 sub qemu_deletescsihw {
4473     my ($conf, $vmid, $opt) = @_;
4474
4475     my $device = parse_drive($opt, $conf->{$opt});
4476
4477     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4478         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4479         return 1;
4480     }
4481
4482     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4483
4484     my $devices_list = vm_devices_list($vmid);
4485     foreach my $opt (keys %{$devices_list}) {
4486         if (PVE::QemuServer::is_valid_drivename($opt)) {
4487             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4488             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4489                 return 1;
4490             }
4491         }
4492     }
4493
4494     my $scsihwid="scsihw$controller";
4495
4496     vm_deviceunplug($vmid, $conf, $scsihwid);
4497
4498     return 1;
4499 }
4500
4501 sub qemu_add_pci_bridge {
4502     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4503
4504     my $bridges = {};
4505
4506     my $bridgeid;
4507
4508     print_pci_addr($device, $bridges, $arch, $machine_type);
4509
4510     while (my ($k, $v) = each %$bridges) {
4511         $bridgeid = $k;
4512     }
4513     return 1 if !defined($bridgeid) || $bridgeid < 1;
4514
4515     my $bridge = "pci.$bridgeid";
4516     my $devices_list = vm_devices_list($vmid);
4517
4518     if (!defined($devices_list->{$bridge})) {
4519         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4520     }
4521
4522     return 1;
4523 }
4524
4525 sub qemu_set_link_status {
4526     my ($vmid, $device, $up) = @_;
4527
4528     vm_mon_cmd($vmid, "set_link", name => $device,
4529                up => $up ? JSON::true : JSON::false);
4530 }
4531
4532 sub qemu_netdevadd {
4533     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4534
4535     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4536     my %options =  split(/[=,]/, $netdev);
4537
4538     vm_mon_cmd($vmid, "netdev_add",  %options);
4539     return 1;
4540 }
4541
4542 sub qemu_netdevdel {
4543     my ($vmid, $deviceid) = @_;
4544
4545     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4546 }
4547
4548 sub qemu_usb_hotplug {
4549     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4550
4551     return if !$device;
4552
4553     # remove the old one first
4554     vm_deviceunplug($vmid, $conf, $deviceid);
4555
4556     # check if xhci controller is necessary and available
4557     if ($device->{usb3}) {
4558
4559         my $devicelist = vm_devices_list($vmid);
4560
4561         if (!$devicelist->{xhci}) {
4562             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4563             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4564         }
4565     }
4566     my $d = parse_usb_device($device->{host});
4567     $d->{usb3} = $device->{usb3};
4568
4569     # add the new one
4570     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4571 }
4572
4573 sub qemu_cpu_hotplug {
4574     my ($vmid, $conf, $vcpus) = @_;
4575
4576     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4577
4578     my $sockets = 1;
4579     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4580     $sockets = $conf->{sockets} if  $conf->{sockets};
4581     my $cores = $conf->{cores} || 1;
4582     my $maxcpus = $sockets * $cores;
4583
4584     $vcpus = $maxcpus if !$vcpus;
4585
4586     die "you can't add more vcpus than maxcpus\n"
4587         if $vcpus > $maxcpus;
4588
4589     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4590
4591     if ($vcpus < $currentvcpus) {
4592
4593         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4594
4595             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4596                 qemu_devicedel($vmid, "cpu$i");
4597                 my $retry = 0;
4598                 my $currentrunningvcpus = undef;
4599                 while (1) {
4600                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4601                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4602                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4603                     $retry++;
4604                     sleep 1;
4605                 }
4606                 #update conf after each succesfull cpu unplug
4607                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4608                 PVE::QemuConfig->write_config($vmid, $conf);
4609             }
4610         } else {
4611             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4612         }
4613
4614         return;
4615     }
4616
4617     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4618     die "vcpus in running vm does not match its configuration\n"
4619         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4620
4621     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4622
4623         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4624             my $cpustr = print_cpu_device($conf, $i);
4625             qemu_deviceadd($vmid, $cpustr);
4626
4627             my $retry = 0;
4628             my $currentrunningvcpus = undef;
4629             while (1) {
4630                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4631                 last if scalar(@{$currentrunningvcpus}) == $i;
4632                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4633                 sleep 1;
4634                 $retry++;
4635             }
4636             #update conf after each succesfull cpu hotplug
4637             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4638             PVE::QemuConfig->write_config($vmid, $conf);
4639         }
4640     } else {
4641
4642         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4643             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4644         }
4645     }
4646 }
4647
4648 sub qemu_block_set_io_throttle {
4649     my ($vmid, $deviceid,
4650         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4651         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4652         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4653         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4654
4655     return if !check_running($vmid) ;
4656
4657     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4658         bps => int($bps),
4659         bps_rd => int($bps_rd),
4660         bps_wr => int($bps_wr),
4661         iops => int($iops),
4662         iops_rd => int($iops_rd),
4663         iops_wr => int($iops_wr),
4664         bps_max => int($bps_max),
4665         bps_rd_max => int($bps_rd_max),
4666         bps_wr_max => int($bps_wr_max),
4667         iops_max => int($iops_max),
4668         iops_rd_max => int($iops_rd_max),
4669         iops_wr_max => int($iops_wr_max),
4670         bps_max_length => int($bps_max_length),
4671         bps_rd_max_length => int($bps_rd_max_length),
4672         bps_wr_max_length => int($bps_wr_max_length),
4673         iops_max_length => int($iops_max_length),
4674         iops_rd_max_length => int($iops_rd_max_length),
4675         iops_wr_max_length => int($iops_wr_max_length),
4676     );
4677
4678 }
4679
4680 # old code, only used to shutdown old VM after update
4681 sub __read_avail {
4682     my ($fh, $timeout) = @_;
4683
4684     my $sel = new IO::Select;
4685     $sel->add($fh);
4686
4687     my $res = '';
4688     my $buf;
4689
4690     my @ready;
4691     while (scalar (@ready = $sel->can_read($timeout))) {
4692         my $count;
4693         if ($count = $fh->sysread($buf, 8192)) {
4694             if ($buf =~ /^(.*)\(qemu\) $/s) {
4695                 $res .= $1;
4696                 last;
4697             } else {
4698                 $res .= $buf;
4699             }
4700         } else {
4701             if (!defined($count)) {
4702                 die "$!\n";
4703             }
4704             last;
4705         }
4706     }
4707
4708     die "monitor read timeout\n" if !scalar(@ready);
4709
4710     return $res;
4711 }
4712
4713 sub qemu_block_resize {
4714     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4715
4716     my $running = check_running($vmid);
4717
4718     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4719
4720     return if !$running;
4721
4722     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4723
4724 }
4725
4726 sub qemu_volume_snapshot {
4727     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4728
4729     my $running = check_running($vmid);
4730
4731     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4732         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4733     } else {
4734         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4735     }
4736 }
4737
4738 sub qemu_volume_snapshot_delete {
4739     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4740
4741     my $running = check_running($vmid);
4742
4743     if($running) {
4744
4745         $running = undef;
4746         my $conf = PVE::QemuConfig->load_config($vmid);
4747         foreach_drive($conf, sub {
4748             my ($ds, $drive) = @_;
4749             $running = 1 if $drive->{file} eq $volid;
4750         });
4751     }
4752
4753     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4754         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4755     } else {
4756         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4757     }
4758 }
4759
4760 sub set_migration_caps {
4761     my ($vmid) = @_;
4762
4763     my $cap_ref = [];
4764
4765     my $enabled_cap = {
4766         "auto-converge" => 1,
4767         "xbzrle" => 1,
4768         "x-rdma-pin-all" => 0,
4769         "zero-blocks" => 0,
4770         "compress" => 0
4771     };
4772
4773     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4774
4775     for my $supported_capability (@$supported_capabilities) {
4776         push @$cap_ref, {
4777             capability => $supported_capability->{capability},
4778             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4779         };
4780     }
4781
4782     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4783 }
4784
4785 my $fast_plug_option = {
4786     'lock' => 1,
4787     'name' => 1,
4788     'onboot' => 1,
4789     'shares' => 1,
4790     'startup' => 1,
4791     'description' => 1,
4792     'protection' => 1,
4793     'vmstatestorage' => 1,
4794     'hookscript' => 1,
4795 };
4796
4797 # hotplug changes in [PENDING]
4798 # $selection hash can be used to only apply specified options, for
4799 # example: { cores => 1 } (only apply changed 'cores')
4800 # $errors ref is used to return error messages
4801 sub vmconfig_hotplug_pending {
4802     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4803
4804     my $defaults = load_defaults();
4805     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4806
4807     # commit values which do not have any impact on running VM first
4808     # Note: those option cannot raise errors, we we do not care about
4809     # $selection and always apply them.
4810
4811     my $add_error = sub {
4812         my ($opt, $msg) = @_;
4813         $errors->{$opt} = "hotplug problem - $msg";
4814     };
4815
4816     my $changes = 0;
4817     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4818         if ($fast_plug_option->{$opt}) {
4819             $conf->{$opt} = $conf->{pending}->{$opt};
4820             delete $conf->{pending}->{$opt};
4821             $changes = 1;
4822         }
4823     }
4824
4825     if ($changes) {
4826         PVE::QemuConfig->write_config($vmid, $conf);
4827         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4828     }
4829
4830     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4831
4832     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4833     foreach my $opt (sort keys %$pending_delete_hash) {
4834         next if $selection && !$selection->{$opt};
4835         my $force = $pending_delete_hash->{$opt}->{force};
4836         eval {
4837             if ($opt eq 'hotplug') {
4838                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4839             } elsif ($opt eq 'tablet') {
4840                 die "skip\n" if !$hotplug_features->{usb};
4841                 if ($defaults->{tablet}) {
4842                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4843                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4844                         if $arch eq 'aarch64';
4845                 } else {
4846                     vm_deviceunplug($vmid, $conf, 'tablet');
4847                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4848                 }
4849             } elsif ($opt =~ m/^usb\d+/) {
4850                 die "skip\n";
4851                 # since we cannot reliably hot unplug usb devices
4852                 # we are disabling it
4853                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4854                 vm_deviceunplug($vmid, $conf, $opt);
4855             } elsif ($opt eq 'vcpus') {
4856                 die "skip\n" if !$hotplug_features->{cpu};
4857                 qemu_cpu_hotplug($vmid, $conf, undef);
4858             } elsif ($opt eq 'balloon') {
4859                 # enable balloon device is not hotpluggable
4860                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4861                 # here we reset the ballooning value to memory
4862                 my $balloon = $conf->{memory} || $defaults->{memory};
4863                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4864             } elsif ($fast_plug_option->{$opt}) {
4865                 # do nothing
4866             } elsif ($opt =~ m/^net(\d+)$/) {
4867                 die "skip\n" if !$hotplug_features->{network};
4868                 vm_deviceunplug($vmid, $conf, $opt);
4869             } elsif (is_valid_drivename($opt)) {
4870                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4871                 vm_deviceunplug($vmid, $conf, $opt);
4872                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4873             } elsif ($opt =~ m/^memory$/) {
4874                 die "skip\n" if !$hotplug_features->{memory};
4875                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4876             } elsif ($opt eq 'cpuunits') {
4877                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4878             } elsif ($opt eq 'cpulimit') {
4879                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4880             } else {
4881                 die "skip\n";
4882             }
4883         };
4884         if (my $err = $@) {
4885             &$add_error($opt, $err) if $err ne "skip\n";
4886         } else {
4887             # save new config if hotplug was successful
4888             delete $conf->{$opt};
4889             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4890             PVE::QemuConfig->write_config($vmid, $conf);
4891             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4892         }
4893     }
4894
4895     my $apply_pending_cloudinit;
4896     $apply_pending_cloudinit = sub {
4897         my ($key, $value) = @_;
4898         $apply_pending_cloudinit = sub {}; # once is enough
4899
4900         my @cloudinit_opts = keys %$confdesc_cloudinit;
4901         foreach my $opt (keys %{$conf->{pending}}) {
4902             next if !grep { $_ eq $opt } @cloudinit_opts;
4903             $conf->{$opt} = delete $conf->{pending}->{$opt};
4904         }
4905
4906         my $new_conf = { %$conf };
4907         $new_conf->{$key} = $value;
4908         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4909     };
4910
4911     foreach my $opt (keys %{$conf->{pending}}) {
4912         next if $selection && !$selection->{$opt};
4913         my $value = $conf->{pending}->{$opt};
4914         eval {
4915             if ($opt eq 'hotplug') {
4916                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4917             } elsif ($opt eq 'tablet') {
4918                 die "skip\n" if !$hotplug_features->{usb};
4919                 if ($value == 1) {
4920                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4921                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4922                         if $arch eq 'aarch64';
4923                 } elsif ($value == 0) {
4924                     vm_deviceunplug($vmid, $conf, 'tablet');
4925                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4926                 }
4927             } elsif ($opt =~ m/^usb\d+$/) {
4928                 die "skip\n";
4929                 # since we cannot reliably hot unplug usb devices
4930                 # we are disabling it
4931                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4932                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4933                 die "skip\n" if !$d;
4934                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4935             } elsif ($opt eq 'vcpus') {
4936                 die "skip\n" if !$hotplug_features->{cpu};
4937                 qemu_cpu_hotplug($vmid, $conf, $value);
4938             } elsif ($opt eq 'balloon') {
4939                 # enable/disable balloning device is not hotpluggable
4940                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4941                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4942                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4943
4944                 # allow manual ballooning if shares is set to zero
4945                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4946                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4947                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4948                 }
4949             } elsif ($opt =~ m/^net(\d+)$/) {
4950                 # some changes can be done without hotplug
4951                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4952                                     $vmid, $opt, $value, $arch, $machine_type);
4953             } elsif (is_valid_drivename($opt)) {
4954                 # some changes can be done without hotplug
4955                 my $drive = parse_drive($opt, $value);
4956                 if (drive_is_cloudinit($drive)) {
4957                     &$apply_pending_cloudinit($opt, $value);
4958                 }
4959                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4960                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4961             } elsif ($opt =~ m/^memory$/) { #dimms
4962                 die "skip\n" if !$hotplug_features->{memory};
4963                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4964             } elsif ($opt eq 'cpuunits') {
4965                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4966             } elsif ($opt eq 'cpulimit') {
4967                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4968                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4969             } else {
4970                 die "skip\n";  # skip non-hot-pluggable options
4971             }
4972         };
4973         if (my $err = $@) {
4974             &$add_error($opt, $err) if $err ne "skip\n";
4975         } else {
4976             # save new config if hotplug was successful
4977             $conf->{$opt} = $value;
4978             delete $conf->{pending}->{$opt};
4979             PVE::QemuConfig->write_config($vmid, $conf);
4980             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4981         }
4982     }
4983 }
4984
4985 sub try_deallocate_drive {
4986     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4987
4988     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4989         my $volid = $drive->{file};
4990         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4991             my $sid = PVE::Storage::parse_volume_id($volid);
4992             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4993
4994             # check if the disk is really unused
4995             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4996                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4997             PVE::Storage::vdisk_free($storecfg, $volid);
4998             return 1;
4999         } else {
5000             # If vm is not owner of this disk remove from config
5001             return 1;
5002         }
5003     }
5004
5005     return undef;
5006 }
5007
5008 sub vmconfig_delete_or_detach_drive {
5009     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5010
5011     my $drive = parse_drive($opt, $conf->{$opt});
5012
5013     my $rpcenv = PVE::RPCEnvironment::get();
5014     my $authuser = $rpcenv->get_user();
5015
5016     if ($force) {
5017         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5018         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5019     } else {
5020         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5021     }
5022 }
5023
5024
5025
5026 sub vmconfig_apply_pending {
5027     my ($vmid, $conf, $storecfg) = @_;
5028
5029     # cold plug
5030
5031     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5032     foreach my $opt (sort keys %$pending_delete_hash) {
5033         die "internal error" if $opt =~ m/^unused/;
5034         my $force = $pending_delete_hash->{$opt}->{force};
5035         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5036         if (!defined($conf->{$opt})) {
5037             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5038             PVE::QemuConfig->write_config($vmid, $conf);
5039         } elsif (is_valid_drivename($opt)) {
5040             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5041             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5042             delete $conf->{$opt};
5043             PVE::QemuConfig->write_config($vmid, $conf);
5044         } else {
5045             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5046             delete $conf->{$opt};
5047             PVE::QemuConfig->write_config($vmid, $conf);
5048         }
5049     }
5050
5051     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5052
5053     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5054         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5055
5056         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5057             # skip if nothing changed
5058         } elsif (is_valid_drivename($opt)) {
5059             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5060                 if defined($conf->{$opt});
5061             $conf->{$opt} = $conf->{pending}->{$opt};
5062         } else {
5063             $conf->{$opt} = $conf->{pending}->{$opt};
5064         }
5065
5066         delete $conf->{pending}->{$opt};
5067         PVE::QemuConfig->write_config($vmid, $conf);
5068     }
5069 }
5070
5071 my $safe_num_ne = sub {
5072     my ($a, $b) = @_;
5073
5074     return 0 if !defined($a) && !defined($b);
5075     return 1 if !defined($a);
5076     return 1 if !defined($b);
5077
5078     return $a != $b;
5079 };
5080
5081 my $safe_string_ne = sub {
5082     my ($a, $b) = @_;
5083
5084     return 0 if !defined($a) && !defined($b);
5085     return 1 if !defined($a);
5086     return 1 if !defined($b);
5087
5088     return $a ne $b;
5089 };
5090
5091 sub vmconfig_update_net {
5092     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5093
5094     my $newnet = parse_net($value);
5095
5096     if ($conf->{$opt}) {
5097         my $oldnet = parse_net($conf->{$opt});
5098
5099         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5100             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5101             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5102             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5103
5104             # for non online change, we try to hot-unplug
5105             die "skip\n" if !$hotplug;
5106             vm_deviceunplug($vmid, $conf, $opt);
5107         } else {
5108
5109             die "internal error" if $opt !~ m/net(\d+)/;
5110             my $iface = "tap${vmid}i$1";
5111
5112             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5113                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5114                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5115                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5116                 PVE::Network::tap_unplug($iface);
5117                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5118             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5119                 # Rate can be applied on its own but any change above needs to
5120                 # include the rate in tap_plug since OVS resets everything.
5121                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5122             }
5123
5124             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5125                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5126             }
5127
5128             return 1;
5129         }
5130     }
5131
5132     if ($hotplug) {
5133         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5134     } else {
5135         die "skip\n";
5136     }
5137 }
5138
5139 sub vmconfig_update_disk {
5140     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5141
5142     # fixme: do we need force?
5143
5144     my $drive = parse_drive($opt, $value);
5145
5146     if ($conf->{$opt}) {
5147
5148         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5149
5150             my $media = $drive->{media} || 'disk';
5151             my $oldmedia = $old_drive->{media} || 'disk';
5152             die "unable to change media type\n" if $media ne $oldmedia;
5153
5154             if (!drive_is_cdrom($old_drive)) {
5155
5156                 if ($drive->{file} ne $old_drive->{file}) {
5157
5158                     die "skip\n" if !$hotplug;
5159
5160                     # unplug and register as unused
5161                     vm_deviceunplug($vmid, $conf, $opt);
5162                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5163
5164                 } else {
5165                     # update existing disk
5166
5167                     # skip non hotpluggable value
5168                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5169                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5170                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5171                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5172                         die "skip\n";
5173                     }
5174
5175                     # apply throttle
5176                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5177                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5178                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5179                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5180                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5181                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5182                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5183                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5184                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5185                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5186                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5187                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5188                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5189                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5190                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5191                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5192                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5193                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5194
5195                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5196                                                    ($drive->{mbps} || 0)*1024*1024,
5197                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5198                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5199                                                    $drive->{iops} || 0,
5200                                                    $drive->{iops_rd} || 0,
5201                                                    $drive->{iops_wr} || 0,
5202                                                    ($drive->{mbps_max} || 0)*1024*1024,
5203                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5204                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5205                                                    $drive->{iops_max} || 0,
5206                                                    $drive->{iops_rd_max} || 0,
5207                                                    $drive->{iops_wr_max} || 0,
5208                                                    $drive->{bps_max_length} || 1,
5209                                                    $drive->{bps_rd_max_length} || 1,
5210                                                    $drive->{bps_wr_max_length} || 1,
5211                                                    $drive->{iops_max_length} || 1,
5212                                                    $drive->{iops_rd_max_length} || 1,
5213                                                    $drive->{iops_wr_max_length} || 1);
5214
5215                     }
5216
5217                     return 1;
5218                 }
5219
5220             } else { # cdrom
5221
5222                 if ($drive->{file} eq 'none') {
5223                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5224                     if (drive_is_cloudinit($old_drive)) {
5225                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5226                     }
5227                 } else {
5228                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5229                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5230                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5231                 }
5232
5233                 return 1;
5234             }
5235         }
5236     }
5237
5238     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5239     # hotplug new disks
5240     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5241     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5242 }
5243
5244 sub vm_start {
5245     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5246         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5247
5248     PVE::QemuConfig->lock_config($vmid, sub {
5249         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5250
5251         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5252
5253         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5254
5255         PVE::QemuConfig->check_lock($conf)
5256             if !($skiplock || $is_suspended);
5257
5258         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5259
5260         # clean up leftover reboot request files
5261         eval { clear_reboot_request($vmid); };
5262         warn $@ if $@;
5263
5264         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5265             vmconfig_apply_pending($vmid, $conf, $storecfg);
5266             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5267         }
5268
5269         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5270
5271         my $defaults = load_defaults();
5272
5273         # set environment variable useful inside network script
5274         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5275
5276         my $local_volumes = {};
5277
5278         if ($targetstorage) {
5279             foreach_drive($conf, sub {
5280                 my ($ds, $drive) = @_;
5281
5282                 return if drive_is_cdrom($drive);
5283
5284                 my $volid = $drive->{file};
5285
5286                 return if !$volid;
5287
5288                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5289
5290                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5291                 return if $scfg->{shared};
5292                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5293             });
5294
5295             my $format = undef;
5296
5297             foreach my $opt (sort keys %$local_volumes) {
5298
5299                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5300                 my $drive = parse_drive($opt, $conf->{$opt});
5301
5302                 #if remote storage is specified, use default format
5303                 if ($targetstorage && $targetstorage ne "1") {
5304                     $storeid = $targetstorage;
5305                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5306                     $format = $defFormat;
5307                 } else {
5308                     #else we use same format than original
5309                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5310                     $format = qemu_img_format($scfg, $volid);
5311                 }
5312
5313                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5314                 my $newdrive = $drive;
5315                 $newdrive->{format} = $format;
5316                 $newdrive->{file} = $newvolid;
5317                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5318                 $local_volumes->{$opt} = $drivestr;
5319                 #pass drive to conf for command line
5320                 $conf->{$opt} = $drivestr;
5321             }
5322         }
5323
5324         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5325
5326         if ($is_suspended) {
5327             # enforce machine type on suspended vm to ensure HW compatibility
5328             $forcemachine = $conf->{runningmachine};
5329             print "Resuming suspended VM\n";
5330         }
5331
5332         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5333
5334         my $migrate_uri;
5335         if ($statefile) {
5336             if ($statefile eq 'tcp') {
5337                 my $localip = "localhost";
5338                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5339                 my $nodename = PVE::INotify::nodename();
5340
5341                 if (!defined($migration_type)) {
5342                     if (defined($datacenterconf->{migration}->{type})) {
5343                         $migration_type = $datacenterconf->{migration}->{type};
5344                     } else {
5345                         $migration_type = 'secure';
5346                     }
5347                 }
5348
5349                 if ($migration_type eq 'insecure') {
5350                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5351                     if ($migrate_network_addr) {
5352                         $localip = $migrate_network_addr;
5353                     } else {
5354                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5355                     }
5356
5357                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5358                 }
5359
5360                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5361                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5362                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5363                 push @$cmd, '-incoming', $migrate_uri;
5364                 push @$cmd, '-S';
5365
5366             } elsif ($statefile eq 'unix') {
5367                 # should be default for secure migrations as a ssh TCP forward
5368                 # tunnel is not deterministic reliable ready and fails regurarly
5369                 # to set up in time, so use UNIX socket forwards
5370                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5371                 unlink $socket_addr;
5372
5373                 $migrate_uri = "unix:$socket_addr";
5374
5375                 push @$cmd, '-incoming', $migrate_uri;
5376                 push @$cmd, '-S';
5377
5378             } elsif (-e $statefile) {
5379                 push @$cmd, '-loadstate', $statefile;
5380             } else {
5381                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5382                 push @$vollist, $statefile;
5383                 push @$cmd, '-loadstate', $statepath;
5384             }
5385         } elsif ($paused) {
5386             push @$cmd, '-S';
5387         }
5388
5389         # host pci devices
5390         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5391           my $d = parse_hostpci($conf->{"hostpci$i"});
5392           next if !$d;
5393           my $pcidevices = $d->{pciid};
5394           foreach my $pcidevice (@$pcidevices) {
5395                 my $pciid = $pcidevice->{id};
5396
5397                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5398                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5399                 die "no pci device info for device '$pciid'\n" if !$info;
5400
5401                 if ($d->{mdev}) {
5402                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5403                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5404                 } else {
5405                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5406                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5407                     die "can't reset pci device '$pciid'\n"
5408                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5409                 }
5410           }
5411         }
5412
5413         PVE::Storage::activate_volumes($storecfg, $vollist);
5414
5415         eval {
5416             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5417                 outfunc => sub {}, errfunc => sub {});
5418         };
5419         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5420         # timeout should be more than enough here...
5421         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5422
5423         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5424                                                   : $defaults->{cpuunits};
5425
5426         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5427         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5428
5429         my %properties = (
5430             Slice => 'qemu.slice',
5431             KillMode => 'none',
5432             CPUShares => $cpuunits
5433         );
5434
5435         if (my $cpulimit = $conf->{cpulimit}) {
5436             $properties{CPUQuota} = int($cpulimit * 100);
5437         }
5438         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5439
5440         my $run_qemu = sub {
5441             PVE::Tools::run_fork sub {
5442                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5443                 run_command($cmd, %run_params);
5444             };
5445         };
5446
5447         if ($conf->{hugepages}) {
5448
5449             my $code = sub {
5450                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5451                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5452
5453                 PVE::QemuServer::Memory::hugepages_mount();
5454                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5455
5456                 eval { $run_qemu->() };
5457                 if (my $err = $@) {
5458                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5459                     die $err;
5460                 }
5461
5462                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5463             };
5464             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5465
5466         } else {
5467             eval { $run_qemu->() };
5468         }
5469
5470         if (my $err = $@) {
5471             # deactivate volumes if start fails
5472             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5473             die "start failed: $err";
5474         }
5475
5476         print "migration listens on $migrate_uri\n" if $migrate_uri;
5477
5478         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5479             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5480             warn $@ if $@;
5481         }
5482
5483         #start nbd server for storage migration
5484         if ($targetstorage) {
5485             my $nodename = PVE::INotify::nodename();
5486             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5487             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5488             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5489             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5490
5491             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5492
5493             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5494
5495             foreach my $opt (sort keys %$local_volumes) {
5496                 my $volid = $local_volumes->{$opt};
5497                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5498                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5499                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5500             }
5501         }
5502
5503         if ($migratedfrom) {
5504             eval {
5505                 set_migration_caps($vmid);
5506             };
5507             warn $@ if $@;
5508
5509             if ($spice_port) {
5510                 print "spice listens on port $spice_port\n";
5511                 if ($spice_ticket) {
5512                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5513                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5514                 }
5515             }
5516
5517         } else {
5518             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5519                 if !$statefile && $conf->{balloon};
5520
5521             foreach my $opt (keys %$conf) {
5522                 next if $opt !~  m/^net\d+$/;
5523                 my $nicconf = parse_net($conf->{$opt});
5524                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5525             }
5526         }
5527
5528         vm_mon_cmd_nocheck($vmid, 'qom-set',
5529                     path => "machine/peripheral/balloon0",
5530                     property => "guest-stats-polling-interval",
5531                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5532
5533         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5534             print "Resumed VM, removing state\n";
5535             delete $conf->@{qw(lock vmstate runningmachine)};
5536             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5537             PVE::Storage::vdisk_free($storecfg, $vmstate);
5538             PVE::QemuConfig->write_config($vmid, $conf);
5539         }
5540
5541         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5542     });
5543 }
5544
5545 sub vm_mon_cmd {
5546     my ($vmid, $execute, %params) = @_;
5547
5548     my $cmd = { execute => $execute, arguments => \%params };
5549     vm_qmp_command($vmid, $cmd);
5550 }
5551
5552 sub vm_mon_cmd_nocheck {
5553     my ($vmid, $execute, %params) = @_;
5554
5555     my $cmd = { execute => $execute, arguments => \%params };
5556     vm_qmp_command($vmid, $cmd, 1);
5557 }
5558
5559 sub vm_qmp_command {
5560     my ($vmid, $cmd, $nocheck) = @_;
5561
5562     my $res;
5563
5564     my $timeout;
5565     if ($cmd->{arguments}) {
5566         $timeout = delete $cmd->{arguments}->{timeout};
5567     }
5568
5569     eval {
5570         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5571         my $sname = qmp_socket($vmid);
5572         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5573             my $qmpclient = PVE::QMPClient->new();
5574
5575             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5576         } else {
5577             die "unable to open monitor socket\n";
5578         }
5579     };
5580     if (my $err = $@) {
5581         syslog("err", "VM $vmid qmp command failed - $err");
5582         die $err;
5583     }
5584
5585     return $res;
5586 }
5587
5588 sub vm_human_monitor_command {
5589     my ($vmid, $cmdline) = @_;
5590
5591     my $cmd = {
5592         execute => 'human-monitor-command',
5593         arguments => { 'command-line' => $cmdline},
5594     };
5595
5596     return vm_qmp_command($vmid, $cmd);
5597 }
5598
5599 sub vm_commandline {
5600     my ($storecfg, $vmid, $snapname) = @_;
5601
5602     my $conf = PVE::QemuConfig->load_config($vmid);
5603
5604     if ($snapname) {
5605         my $snapshot = $conf->{snapshots}->{$snapname};
5606         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5607
5608         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5609
5610         $conf = $snapshot;
5611     }
5612
5613     my $defaults = load_defaults();
5614
5615     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5616
5617     return PVE::Tools::cmd2string($cmd);
5618 }
5619
5620 sub vm_reset {
5621     my ($vmid, $skiplock) = @_;
5622
5623     PVE::QemuConfig->lock_config($vmid, sub {
5624
5625         my $conf = PVE::QemuConfig->load_config($vmid);
5626
5627         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5628
5629         vm_mon_cmd($vmid, "system_reset");
5630     });
5631 }
5632
5633 sub get_vm_volumes {
5634     my ($conf) = @_;
5635
5636     my $vollist = [];
5637     foreach_volid($conf, sub {
5638         my ($volid, $attr) = @_;
5639
5640         return if $volid =~ m|^/|;
5641
5642         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5643         return if !$sid;
5644
5645         push @$vollist, $volid;
5646     });
5647
5648     return $vollist;
5649 }
5650
5651 sub vm_stop_cleanup {
5652     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5653
5654     eval {
5655
5656         if (!$keepActive) {
5657             my $vollist = get_vm_volumes($conf);
5658             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5659         }
5660
5661         foreach my $ext (qw(mon qmp pid vnc qga)) {
5662             unlink "/var/run/qemu-server/${vmid}.$ext";
5663         }
5664
5665         if ($conf->{ivshmem}) {
5666             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5667             # just delete it for now, VMs which have this already open do not
5668             # are affected, but new VMs will get a separated one. If this
5669             # becomes an issue we either add some sort of ref-counting or just
5670             # add a "don't delete on stop" flag to the ivshmem format.
5671             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5672         }
5673
5674         foreach my $key (keys %$conf) {
5675             next if $key !~ m/^hostpci(\d+)$/;
5676             my $hostpciindex = $1;
5677             my $d = parse_hostpci($conf->{$key});
5678             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5679
5680             foreach my $pci (@{$d->{pciid}}) {
5681                 my $pciid = $pci->{id};
5682                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5683             }
5684         }
5685
5686         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5687     };
5688     warn $@ if $@; # avoid errors - just warn
5689 }
5690
5691 # call only in locked context
5692 sub _do_vm_stop {
5693     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5694
5695     my $pid = check_running($vmid, $nocheck);
5696     return if !$pid;
5697
5698     my $conf;
5699     if (!$nocheck) {
5700         $conf = PVE::QemuConfig->load_config($vmid);
5701         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5702         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5703             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5704             $timeout = $opts->{down} if $opts->{down};
5705         }
5706         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5707     }
5708
5709     eval {
5710         if ($shutdown) {
5711             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5712                 vm_qmp_command($vmid, {
5713                         execute => "guest-shutdown",
5714                         arguments => { timeout => $timeout }
5715                     }, $nocheck);
5716             } else {
5717                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5718             }
5719         } else {
5720             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5721         }
5722     };
5723     my $err = $@;
5724
5725     if (!$err) {
5726         $timeout = 60 if !defined($timeout);
5727
5728         my $count = 0;
5729         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5730             $count++;
5731             sleep 1;
5732         }
5733
5734         if ($count >= $timeout) {
5735             if ($force) {
5736                 warn "VM still running - terminating now with SIGTERM\n";
5737                 kill 15, $pid;
5738             } else {
5739                 die "VM quit/powerdown failed - got timeout\n";
5740             }
5741         } else {
5742             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5743             return;
5744         }
5745     } else {
5746         if ($force) {
5747             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5748             kill 15, $pid;
5749         } else {
5750             die "VM quit/powerdown failed\n";
5751         }
5752     }
5753
5754     # wait again
5755     $timeout = 10;
5756
5757     my $count = 0;
5758     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5759         $count++;
5760         sleep 1;
5761     }
5762
5763     if ($count >= $timeout) {
5764         warn "VM still running - terminating now with SIGKILL\n";
5765         kill 9, $pid;
5766         sleep 1;
5767     }
5768
5769     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5770 }
5771
5772 # Note: use $nocheck to skip tests if VM configuration file exists.
5773 # We need that when migration VMs to other nodes (files already moved)
5774 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5775 sub vm_stop {
5776     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5777
5778     $force = 1 if !defined($force) && !$shutdown;
5779
5780     if ($migratedfrom){
5781         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5782         kill 15, $pid if $pid;
5783         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5784         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5785         return;
5786     }
5787
5788     PVE::QemuConfig->lock_config($vmid, sub {
5789         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5790    });
5791 }
5792
5793 sub vm_reboot {
5794     my ($vmid, $timeout) = @_;
5795
5796     PVE::QemuConfig->lock_config($vmid, sub {
5797
5798         # only reboot if running, as qmeventd starts it again on a stop event
5799         return if !check_running($vmid);
5800
5801         create_reboot_request($vmid);
5802
5803         my $storecfg = PVE::Storage::config();
5804         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5805
5806    });
5807 }
5808
5809 sub vm_suspend {
5810     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5811
5812     my $conf;
5813     my $path;
5814     my $storecfg;
5815     my $vmstate;
5816
5817     PVE::QemuConfig->lock_config($vmid, sub {
5818
5819         $conf = PVE::QemuConfig->load_config($vmid);
5820
5821         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5822         PVE::QemuConfig->check_lock($conf)
5823             if !($skiplock || $is_backing_up);
5824
5825         die "cannot suspend to disk during backup\n"
5826             if $is_backing_up && $includestate;
5827
5828         if ($includestate) {
5829             $conf->{lock} = 'suspending';
5830             my $date = strftime("%Y-%m-%d", localtime(time()));
5831             $storecfg = PVE::Storage::config();
5832             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5833             $path = PVE::Storage::path($storecfg, $vmstate);
5834             PVE::QemuConfig->write_config($vmid, $conf);
5835         } else {
5836             vm_mon_cmd($vmid, "stop");
5837         }
5838     });
5839
5840     if ($includestate) {
5841         # save vm state
5842         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5843
5844         eval {
5845             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5846             for(;;) {
5847                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5848                 if (!$state->{status}) {
5849                     die "savevm not active\n";
5850                 } elsif ($state->{status} eq 'active') {
5851                     sleep(1);
5852                     next;
5853                 } elsif ($state->{status} eq 'completed') {
5854                     print "State saved, quitting\n";
5855                     last;
5856                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5857                     die "query-savevm failed with error '$state->{error}'\n"
5858                 } else {
5859                     die "query-savevm returned status '$state->{status}'\n";
5860                 }
5861             }
5862         };
5863         my $err = $@;
5864
5865         PVE::QemuConfig->lock_config($vmid, sub {
5866             $conf = PVE::QemuConfig->load_config($vmid);
5867             if ($err) {
5868                 # cleanup, but leave suspending lock, to indicate something went wrong
5869                 eval {
5870                     vm_mon_cmd($vmid, "savevm-end");
5871                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5872                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5873                     delete $conf->@{qw(vmstate runningmachine)};
5874                     PVE::QemuConfig->write_config($vmid, $conf);
5875                 };
5876                 warn $@ if $@;
5877                 die $err;
5878             }
5879
5880             die "lock changed unexpectedly\n"
5881                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5882
5883             vm_qmp_command($vmid, { execute => "quit" });
5884             $conf->{lock} = 'suspended';
5885             PVE::QemuConfig->write_config($vmid, $conf);
5886         });
5887     }
5888 }
5889
5890 sub vm_resume {
5891     my ($vmid, $skiplock, $nocheck) = @_;
5892
5893     PVE::QemuConfig->lock_config($vmid, sub {
5894         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5895         my $res = $vm_mon_cmd->($vmid, 'query-status');
5896         my $resume_cmd = 'cont';
5897
5898         if ($res->{status} && $res->{status} eq 'suspended') {
5899             $resume_cmd = 'system_wakeup';
5900         }
5901
5902         if (!$nocheck) {
5903
5904             my $conf = PVE::QemuConfig->load_config($vmid);
5905
5906             PVE::QemuConfig->check_lock($conf)
5907                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5908         }
5909
5910         $vm_mon_cmd->($vmid, $resume_cmd);
5911     });
5912 }
5913
5914 sub vm_sendkey {
5915     my ($vmid, $skiplock, $key) = @_;
5916
5917     PVE::QemuConfig->lock_config($vmid, sub {
5918
5919         my $conf = PVE::QemuConfig->load_config($vmid);
5920
5921         # there is no qmp command, so we use the human monitor command
5922         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5923         die $res if $res ne '';
5924     });
5925 }
5926
5927 # vzdump restore implementaion
5928
5929 sub tar_archive_read_firstfile {
5930     my $archive = shift;
5931
5932     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5933
5934     # try to detect archive type first
5935     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5936         die "unable to open file '$archive'\n";
5937     my $firstfile = <$fh>;
5938     kill 15, $pid;
5939     close $fh;
5940
5941     die "ERROR: archive contaions no data\n" if !$firstfile;
5942     chomp $firstfile;
5943
5944     return $firstfile;
5945 }
5946
5947 sub tar_restore_cleanup {
5948     my ($storecfg, $statfile) = @_;
5949
5950     print STDERR "starting cleanup\n";
5951
5952     if (my $fd = IO::File->new($statfile, "r")) {
5953         while (defined(my $line = <$fd>)) {
5954             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5955                 my $volid = $2;
5956                 eval {
5957                     if ($volid =~ m|^/|) {
5958                         unlink $volid || die 'unlink failed\n';
5959                     } else {
5960                         PVE::Storage::vdisk_free($storecfg, $volid);
5961                     }
5962                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5963                 };
5964                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5965             } else {
5966                 print STDERR "unable to parse line in statfile - $line";
5967             }
5968         }
5969         $fd->close();
5970     }
5971 }
5972
5973 sub restore_archive {
5974     my ($archive, $vmid, $user, $opts) = @_;
5975
5976     my $format = $opts->{format};
5977     my $comp;
5978
5979     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5980         $format = 'tar' if !$format;
5981         $comp = 'gzip';
5982     } elsif ($archive =~ m/\.tar$/) {
5983         $format = 'tar' if !$format;
5984     } elsif ($archive =~ m/.tar.lzo$/) {
5985         $format = 'tar' if !$format;
5986         $comp = 'lzop';
5987     } elsif ($archive =~ m/\.vma$/) {
5988         $format = 'vma' if !$format;
5989     } elsif ($archive =~ m/\.vma\.gz$/) {
5990         $format = 'vma' if !$format;
5991         $comp = 'gzip';
5992     } elsif ($archive =~ m/\.vma\.lzo$/) {
5993         $format = 'vma' if !$format;
5994         $comp = 'lzop';
5995     } else {
5996         $format = 'vma' if !$format; # default
5997     }
5998
5999     # try to detect archive format
6000     if ($format eq 'tar') {
6001         return restore_tar_archive($archive, $vmid, $user, $opts);
6002     } else {
6003         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6004     }
6005 }
6006
6007 sub restore_update_config_line {
6008     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6009
6010     return if $line =~ m/^\#qmdump\#/;
6011     return if $line =~ m/^\#vzdump\#/;
6012     return if $line =~ m/^lock:/;
6013     return if $line =~ m/^unused\d+:/;
6014     return if $line =~ m/^parent:/;
6015
6016     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6017     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6018         # try to convert old 1.X settings
6019         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6020         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6021             my ($model, $macaddr) = split(/\=/, $devconfig);
6022             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6023             my $net = {
6024                 model => $model,
6025                 bridge => "vmbr$ind",
6026                 macaddr => $macaddr,
6027             };
6028             my $netstr = print_net($net);
6029
6030             print $outfd "net$cookie->{netcount}: $netstr\n";
6031             $cookie->{netcount}++;
6032         }
6033     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6034         my ($id, $netstr) = ($1, $2);
6035         my $net = parse_net($netstr);
6036         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6037         $netstr = print_net($net);
6038         print $outfd "$id: $netstr\n";
6039     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6040         my $virtdev = $1;
6041         my $value = $3;
6042         my $di = parse_drive($virtdev, $value);
6043         if (defined($di->{backup}) && !$di->{backup}) {
6044             print $outfd "#$line";
6045         } elsif ($map->{$virtdev}) {
6046             delete $di->{format}; # format can change on restore
6047             $di->{file} = $map->{$virtdev};
6048             $value = print_drive($vmid, $di);
6049             print $outfd "$virtdev: $value\n";
6050         } else {
6051             print $outfd $line;
6052         }
6053     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6054         my $vmgenid = $1;
6055         if ($vmgenid ne '0') {
6056             # always generate a new vmgenid if there was a valid one setup
6057             $vmgenid = generate_uuid();
6058         }
6059         print $outfd "vmgenid: $vmgenid\n";
6060     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6061         my ($uuid, $uuid_str);
6062         UUID::generate($uuid);
6063         UUID::unparse($uuid, $uuid_str);
6064         my $smbios1 = parse_smbios1($2);
6065         $smbios1->{uuid} = $uuid_str;
6066         print $outfd $1.print_smbios1($smbios1)."\n";
6067     } else {
6068         print $outfd $line;
6069     }
6070 }
6071
6072 sub scan_volids {
6073     my ($cfg, $vmid) = @_;
6074
6075     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6076
6077     my $volid_hash = {};
6078     foreach my $storeid (keys %$info) {
6079         foreach my $item (@{$info->{$storeid}}) {
6080             next if !($item->{volid} && $item->{size});
6081             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6082             $volid_hash->{$item->{volid}} = $item;
6083         }
6084     }
6085
6086     return $volid_hash;
6087 }
6088
6089 sub is_volume_in_use {
6090     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6091
6092     my $path = PVE::Storage::path($storecfg, $volid);
6093
6094     my $scan_config = sub {
6095         my ($cref, $snapname) = @_;
6096
6097         foreach my $key (keys %$cref) {
6098             my $value = $cref->{$key};
6099             if (is_valid_drivename($key)) {
6100                 next if $skip_drive && $key eq $skip_drive;
6101                 my $drive = parse_drive($key, $value);
6102                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6103                 return 1 if $volid eq $drive->{file};
6104                 if ($drive->{file} =~ m!^/!) {
6105                     return 1 if $drive->{file} eq $path;
6106                 } else {
6107                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6108                     next if !$storeid;
6109                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6110                     next if !$scfg;
6111                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6112                 }
6113             }
6114         }
6115
6116         return 0;
6117     };
6118
6119     return 1 if &$scan_config($conf);
6120
6121     undef $skip_drive;
6122
6123     foreach my $snapname (keys %{$conf->{snapshots}}) {
6124         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6125     }
6126
6127     return 0;
6128 }
6129
6130 sub update_disksize {
6131     my ($vmid, $conf, $volid_hash) = @_;
6132
6133     my $changes;
6134     my $prefix = "VM $vmid:";
6135
6136     # used and unused disks
6137     my $referenced = {};
6138
6139     # Note: it is allowed to define multiple storages with same path (alias), so
6140     # we need to check both 'volid' and real 'path' (two different volid can point
6141     # to the same path).
6142
6143     my $referencedpath = {};
6144
6145     # update size info
6146     foreach my $opt (keys %$conf) {
6147         if (is_valid_drivename($opt)) {
6148             my $drive = parse_drive($opt, $conf->{$opt});
6149             my $volid = $drive->{file};
6150             next if !$volid;
6151
6152             $referenced->{$volid} = 1;
6153             if ($volid_hash->{$volid} &&
6154                 (my $path = $volid_hash->{$volid}->{path})) {
6155                 $referencedpath->{$path} = 1;
6156             }
6157
6158             next if drive_is_cdrom($drive);
6159             next if !$volid_hash->{$volid};
6160
6161             $drive->{size} = $volid_hash->{$volid}->{size};
6162             my $new = print_drive($vmid, $drive);
6163             if ($new ne $conf->{$opt}) {
6164                 $changes = 1;
6165                 $conf->{$opt} = $new;
6166                 print "$prefix update disk '$opt' information.\n";
6167             }
6168         }
6169     }
6170
6171     # remove 'unusedX' entry if volume is used
6172     foreach my $opt (keys %$conf) {
6173         next if $opt !~ m/^unused\d+$/;
6174         my $volid = $conf->{$opt};
6175         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6176         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6177             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6178             $changes = 1;
6179             delete $conf->{$opt};
6180         }
6181
6182         $referenced->{$volid} = 1;
6183         $referencedpath->{$path} = 1 if $path;
6184     }
6185
6186     foreach my $volid (sort keys %$volid_hash) {
6187         next if $volid =~ m/vm-$vmid-state-/;
6188         next if $referenced->{$volid};
6189         my $path = $volid_hash->{$volid}->{path};
6190         next if !$path; # just to be sure
6191         next if $referencedpath->{$path};
6192         $changes = 1;
6193         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6194         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6195         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6196     }
6197
6198     return $changes;
6199 }
6200
6201 sub rescan {
6202     my ($vmid, $nolock, $dryrun) = @_;
6203
6204     my $cfg = PVE::Storage::config();
6205
6206     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6207     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6208     foreach my $stor (keys %{$cfg->{ids}}) {
6209         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6210     }
6211
6212     print "rescan volumes...\n";
6213     my $volid_hash = scan_volids($cfg, $vmid);
6214
6215     my $updatefn =  sub {
6216         my ($vmid) = @_;
6217
6218         my $conf = PVE::QemuConfig->load_config($vmid);
6219
6220         PVE::QemuConfig->check_lock($conf);
6221
6222         my $vm_volids = {};
6223         foreach my $volid (keys %$volid_hash) {
6224             my $info = $volid_hash->{$volid};
6225             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6226         }
6227
6228         my $changes = update_disksize($vmid, $conf, $vm_volids);
6229
6230         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6231     };
6232
6233     if (defined($vmid)) {
6234         if ($nolock) {
6235             &$updatefn($vmid);
6236         } else {
6237             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6238         }
6239     } else {
6240         my $vmlist = config_list();
6241         foreach my $vmid (keys %$vmlist) {
6242             if ($nolock) {
6243                 &$updatefn($vmid);
6244             } else {
6245                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6246             }
6247         }
6248     }
6249 }
6250
6251 sub restore_vma_archive {
6252     my ($archive, $vmid, $user, $opts, $comp) = @_;
6253
6254     my $readfrom = $archive;
6255
6256     my $cfg = PVE::Storage::config();
6257     my $commands = [];
6258     my $bwlimit = $opts->{bwlimit};
6259
6260     my $dbg_cmdstring = '';
6261     my $add_pipe = sub {
6262         my ($cmd) = @_;
6263         push @$commands, $cmd;
6264         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6265         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6266         $readfrom = '-';
6267     };
6268
6269     my $input = undef;
6270     if ($archive eq '-') {
6271         $input = '<&STDIN';
6272     } else {
6273         # If we use a backup from a PVE defined storage we also consider that
6274         # storage's rate limit:
6275         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6276         if (defined($volid)) {
6277             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6278             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6279             if ($readlimit) {
6280                 print STDERR "applying read rate limit: $readlimit\n";
6281                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6282                 $add_pipe->($cstream);
6283             }
6284         }
6285     }
6286
6287     if ($comp) {
6288         my $cmd;
6289         if ($comp eq 'gzip') {
6290             $cmd = ['zcat', $readfrom];
6291         } elsif ($comp eq 'lzop') {
6292             $cmd = ['lzop', '-d', '-c', $readfrom];
6293         } else {
6294             die "unknown compression method '$comp'\n";
6295         }
6296         $add_pipe->($cmd);
6297     }
6298
6299     my $tmpdir = "/var/tmp/vzdumptmp$$";
6300     rmtree $tmpdir;
6301
6302     # disable interrupts (always do cleanups)
6303     local $SIG{INT} =
6304         local $SIG{TERM} =
6305         local $SIG{QUIT} =
6306         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6307
6308     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6309     POSIX::mkfifo($mapfifo, 0600);
6310     my $fifofh;
6311
6312     my $openfifo = sub {
6313         open($fifofh, '>', $mapfifo) || die $!;
6314     };
6315
6316     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6317
6318     my $oldtimeout;
6319     my $timeout = 5;
6320
6321     my $devinfo = {};
6322
6323     my $rpcenv = PVE::RPCEnvironment::get();
6324
6325     my $conffile = PVE::QemuConfig->config_file($vmid);
6326     my $tmpfn = "$conffile.$$.tmp";
6327
6328     # Note: $oldconf is undef if VM does not exists
6329     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6330     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6331
6332     my %storage_limits;
6333
6334     my $print_devmap = sub {
6335         my $virtdev_hash = {};
6336
6337         my $cfgfn = "$tmpdir/qemu-server.conf";
6338
6339         # we can read the config - that is already extracted
6340         my $fh = IO::File->new($cfgfn, "r") ||
6341             "unable to read qemu-server.conf - $!\n";
6342
6343         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6344         if (-f $fwcfgfn) {
6345             my $pve_firewall_dir = '/etc/pve/firewall';
6346             mkdir $pve_firewall_dir; # make sure the dir exists
6347             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6348         }
6349
6350         while (defined(my $line = <$fh>)) {
6351             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6352                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6353                 die "archive does not contain data for drive '$virtdev'\n"
6354                     if !$devinfo->{$devname};
6355                 if (defined($opts->{storage})) {
6356                     $storeid = $opts->{storage} || 'local';
6357                 } elsif (!$storeid) {
6358                     $storeid = 'local';
6359                 }
6360                 $format = 'raw' if !$format;
6361                 $devinfo->{$devname}->{devname} = $devname;
6362                 $devinfo->{$devname}->{virtdev} = $virtdev;
6363                 $devinfo->{$devname}->{format} = $format;
6364                 $devinfo->{$devname}->{storeid} = $storeid;
6365
6366                 # check permission on storage
6367                 my $pool = $opts->{pool}; # todo: do we need that?
6368                 if ($user ne 'root@pam') {
6369                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6370                 }
6371
6372                 $storage_limits{$storeid} = $bwlimit;
6373
6374                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6375             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6376                 my $virtdev = $1;
6377                 my $drive = parse_drive($virtdev, $2);
6378                 if (drive_is_cloudinit($drive)) {
6379                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6380                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6381                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6382
6383                     my $d = {
6384                         format => $format,
6385                         storeid => $opts->{storage} // $storeid,
6386                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6387                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6388                         name => "vm-$vmid-cloudinit",
6389                         is_cloudinit => 1,
6390                     };
6391                     $virtdev_hash->{$virtdev} = $d;
6392                 }
6393             }
6394         }
6395
6396         foreach my $key (keys %storage_limits) {
6397             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6398             next if !$limit;
6399             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6400             $storage_limits{$key} = $limit * 1024;
6401         }
6402
6403         foreach my $devname (keys %$devinfo) {
6404             die "found no device mapping information for device '$devname'\n"
6405                 if !$devinfo->{$devname}->{virtdev};
6406         }
6407
6408         # create empty/temp config
6409         if ($oldconf) {
6410             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6411             foreach_drive($oldconf, sub {
6412                 my ($ds, $drive) = @_;
6413
6414                 return if drive_is_cdrom($drive, 1);
6415
6416                 my $volid = $drive->{file};
6417                 return if !$volid || $volid =~ m|^/|;
6418
6419                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6420                 return if !$path || !$owner || ($owner != $vmid);
6421
6422                 # Note: only delete disk we want to restore
6423                 # other volumes will become unused
6424                 if ($virtdev_hash->{$ds}) {
6425                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6426                     if (my $err = $@) {
6427                         warn $err;
6428                     }
6429                 }
6430             });
6431
6432             # delete vmstate files, after the restore we have no snapshots anymore
6433             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6434                 my $snap = $oldconf->{snapshots}->{$snapname};
6435                 if ($snap->{vmstate}) {
6436                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6437                     if (my $err = $@) {
6438                         warn $err;
6439                     }
6440                 }
6441             }
6442         }
6443
6444         my $map = {};
6445         foreach my $virtdev (sort keys %$virtdev_hash) {
6446             my $d = $virtdev_hash->{$virtdev};
6447             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6448             my $storeid = $d->{storeid};
6449             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6450
6451             my $map_opts = '';
6452             if (my $limit = $storage_limits{$storeid}) {
6453                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6454             }
6455
6456             # test if requested format is supported
6457             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6458             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6459             $d->{format} = $defFormat if !$supported;
6460
6461             my $name;
6462             if ($d->{is_cloudinit}) {
6463                 $name = $d->{name};
6464                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6465             }
6466
6467             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6468             print STDERR "new volume ID is '$volid'\n";
6469             $d->{volid} = $volid;
6470
6471             PVE::Storage::activate_volumes($cfg, [$volid]);
6472
6473             my $write_zeros = 1;
6474             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6475                 $write_zeros = 0;
6476             }
6477
6478             if (!$d->{is_cloudinit}) {
6479                 my $path = PVE::Storage::path($cfg, $volid);
6480
6481                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6482
6483                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6484             }
6485             $map->{$virtdev} = $volid;
6486         }
6487
6488         $fh->seek(0, 0) || die "seek failed - $!\n";
6489
6490         my $outfd = new IO::File ($tmpfn, "w") ||
6491             die "unable to write config for VM $vmid\n";
6492
6493         my $cookie = { netcount => 0 };
6494         while (defined(my $line = <$fh>)) {
6495             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6496         }
6497
6498         $fh->close();
6499         $outfd->close();
6500     };
6501
6502     eval {
6503         # enable interrupts
6504         local $SIG{INT} =
6505             local $SIG{TERM} =
6506             local $SIG{QUIT} =
6507             local $SIG{HUP} =
6508             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6509         local $SIG{ALRM} = sub { die "got timeout\n"; };
6510
6511         $oldtimeout = alarm($timeout);
6512
6513         my $parser = sub {
6514             my $line = shift;
6515
6516             print "$line\n";
6517
6518             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6519                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6520                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6521             } elsif ($line =~ m/^CTIME: /) {
6522                 # we correctly received the vma config, so we can disable
6523                 # the timeout now for disk allocation (set to 10 minutes, so
6524                 # that we always timeout if something goes wrong)
6525                 alarm(600);
6526                 &$print_devmap();
6527                 print $fifofh "done\n";
6528                 my $tmp = $oldtimeout || 0;
6529                 $oldtimeout = undef;
6530                 alarm($tmp);
6531                 close($fifofh);
6532             }
6533         };
6534
6535         print "restore vma archive: $dbg_cmdstring\n";
6536         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6537     };
6538     my $err = $@;
6539
6540     alarm($oldtimeout) if $oldtimeout;
6541
6542     my $vollist = [];
6543     foreach my $devname (keys %$devinfo) {
6544         my $volid = $devinfo->{$devname}->{volid};
6545         push @$vollist, $volid if $volid;
6546     }
6547
6548     PVE::Storage::deactivate_volumes($cfg, $vollist);
6549
6550     unlink $mapfifo;
6551
6552     if ($err) {
6553         rmtree $tmpdir;
6554         unlink $tmpfn;
6555
6556         foreach my $devname (keys %$devinfo) {
6557             my $volid = $devinfo->{$devname}->{volid};
6558             next if !$volid;
6559             eval {
6560                 if ($volid =~ m|^/|) {
6561                     unlink $volid || die 'unlink failed\n';
6562                 } else {
6563                     PVE::Storage::vdisk_free($cfg, $volid);
6564                 }
6565                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6566             };
6567             print STDERR "unable to cleanup '$volid' - $@" if $@;
6568         }
6569         die $err;
6570     }
6571
6572     rmtree $tmpdir;
6573
6574     rename($tmpfn, $conffile) ||
6575         die "unable to commit configuration file '$conffile'\n";
6576
6577     PVE::Cluster::cfs_update(); # make sure we read new file
6578
6579     eval { rescan($vmid, 1); };
6580     warn $@ if $@;
6581 }
6582
6583 sub restore_tar_archive {
6584     my ($archive, $vmid, $user, $opts) = @_;
6585
6586     if ($archive ne '-') {
6587         my $firstfile = tar_archive_read_firstfile($archive);
6588         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6589             if $firstfile ne 'qemu-server.conf';
6590     }
6591
6592     my $storecfg = PVE::Storage::config();
6593
6594     # avoid zombie disks when restoring over an existing VM -> cleanup first
6595     # pass keep_empty_config=1 to keep the config (thus VMID) reserved for us
6596     # skiplock=1 because qmrestore has set the 'create' lock itself already
6597     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6598     destroy_vm($storecfg, $vmid, 1, { lock => 'restore' }) if -f $vmcfgfn;
6599
6600     my $tocmd = "/usr/lib/qemu-server/qmextract";
6601
6602     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6603     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6604     $tocmd .= ' --prealloc' if $opts->{prealloc};
6605     $tocmd .= ' --info' if $opts->{info};
6606
6607     # tar option "xf" does not autodetect compression when read from STDIN,
6608     # so we pipe to zcat
6609     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6610         PVE::Tools::shellquote("--to-command=$tocmd");
6611
6612     my $tmpdir = "/var/tmp/vzdumptmp$$";
6613     mkpath $tmpdir;
6614
6615     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6616     local $ENV{VZDUMP_VMID} = $vmid;
6617     local $ENV{VZDUMP_USER} = $user;
6618
6619     my $conffile = PVE::QemuConfig->config_file($vmid);
6620     my $tmpfn = "$conffile.$$.tmp";
6621
6622     # disable interrupts (always do cleanups)
6623     local $SIG{INT} =
6624         local $SIG{TERM} =
6625         local $SIG{QUIT} =
6626         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6627
6628     eval {
6629         # enable interrupts
6630         local $SIG{INT} =
6631             local $SIG{TERM} =
6632             local $SIG{QUIT} =
6633             local $SIG{HUP} =
6634             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6635
6636         if ($archive eq '-') {
6637             print "extracting archive from STDIN\n";
6638             run_command($cmd, input => "<&STDIN");
6639         } else {
6640             print "extracting archive '$archive'\n";
6641             run_command($cmd);
6642         }
6643
6644         return if $opts->{info};
6645
6646         # read new mapping
6647         my $map = {};
6648         my $statfile = "$tmpdir/qmrestore.stat";
6649         if (my $fd = IO::File->new($statfile, "r")) {
6650             while (defined (my $line = <$fd>)) {
6651                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6652                     $map->{$1} = $2 if $1;
6653                 } else {
6654                     print STDERR "unable to parse line in statfile - $line\n";
6655                 }
6656             }
6657             $fd->close();
6658         }
6659
6660         my $confsrc = "$tmpdir/qemu-server.conf";
6661
6662         my $srcfd = new IO::File($confsrc, "r") ||
6663             die "unable to open file '$confsrc'\n";
6664
6665         my $outfd = new IO::File ($tmpfn, "w") ||
6666             die "unable to write config for VM $vmid\n";
6667
6668         my $cookie = { netcount => 0 };
6669         while (defined (my $line = <$srcfd>)) {
6670             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6671         }
6672
6673         $srcfd->close();
6674         $outfd->close();
6675     };
6676     if (my $err = $@) {
6677         unlink $tmpfn;
6678         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6679         die $err;
6680     }
6681
6682     rmtree $tmpdir;
6683
6684     rename $tmpfn, $conffile ||
6685         die "unable to commit configuration file '$conffile'\n";
6686
6687     PVE::Cluster::cfs_update(); # make sure we read new file
6688
6689     eval { rescan($vmid, 1); };
6690     warn $@ if $@;
6691 };
6692
6693 sub foreach_storage_used_by_vm {
6694     my ($conf, $func) = @_;
6695
6696     my $sidhash = {};
6697
6698     foreach_drive($conf, sub {
6699         my ($ds, $drive) = @_;
6700         return if drive_is_cdrom($drive);
6701
6702         my $volid = $drive->{file};
6703
6704         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6705         $sidhash->{$sid} = $sid if $sid;
6706     });
6707
6708     foreach my $sid (sort keys %$sidhash) {
6709         &$func($sid);
6710     }
6711 }
6712
6713 sub do_snapshots_with_qemu {
6714     my ($storecfg, $volid) = @_;
6715
6716     my $storage_name = PVE::Storage::parse_volume_id($volid);
6717     my $scfg = $storecfg->{ids}->{$storage_name};
6718
6719     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6720         return 1;
6721     }
6722
6723     if ($volid =~ m/\.(qcow2|qed)$/){
6724         return 1;
6725     }
6726
6727     return undef;
6728 }
6729
6730 sub qga_check_running {
6731     my ($vmid, $nowarn) = @_;
6732
6733     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6734     if ($@) {
6735         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6736         return 0;
6737     }
6738     return 1;
6739 }
6740
6741 sub template_create {
6742     my ($vmid, $conf, $disk) = @_;
6743
6744     my $storecfg = PVE::Storage::config();
6745
6746     foreach_drive($conf, sub {
6747         my ($ds, $drive) = @_;
6748
6749         return if drive_is_cdrom($drive);
6750         return if $disk && $ds ne $disk;
6751
6752         my $volid = $drive->{file};
6753         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6754
6755         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6756         $drive->{file} = $voliddst;
6757         $conf->{$ds} = print_drive($vmid, $drive);
6758         PVE::QemuConfig->write_config($vmid, $conf);
6759     });
6760 }
6761
6762 sub convert_iscsi_path {
6763     my ($path) = @_;
6764
6765     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6766         my $portal = $1;
6767         my $target = $2;
6768         my $lun = $3;
6769
6770         my $initiator_name = get_initiator_name();
6771
6772         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6773                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6774     }
6775
6776     die "cannot convert iscsi path '$path', unkown format\n";
6777 }
6778
6779 sub qemu_img_convert {
6780     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6781
6782     my $storecfg = PVE::Storage::config();
6783     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6784     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6785
6786     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6787
6788     my $cachemode;
6789     my $src_path;
6790     my $src_is_iscsi = 0;
6791     my $src_format = 'raw';
6792
6793     if ($src_storeid) {
6794         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6795         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6796         $src_format = qemu_img_format($src_scfg, $src_volname);
6797         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6798         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6799         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6800     } elsif (-f $src_volid) {
6801         $src_path = $src_volid;
6802         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6803             $src_format = $1;
6804         }
6805     }
6806
6807     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6808
6809     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6810     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6811     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6812     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6813
6814     my $cmd = [];
6815     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6816     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6817     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6818     push @$cmd, '-T', $cachemode if defined($cachemode);
6819
6820     if ($src_is_iscsi) {
6821         push @$cmd, '--image-opts';
6822         $src_path = convert_iscsi_path($src_path);
6823     } else {
6824         push @$cmd, '-f', $src_format;
6825     }
6826
6827     if ($dst_is_iscsi) {
6828         push @$cmd, '--target-image-opts';
6829         $dst_path = convert_iscsi_path($dst_path);
6830     } else {
6831         push @$cmd, '-O', $dst_format;
6832     }
6833
6834     push @$cmd, $src_path;
6835
6836     if (!$dst_is_iscsi && $is_zero_initialized) {
6837         push @$cmd, "zeroinit:$dst_path";
6838     } else {
6839         push @$cmd, $dst_path;
6840     }
6841
6842     my $parser = sub {
6843         my $line = shift;
6844         if($line =~ m/\((\S+)\/100\%\)/){
6845             my $percent = $1;
6846             my $transferred = int($size * $percent / 100);
6847             my $remaining = $size - $transferred;
6848
6849             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6850         }
6851
6852     };
6853
6854     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6855     my $err = $@;
6856     die "copy failed: $err" if $err;
6857 }
6858
6859 sub qemu_img_format {
6860     my ($scfg, $volname) = @_;
6861
6862     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6863         return $1;
6864     } else {
6865         return "raw";
6866     }
6867 }
6868
6869 sub qemu_drive_mirror {
6870     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6871
6872     $jobs = {} if !$jobs;
6873
6874     my $qemu_target;
6875     my $format;
6876     $jobs->{"drive-$drive"} = {};
6877
6878     if ($dst_volid =~ /^nbd:/) {
6879         $qemu_target = $dst_volid;
6880         $format = "nbd";
6881     } else {
6882         my $storecfg = PVE::Storage::config();
6883         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6884
6885         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6886
6887         $format = qemu_img_format($dst_scfg, $dst_volname);
6888
6889         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6890
6891         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6892     }
6893
6894     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6895     $opts->{format} = $format if $format;
6896
6897     if (defined($bwlimit)) {
6898         $opts->{speed} = $bwlimit * 1024;
6899         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6900     } else {
6901         print "drive mirror is starting for drive-$drive\n";
6902     }
6903
6904     # if a job already runs for this device we get an error, catch it for cleanup
6905     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6906     if (my $err = $@) {
6907         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6908         warn "$@\n" if $@;
6909         die "mirroring error: $err\n";
6910     }
6911
6912     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6913 }
6914
6915 sub qemu_drive_mirror_monitor {
6916     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6917
6918     eval {
6919         my $err_complete = 0;
6920
6921         while (1) {
6922             die "storage migration timed out\n" if $err_complete > 300;
6923
6924             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6925
6926             my $running_mirror_jobs = {};
6927             foreach my $stat (@$stats) {
6928                 next if $stat->{type} ne 'mirror';
6929                 $running_mirror_jobs->{$stat->{device}} = $stat;
6930             }
6931
6932             my $readycounter = 0;
6933
6934             foreach my $job (keys %$jobs) {
6935
6936                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6937                     print "$job : finished\n";
6938                     delete $jobs->{$job};
6939                     next;
6940                 }
6941
6942                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6943
6944                 my $busy = $running_mirror_jobs->{$job}->{busy};
6945                 my $ready = $running_mirror_jobs->{$job}->{ready};
6946                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6947                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6948                     my $remaining = $total - $transferred;
6949                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6950
6951                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6952                 }
6953
6954                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6955             }
6956
6957             last if scalar(keys %$jobs) == 0;
6958
6959             if ($readycounter == scalar(keys %$jobs)) {
6960                 print "all mirroring jobs are ready \n";
6961                 last if $skipcomplete; #do the complete later
6962
6963                 if ($vmiddst && $vmiddst != $vmid) {
6964                     my $agent_running = $qga && qga_check_running($vmid);
6965                     if ($agent_running) {
6966                         print "freeze filesystem\n";
6967                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6968                     } else {
6969                         print "suspend vm\n";
6970                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6971                     }
6972
6973                     # if we clone a disk for a new target vm, we don't switch the disk
6974                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6975
6976                     if ($agent_running) {
6977                         print "unfreeze filesystem\n";
6978                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6979                     } else {
6980                         print "resume vm\n";
6981                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6982                     }
6983
6984                     last;
6985                 } else {
6986
6987                     foreach my $job (keys %$jobs) {
6988                         # try to switch the disk if source and destination are on the same guest
6989                         print "$job: Completing block job...\n";
6990
6991                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6992                         if ($@ =~ m/cannot be completed/) {
6993                             print "$job: Block job cannot be completed, try again.\n";
6994                             $err_complete++;
6995                         }else {
6996                             print "$job: Completed successfully.\n";
6997                             $jobs->{$job}->{complete} = 1;
6998                         }
6999                     }
7000                 }
7001             }
7002             sleep 1;
7003         }
7004     };
7005     my $err = $@;
7006
7007     if ($err) {
7008         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7009         die "mirroring error: $err";
7010     }
7011
7012 }
7013
7014 sub qemu_blockjobs_cancel {
7015     my ($vmid, $jobs) = @_;
7016
7017     foreach my $job (keys %$jobs) {
7018         print "$job: Cancelling block job\n";
7019         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7020         $jobs->{$job}->{cancel} = 1;
7021     }
7022
7023     while (1) {
7024         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7025
7026         my $running_jobs = {};
7027         foreach my $stat (@$stats) {
7028             $running_jobs->{$stat->{device}} = $stat;
7029         }
7030
7031         foreach my $job (keys %$jobs) {
7032
7033             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7034                 print "$job: Done.\n";
7035                 delete $jobs->{$job};
7036             }
7037         }
7038
7039         last if scalar(keys %$jobs) == 0;
7040
7041         sleep 1;
7042     }
7043 }
7044
7045 sub clone_disk {
7046     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7047         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7048
7049     my $newvolid;
7050
7051     if (!$full) {
7052         print "create linked clone of drive $drivename ($drive->{file})\n";
7053         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7054         push @$newvollist, $newvolid;
7055     } else {
7056
7057         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7058         $storeid = $storage if $storage;
7059
7060         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7061         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7062
7063         print "create full clone of drive $drivename ($drive->{file})\n";
7064         my $name = undef;
7065         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7066         push @$newvollist, $newvolid;
7067
7068         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7069
7070         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7071         if (!$running || $snapname) {
7072             # TODO: handle bwlimits
7073             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7074         } else {
7075
7076             my $kvmver = get_running_qemu_version ($vmid);
7077             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7078                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7079                     if $drive->{iothread};
7080             }
7081
7082             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7083         }
7084     }
7085
7086     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7087
7088     my $disk = $drive;
7089     $disk->{format} = undef;
7090     $disk->{file} = $newvolid;
7091     $disk->{size} = $size;
7092
7093     return $disk;
7094 }
7095
7096 # this only works if VM is running
7097 sub get_current_qemu_machine {
7098     my ($vmid) = @_;
7099
7100     my $cmd = { execute => 'query-machines', arguments => {} };
7101     my $res = vm_qmp_command($vmid, $cmd);
7102
7103     my ($current, $default);
7104     foreach my $e (@$res) {
7105         $default = $e->{name} if $e->{'is-default'};
7106         $current = $e->{name} if $e->{'is-current'};
7107     }
7108
7109     # fallback to the default machine if current is not supported by qemu
7110     return $current || $default || 'pc';
7111 }
7112
7113 sub get_running_qemu_version {
7114     my ($vmid) = @_;
7115     my $cmd = { execute => 'query-version', arguments => {} };
7116     my $res = vm_qmp_command($vmid, $cmd);
7117     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7118 }
7119
7120 sub qemu_machine_feature_enabled {
7121     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7122
7123     my $current_major;
7124     my $current_minor;
7125
7126     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7127
7128         $current_major = $3;
7129         $current_minor = $4;
7130
7131     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7132
7133         $current_major = $1;
7134         $current_minor = $2;
7135     }
7136
7137     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7138 }
7139
7140 # gets in pairs the versions you want to compares, i.e.:
7141 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7142 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7143 sub version_cmp {
7144     my @versions = @_;
7145
7146     my $size = scalar(@versions);
7147
7148     return 0 if $size == 0;
7149     die "cannot compare odd count of versions" if $size & 1;
7150
7151     for (my $i = 0; $i < $size; $i += 2) {
7152         my ($a, $b) = splice(@versions, 0, 2);
7153         $a //= 0;
7154         $b //= 0;
7155
7156         return 1 if $a > $b;
7157         return -1 if $a < $b;
7158     }
7159     return 0;
7160 }
7161
7162 # dies if a) VM not running or not exisiting b) Version query failed
7163 # So, any defined return value is valid, any invalid state can be caught by eval
7164 sub runs_at_least_qemu_version {
7165     my ($vmid, $major, $minor, $extra) = @_;
7166
7167     my $v = vm_qmp_command($vmid, { execute => 'query-version' });
7168     die "could not query currently running version for VM $vmid\n" if !defined($v);
7169     $v = $v->{qemu};
7170
7171     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7172 }
7173
7174 sub qemu_machine_pxe {
7175     my ($vmid, $conf) = @_;
7176
7177     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7178
7179     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7180         $machine .= '.pxe';
7181     }
7182
7183     return $machine;
7184 }
7185
7186 sub qemu_use_old_bios_files {
7187     my ($machine_type) = @_;
7188
7189     return if !$machine_type;
7190
7191     my $use_old_bios_files = undef;
7192
7193     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7194         $machine_type = $1;
7195         $use_old_bios_files = 1;
7196     } else {
7197         my $kvmver = kvm_user_version();
7198         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7199         # load new efi bios files on migration. So this hack is required to allow
7200         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7201         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7202         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7203     }
7204
7205     return ($use_old_bios_files, $machine_type);
7206 }
7207
7208 sub create_efidisk($$$$$) {
7209     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7210
7211     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7212     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7213
7214     my $vars_size_b = -s $ovmf_vars;
7215     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7216     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7217     PVE::Storage::activate_volumes($storecfg, [$volid]);
7218
7219     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7220
7221     return ($volid, $vars_size);
7222 }
7223
7224 sub vm_iothreads_list {
7225     my ($vmid) = @_;
7226
7227     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7228
7229     my $iothreads = {};
7230     foreach my $iothread (@$res) {
7231         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7232     }
7233
7234     return $iothreads;
7235 }
7236
7237 sub scsihw_infos {
7238     my ($conf, $drive) = @_;
7239
7240     my $maxdev = 0;
7241
7242     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7243         $maxdev = 7;
7244     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7245         $maxdev = 1;
7246     } else {
7247         $maxdev = 256;
7248     }
7249
7250     my $controller = int($drive->{index} / $maxdev);
7251     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7252
7253     return ($maxdev, $controller, $controller_prefix);
7254 }
7255
7256 sub add_hyperv_enlightenments {
7257     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7258
7259     return if $winversion < 6;
7260     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7261
7262     if ($gpu_passthrough || defined($hv_vendor_id)) {
7263         $hv_vendor_id //= 'proxmox';
7264         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7265     }
7266
7267     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7268         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7269         push @$cpuFlags , 'hv_vapic';
7270         push @$cpuFlags , 'hv_time';
7271     } else {
7272         push @$cpuFlags , 'hv_spinlocks=0xffff';
7273     }
7274
7275     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7276         push @$cpuFlags , 'hv_reset';
7277         push @$cpuFlags , 'hv_vpindex';
7278         push @$cpuFlags , 'hv_runtime';
7279     }
7280
7281     if ($winversion >= 7) {
7282         push @$cpuFlags , 'hv_relaxed';
7283
7284         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7285             push @$cpuFlags , 'hv_synic';
7286             push @$cpuFlags , 'hv_stimer';
7287         }
7288
7289         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7290             push @$cpuFlags , 'hv_ipi';
7291         }
7292     }
7293 }
7294
7295 sub windows_version {
7296     my ($ostype) = @_;
7297
7298     return 0 if !$ostype;
7299
7300     my $winversion = 0;
7301
7302     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7303         $winversion = 5;
7304     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7305         $winversion = 6;
7306     } elsif ($ostype =~ m/^win(\d+)$/) {
7307         $winversion = $1;
7308     }
7309
7310     return $winversion;
7311 }
7312
7313 sub resolve_dst_disk_format {
7314         my ($storecfg, $storeid, $src_volname, $format) = @_;
7315         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7316
7317         if (!$format) {
7318             # if no target format is specified, use the source disk format as hint
7319             if ($src_volname) {
7320                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7321                 $format = qemu_img_format($scfg, $src_volname);
7322             } else {
7323                 return $defFormat;
7324             }
7325         }
7326
7327         # test if requested format is supported - else use default
7328         my $supported = grep { $_ eq $format } @$validFormats;
7329         $format = $defFormat if !$supported;
7330         return $format;
7331 }
7332
7333 sub resolve_first_disk {
7334     my $conf = shift;
7335     my @disks = PVE::QemuServer::valid_drive_names();
7336     my $firstdisk;
7337     foreach my $ds (reverse @disks) {
7338         next if !$conf->{$ds};
7339         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7340         next if PVE::QemuServer::drive_is_cdrom($disk);
7341         $firstdisk = $ds;
7342     }
7343     return $firstdisk;
7344 }
7345
7346 sub generate_uuid {
7347     my ($uuid, $uuid_str);
7348     UUID::generate($uuid);
7349     UUID::unparse($uuid, $uuid_str);
7350     return $uuid_str;
7351 }
7352
7353 sub generate_smbios1_uuid {
7354     return "uuid=".generate_uuid();
7355 }
7356
7357 sub nbd_stop {
7358     my ($vmid) = @_;
7359
7360     vm_mon_cmd($vmid, 'nbd-server-stop');
7361 }
7362
7363 sub create_reboot_request {
7364     my ($vmid) = @_;
7365     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7366         or die "failed to create reboot trigger file: $!\n";
7367     close($fh);
7368 }
7369
7370 sub clear_reboot_request {
7371     my ($vmid) = @_;
7372     my $path = "/run/qemu-server/$vmid.reboot";
7373     my $res = 0;
7374
7375     $res = unlink($path);
7376     die "could not remove reboot request for $vmid: $!"
7377         if !$res && $! != POSIX::ENOENT;
7378
7379     return $res;
7380 }
7381
7382 # bash completion helper
7383
7384 sub complete_backup_archives {
7385     my ($cmdname, $pname, $cvalue) = @_;
7386
7387     my $cfg = PVE::Storage::config();
7388
7389     my $storeid;
7390
7391     if ($cvalue =~ m/^([^:]+):/) {
7392         $storeid = $1;
7393     }
7394
7395     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7396
7397     my $res = [];
7398     foreach my $id (keys %$data) {
7399         foreach my $item (@{$data->{$id}}) {
7400             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7401             push @$res, $item->{volid} if defined($item->{volid});
7402         }
7403     }
7404
7405     return $res;
7406 }
7407
7408 my $complete_vmid_full = sub {
7409     my ($running) = @_;
7410
7411     my $idlist = vmstatus();
7412
7413     my $res = [];
7414
7415     foreach my $id (keys %$idlist) {
7416         my $d = $idlist->{$id};
7417         if (defined($running)) {
7418             next if $d->{template};
7419             next if $running && $d->{status} ne 'running';
7420             next if !$running && $d->{status} eq 'running';
7421         }
7422         push @$res, $id;
7423
7424     }
7425     return $res;
7426 };
7427
7428 sub complete_vmid {
7429     return &$complete_vmid_full();
7430 }
7431
7432 sub complete_vmid_stopped {
7433     return &$complete_vmid_full(0);
7434 }
7435
7436 sub complete_vmid_running {
7437     return &$complete_vmid_full(1);
7438 }
7439
7440 sub complete_storage {
7441
7442     my $cfg = PVE::Storage::config();
7443     my $ids = $cfg->{ids};
7444
7445     my $res = [];
7446     foreach my $sid (keys %$ids) {
7447         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7448         next if !$ids->{$sid}->{content}->{images};
7449         push @$res, $sid;
7450     }
7451
7452     return $res;
7453 }
7454
7455 1;