PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5
   6 use Cwd 'abs_path';
   7 use Digest::SHA;
   8 use Fcntl ':flock';
   9 use Fcntl;
  10 use File::Basename;
  11 use File::Copy qw(copy);
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use IO::Dir;
  16 use IO::File;
  17 use IO::Handle;
  18 use IO::Select;
  19 use IO::Socket::UNIX;
  20 use IPC::Open3;
  21 use JSON;
  22 use MIME::Base64;
  23 use POSIX;
  24 use Storable qw(dclone);
  25 use Time::HiRes qw(gettimeofday);
  26 use URI::Escape;
  27 use UUID;
  28
  29 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  30 use PVE::Exception qw(raise raise_param_exc);
  31 use PVE::GuestHelpers;
  32 use PVE::INotify;
  33 use PVE::JSONSchema qw(get_standard_option);
  34 use PVE::ProcFSTools;
  35 use PVE::RPCEnvironment;
  36 use PVE::SafeSyslog;
  37 use PVE::Storage;
  38 use PVE::SysFSTools;
  39 use PVE::Systemd;
  40 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach get_host_arch $IPV6RE);
  41
  42 use PVE::QMPClient;
  43 use PVE::QemuConfig;
  44 use PVE::QemuServer::Cloudinit;
  45 use PVE::QemuServer::Memory;
  46 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr print_pcie_root_port);
  47 use PVE::QemuServer::USB qw(parse_usb_device);
  48
  49 my $EDK2_FW_BASE = '/usr/share/pve-edk2-firmware/';
  50 my $OVMF = {
  51     x86_64 => [
  52         "$EDK2_FW_BASE/OVMF_CODE.fd",
  53         "$EDK2_FW_BASE/OVMF_VARS.fd"
  54     ],
  55     aarch64 => [
  56         "$EDK2_FW_BASE/AAVMF_CODE.fd",
  57         "$EDK2_FW_BASE/AAVMF_VARS.fd"
  58     ],
  59 };
  60
  61 my $qemu_snap_storage = { rbd => 1 };
  62
  63 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  64
  65 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  66
  67 # Note about locking: we use flock on the config file protect
  68 # against concurent actions.
  69 # Aditionaly, we have a 'lock' setting in the config file. This
  70 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  71 # allowed when such lock is set. But you can ignore this kind of
  72 # lock with the --skiplock flag.
  73
  74 cfs_register_file('/qemu-server/',
  75                   \&parse_vm_config,
  76                   \&write_vm_config);
  77
  78 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  79     description => "Some command save/restore state from this location.",
  80     type => 'string',
  81     maxLength => 128,
  82     optional => 1,
  83 });
  84
  85 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  86     type => 'string',
  87     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  88     description => "The drive's backing file's data format.",
  89     optional => 1,
  90 });
  91
  92 PVE::JSONSchema::register_standard_option('pve-qemu-machine', {
  93         description => "Specifies the Qemu machine type.",
  94         type => 'string',
  95         pattern => '(pc|pc(-i440fx)?-\d+(\.\d+)+(\.pxe)?|q35|pc-q35-\d+(\.\d+)+(\.pxe)?|virt(?:-\d+(\.\d+)+)?)',
  96         maxLength => 40,
  97         optional => 1,
  98 });
  99
 100 #no warnings 'redefine';
 101
 102 sub cgroups_write {
 103    my ($controller, $vmid, $option, $value) = @_;
 104
 105    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
 106    PVE::ProcFSTools::write_proc_entry($path, $value);
 107
 108 }
 109
 110 my $nodename = PVE::INotify::nodename();
 111
 112 mkdir "/etc/pve/nodes/$nodename";
 113 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
 114 mkdir $confdir;
 115
 116 my $var_run_tmpdir = "/var/run/qemu-server";
 117 mkdir $var_run_tmpdir;
 118
 119 my $lock_dir = "/var/lock/qemu-server";
 120 mkdir $lock_dir;
 121
 122 my $cpu_vendor_list = {
 123     # Intel CPUs
 124     486 => 'GenuineIntel',
 125     pentium => 'GenuineIntel',
 126     pentium2  => 'GenuineIntel',
 127     pentium3  => 'GenuineIntel',
 128     coreduo => 'GenuineIntel',
 129     core2duo => 'GenuineIntel',
 130     Conroe  => 'GenuineIntel',
 131     Penryn  => 'GenuineIntel',
 132     Nehalem  => 'GenuineIntel',
 133     'Nehalem-IBRS'  => 'GenuineIntel',
 134     Westmere => 'GenuineIntel',
 135     'Westmere-IBRS' => 'GenuineIntel',
 136     SandyBridge => 'GenuineIntel',
 137     'SandyBridge-IBRS' => 'GenuineIntel',
 138     IvyBridge => 'GenuineIntel',
 139     'IvyBridge-IBRS' => 'GenuineIntel',
 140     Haswell => 'GenuineIntel',
 141     'Haswell-IBRS' => 'GenuineIntel',
 142     'Haswell-noTSX' => 'GenuineIntel',
 143     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 144     Broadwell => 'GenuineIntel',
 145     'Broadwell-IBRS' => 'GenuineIntel',
 146     'Broadwell-noTSX' => 'GenuineIntel',
 147     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 148     'Skylake-Client' => 'GenuineIntel',
 149     'Skylake-Client-IBRS' => 'GenuineIntel',
 150     'Skylake-Server' => 'GenuineIntel',
 151     'Skylake-Server-IBRS' => 'GenuineIntel',
 152     'Cascadelake-Server' => 'GenuineIntel',
 153     KnightsMill => 'GenuineIntel',
 154
 155
 156     # AMD CPUs
 157     athlon => 'AuthenticAMD',
 158     phenom  => 'AuthenticAMD',
 159     Opteron_G1  => 'AuthenticAMD',
 160     Opteron_G2  => 'AuthenticAMD',
 161     Opteron_G3  => 'AuthenticAMD',
 162     Opteron_G4  => 'AuthenticAMD',
 163     Opteron_G5  => 'AuthenticAMD',
 164     EPYC => 'AuthenticAMD',
 165     'EPYC-IBPB' => 'AuthenticAMD',
 166
 167     # generic types, use vendor from host node
 168     host => 'default',
 169     kvm32 => 'default',
 170     kvm64 => 'default',
 171     qemu32 => 'default',
 172     qemu64 => 'default',
 173     max => 'default',
 174 };
 175
 176 my @supported_cpu_flags = (
 177     'pcid',
 178     'spec-ctrl',
 179     'ibpb',
 180     'ssbd',
 181     'virt-ssbd',
 182     'amd-ssbd',
 183     'amd-no-ssb',
 184     'pdpe1gb',
 185     'md-clear',
 186     'hv-tlbflush',
 187     'hv-evmcs',
 188     'aes'
 189 );
 190 my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
 191
 192 my $cpu_fmt = {
 193     cputype => {
 194         description => "Emulated CPU type.",
 195         type => 'string',
 196         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 197         default => 'kvm64',
 198         default_key => 1,
 199     },
 200     hidden => {
 201         description => "Do not identify as a KVM virtual machine.",
 202         type => 'boolean',
 203         optional => 1,
 204         default => 0
 205     },
 206     'hv-vendor-id' => {
 207         type => 'string',
 208         pattern => qr/[a-zA-Z0-9]{1,12}/,
 209         format_description => 'vendor-id',
 210         description => 'The Hyper-V vendor ID. Some drivers or programs inside Windows guests need a specific ID.',
 211         optional => 1,
 212     },
 213     flags => {
 214         description => "List of additional CPU flags separated by ';'."
 215                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 216                      . " Currently supported flags: @{[join(', ', @supported_cpu_flags)]}.",
 217         format_description => '+FLAG[;-FLAG...]',
 218         type => 'string',
 219         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 220         optional => 1,
 221     },
 222 };
 223
 224 my $watchdog_fmt = {
 225     model => {
 226         default_key => 1,
 227         type => 'string',
 228         enum => [qw(i6300esb ib700)],
 229         description => "Watchdog type to emulate.",
 230         default => 'i6300esb',
 231         optional => 1,
 232     },
 233     action => {
 234         type => 'string',
 235         enum => [qw(reset shutdown poweroff pause debug none)],
 236         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 237         optional => 1,
 238     },
 239 };
 240 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 241
 242 my $agent_fmt = {
 243     enabled => {
 244         description => "Enable/disable Qemu GuestAgent.",
 245         type => 'boolean',
 246         default => 0,
 247         default_key => 1,
 248     },
 249     fstrim_cloned_disks => {
 250         description => "Run fstrim after cloning/moving a disk.",
 251         type => 'boolean',
 252         optional => 1,
 253         default => 0
 254     },
 255 };
 256
 257 my $vga_fmt = {
 258     type => {
 259         description => "Select the VGA type.",
 260         type => 'string',
 261         default => 'std',
 262         optional => 1,
 263         default_key => 1,
 264         enum => [qw(cirrus qxl qxl2 qxl3 qxl4 none serial0 serial1 serial2 serial3 std virtio vmware)],
 265     },
 266     memory => {
 267         description => "Sets the VGA memory (in MiB). Has no effect with serial display.",
 268         type => 'integer',
 269         optional => 1,
 270         minimum => 4,
 271         maximum => 512,
 272     },
 273 };
 274
 275 my $ivshmem_fmt = {
 276     size => {
 277         type => 'integer',
 278         minimum => 1,
 279         description => "The size of the file in MB.",
 280     },
 281     name => {
 282         type => 'string',
 283         pattern => '[a-zA-Z0-9\-]+',
 284         optional => 1,
 285         format_description => 'string',
 286         description => "The name of the file. Will be prefixed with 'pve-shm-'. Default is the VMID. Will be deleted when the VM is stopped.",
 287     },
 288 };
 289
 290 my $audio_fmt = {
 291     device => {
 292         type => 'string',
 293         enum => [qw(ich9-intel-hda intel-hda AC97)],
 294         description =>  "Configure an audio device."
 295     },
 296     driver =>  {
 297         type => 'string',
 298         enum => ['spice'],
 299         default => 'spice',
 300         optional => 1,
 301         description => "Driver backend for the audio device."
 302     },
 303 };
 304
 305 my $spice_enhancements_fmt = {
 306     foldersharing => {
 307         type => 'boolean',
 308         optional => 1,
 309         default => '0',
 310         description =>  "Enable folder sharing via SPICE. Needs Spice-WebDAV daemon installed in the VM."
 311     },
 312     videostreaming =>  {
 313         type => 'string',
 314         enum => ['off', 'all', 'filter'],
 315         default => 'off',
 316         optional => 1,
 317         description => "Enable video streaming. Uses compression for detected video streams."
 318     },
 319 };
 320
 321 my $confdesc = {
 322     onboot => {
 323         optional => 1,
 324         type => 'boolean',
 325         description => "Specifies whether a VM will be started during system bootup.",
 326         default => 0,
 327     },
 328     autostart => {
 329         optional => 1,
 330         type => 'boolean',
 331         description => "Automatic restart after crash (currently ignored).",
 332         default => 0,
 333     },
 334     hotplug => {
 335         optional => 1,
 336         type => 'string', format => 'pve-hotplug-features',
 337         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 338         default => 'network,disk,usb',
 339     },
 340     reboot => {
 341         optional => 1,
 342         type => 'boolean',
 343         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 344         default => 1,
 345     },
 346     lock => {
 347         optional => 1,
 348         type => 'string',
 349         description => "Lock/unlock the VM.",
 350         enum => [qw(backup clone create migrate rollback snapshot snapshot-delete suspending suspended)],
 351     },
 352     cpulimit => {
 353         optional => 1,
 354         type => 'number',
 355         description => "Limit of CPU usage.",
 356         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 357         minimum => 0,
 358         maximum => 128,
 359         default => 0,
 360     },
 361     cpuunits => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "CPU weight for a VM.",
 365         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 366         minimum => 2,
 367         maximum => 262144,
 368         default => 1024,
 369     },
 370     memory => {
 371         optional => 1,
 372         type => 'integer',
 373         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 374         minimum => 16,
 375         default => 512,
 376     },
 377     balloon => {
 378         optional => 1,
 379         type => 'integer',
 380         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 381         minimum => 0,
 382     },
 383     shares => {
 384         optional => 1,
 385         type => 'integer',
 386         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning. Auto-ballooning is done by pvestatd.",
 387         minimum => 0,
 388         maximum => 50000,
 389         default => 1000,
 390     },
 391     keyboard => {
 392         optional => 1,
 393         type => 'string',
 394         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.cfg' configuration file.".
 395                        "It should not be necessary to set it.",
 396         enum => PVE::Tools::kvmkeymaplist(),
 397         default => undef,
 398     },
 399     name => {
 400         optional => 1,
 401         type => 'string', format => 'dns-name',
 402         description => "Set a name for the VM. Only used on the configuration web interface.",
 403     },
 404     scsihw => {
 405         optional => 1,
 406         type => 'string',
 407         description => "SCSI controller model",
 408         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 409         default => 'lsi',
 410     },
 411     description => {
 412         optional => 1,
 413         type => 'string',
 414         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 415     },
 416     ostype => {
 417         optional => 1,
 418         type => 'string',
 419         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 420         description => "Specify guest operating system.",
 421         verbose_description => <<EODESC,
 422 Specify guest operating system. This is used to enable special
 423 optimization/features for specific operating systems:
 424
 425 [horizontal]
 426 other;; unspecified OS
 427 wxp;; Microsoft Windows XP
 428 w2k;; Microsoft Windows 2000
 429 w2k3;; Microsoft Windows 2003
 430 w2k8;; Microsoft Windows 2008
 431 wvista;; Microsoft Windows Vista
 432 win7;; Microsoft Windows 7
 433 win8;; Microsoft Windows 8/2012/2012r2
 434 win10;; Microsoft Windows 10/2016
 435 l24;; Linux 2.4 Kernel
 436 l26;; Linux 2.6/3.X Kernel
 437 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 438 EODESC
 439     },
 440     boot => {
 441         optional => 1,
 442         type => 'string',
 443         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 444         pattern => '[acdn]{1,4}',
 445         default => 'cdn',
 446     },
 447     bootdisk => {
 448         optional => 1,
 449         type => 'string', format => 'pve-qm-bootdisk',
 450         description => "Enable booting from specified disk.",
 451         pattern => '(ide|sata|scsi|virtio)\d+',
 452     },
 453     smp => {
 454         optional => 1,
 455         type => 'integer',
 456         description => "The number of CPUs. Please use option -sockets instead.",
 457         minimum => 1,
 458         default => 1,
 459     },
 460     sockets => {
 461         optional => 1,
 462         type => 'integer',
 463         description => "The number of CPU sockets.",
 464         minimum => 1,
 465         default => 1,
 466     },
 467     cores => {
 468         optional => 1,
 469         type => 'integer',
 470         description => "The number of cores per socket.",
 471         minimum => 1,
 472         default => 1,
 473     },
 474     numa => {
 475         optional => 1,
 476         type => 'boolean',
 477         description => "Enable/disable NUMA.",
 478         default => 0,
 479     },
 480     hugepages => {
 481         optional => 1,
 482         type => 'string',
 483         description => "Enable/disable hugepages memory.",
 484         enum => [qw(any 2 1024)],
 485     },
 486     vcpus => {
 487         optional => 1,
 488         type => 'integer',
 489         description => "Number of hotplugged vcpus.",
 490         minimum => 1,
 491         default => 0,
 492     },
 493     acpi => {
 494         optional => 1,
 495         type => 'boolean',
 496         description => "Enable/disable ACPI.",
 497         default => 1,
 498     },
 499     agent => {
 500         optional => 1,
 501         description => "Enable/disable Qemu GuestAgent and its properties.",
 502         type => 'string',
 503         format => $agent_fmt,
 504     },
 505     kvm => {
 506         optional => 1,
 507         type => 'boolean',
 508         description => "Enable/disable KVM hardware virtualization.",
 509         default => 1,
 510     },
 511     tdf => {
 512         optional => 1,
 513         type => 'boolean',
 514         description => "Enable/disable time drift fix.",
 515         default => 0,
 516     },
 517     localtime => {
 518         optional => 1,
 519         type => 'boolean',
 520         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 521     },
 522     freeze => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 526     },
 527     vga => {
 528         optional => 1,
 529         type => 'string', format => $vga_fmt,
 530         description => "Configure the VGA hardware.",
 531         verbose_description => "Configure the VGA Hardware. If you want to use ".
 532             "high resolution modes (>= 1280x1024x16) you may need to increase " .
 533             "the vga memory option. Since QEMU 2.9 the default VGA display type " .
 534             "is 'std' for all OS types besides some Windows versions (XP and " .
 535             "older) which use 'cirrus'. The 'qxl' option enables the SPICE " .
 536             "display server. For win* OS you can select how many independent " .
 537             "displays you want, Linux guests can add displays them self.\n".
 538             "You can also run without any graphic card, using a serial device as terminal.",
 539     },
 540     watchdog => {
 541         optional => 1,
 542         type => 'string', format => 'pve-qm-watchdog',
 543         description => "Create a virtual hardware watchdog device.",
 544         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 545             " (by a guest action), the watchdog must be periodically polled " .
 546             "by an agent inside the guest or else the watchdog will reset " .
 547             "the guest (or execute the respective action specified)",
 548     },
 549     startdate => {
 550         optional => 1,
 551         type => 'string',
 552         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 553         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 554         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 555         default => 'now',
 556     },
 557     startup =>  get_standard_option('pve-startup-order'),
 558     template => {
 559         optional => 1,
 560         type => 'boolean',
 561         description => "Enable/disable Template.",
 562         default => 0,
 563     },
 564     args => {
 565         optional => 1,
 566         type => 'string',
 567         description => "Arbitrary arguments passed to kvm.",
 568         verbose_description => <<EODESCR,
 569 Arbitrary arguments passed to kvm, for example:
 570
 571 args: -no-reboot -no-hpet
 572
 573 NOTE: this option is for experts only.
 574 EODESCR
 575     },
 576     tablet => {
 577         optional => 1,
 578         type => 'boolean',
 579         default => 1,
 580         description => "Enable/disable the USB tablet device.",
 581         verbose_description => "Enable/disable the USB tablet device. This device is " .
 582             "usually needed to allow absolute mouse positioning with VNC. " .
 583             "Else the mouse runs out of sync with normal VNC clients. " .
 584             "If you're running lots of console-only guests on one host, " .
 585             "you may consider disabling this to save some context switches. " .
 586             "This is turned off by default if you use spice (-vga=qxl).",
 587     },
 588     migrate_speed => {
 589         optional => 1,
 590         type => 'integer',
 591         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 592         minimum => 0,
 593         default => 0,
 594     },
 595     migrate_downtime => {
 596         optional => 1,
 597         type => 'number',
 598         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 599         minimum => 0,
 600         default => 0.1,
 601     },
 602     cdrom => {
 603         optional => 1,
 604         type => 'string', format => 'pve-qm-ide',
 605         typetext => '<volume>',
 606         description => "This is an alias for option -ide2",
 607     },
 608     cpu => {
 609         optional => 1,
 610         description => "Emulated CPU type.",
 611         type => 'string',
 612         format => $cpu_fmt,
 613     },
 614     parent => get_standard_option('pve-snapshot-name', {
 615         optional => 1,
 616         description => "Parent snapshot name. This is used internally, and should not be modified.",
 617     }),
 618     snaptime => {
 619         optional => 1,
 620         description => "Timestamp for snapshots.",
 621         type => 'integer',
 622         minimum => 0,
 623     },
 624     vmstate => {
 625         optional => 1,
 626         type => 'string', format => 'pve-volume-id',
 627         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 628     },
 629     vmstatestorage => get_standard_option('pve-storage-id', {
 630         description => "Default storage for VM state volumes/files.",
 631         optional => 1,
 632     }),
 633     runningmachine => get_standard_option('pve-qemu-machine', {
 634         description => "Specifies the Qemu machine type of the running vm. This is used internally for snapshots.",
 635     }),
 636     machine => get_standard_option('pve-qemu-machine'),
 637     arch => {
 638         description => "Virtual processor architecture. Defaults to the host.",
 639         optional => 1,
 640         type => 'string',
 641         enum => [qw(x86_64 aarch64)],
 642     },
 643     smbios1 => {
 644         description => "Specify SMBIOS type 1 fields.",
 645         type => 'string', format => 'pve-qm-smbios1',
 646         maxLength => 512,
 647         optional => 1,
 648     },
 649     protection => {
 650         optional => 1,
 651         type => 'boolean',
 652         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 653         default => 0,
 654     },
 655     bios => {
 656         optional => 1,
 657         type => 'string',
 658         enum => [ qw(seabios ovmf) ],
 659         description => "Select BIOS implementation.",
 660         default => 'seabios',
 661     },
 662     vmgenid => {
 663         type => 'string',
 664         pattern => '(?:[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}|[01])',
 665         format_description => 'UUID',
 666         description => "Set VM Generation ID. Use '1' to autogenerate on create or update, pass '0' to disable explicitly.",
 667         verbose_description => "The VM generation ID (vmgenid) device exposes a".
 668             " 128-bit integer value identifier to the guest OS. This allows to".
 669             " notify the guest operating system when the virtual machine is".
 670             " executed with a different configuration (e.g. snapshot execution".
 671             " or creation from a template). The guest operating system notices".
 672             " the change, and is then able to react as appropriate by marking".
 673             " its copies of distributed databases as dirty, re-initializing its".
 674             " random number generator, etc.\n".
 675             "Note that auto-creation only works when done throug API/CLI create".
 676             " or update methods, but not when manually editing the config file.",
 677         default => "1 (autogenerated)",
 678         optional => 1,
 679     },
 680     hookscript => {
 681         type => 'string',
 682         format => 'pve-volume-id',
 683         optional => 1,
 684         description => "Script that will be executed during various steps in the vms lifetime.",
 685     },
 686     ivshmem => {
 687         type => 'string',
 688         format => $ivshmem_fmt,
 689         description => "Inter-VM shared memory. Useful for direct communication between VMs, or to the host.",
 690         optional => 1,
 691     },
 692     audio0 => {
 693         type => 'string',
 694         format => $audio_fmt,
 695         description => "Configure a audio device, useful in combination with QXL/Spice.",
 696         optional => 1
 697     },
 698     spice_enhancements => {
 699         type => 'string',
 700         format => $spice_enhancements_fmt,
 701         description => "Configure additional enhancements for SPICE.",
 702         optional => 1
 703     },
 704 };
 705
 706 my $cicustom_fmt = {
 707     meta => {
 708         type => 'string',
 709         optional => 1,
 710         description => 'Specify a custom file containing all meta data passed to the VM via cloud-init. This is provider specific meaning configdrive2 and nocloud differ.',
 711         format => 'pve-volume-id',
 712         format_description => 'volume',
 713     },
 714     network => {
 715         type => 'string',
 716         optional => 1,
 717         description => 'Specify a custom file containing all network data passed to the VM via cloud-init.',
 718         format => 'pve-volume-id',
 719         format_description => 'volume',
 720     },
 721     user => {
 722         type => 'string',
 723         optional => 1,
 724         description => 'Specify a custom file containing all user data passed to the VM via cloud-init.',
 725         format => 'pve-volume-id',
 726         format_description => 'volume',
 727     },
 728 };
 729 PVE::JSONSchema::register_format('pve-qm-cicustom', $cicustom_fmt);
 730
 731 my $confdesc_cloudinit = {
 732     citype => {
 733         optional => 1,
 734         type => 'string',
 735         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 736         enum => ['configdrive2', 'nocloud'],
 737     },
 738     ciuser => {
 739         optional => 1,
 740         type => 'string',
 741         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 742     },
 743     cipassword => {
 744         optional => 1,
 745         type => 'string',
 746         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 747     },
 748     cicustom => {
 749         optional => 1,
 750         type => 'string',
 751         description => 'cloud-init: Specify custom files to replace the automatically generated ones at start.',
 752         format => 'pve-qm-cicustom',
 753     },
 754     searchdomain => {
 755         optional => 1,
 756         type => 'string',
 757         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 758     },
 759     nameserver => {
 760         optional => 1,
 761         type => 'string', format => 'address-list',
 762         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 763     },
 764     sshkeys => {
 765         optional => 1,
 766         type => 'string',
 767         format => 'urlencoded',
 768         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 769     },
 770 };
 771
 772 # what about other qemu settings ?
 773 #cpu => 'string',
 774 #machine => 'string',
 775 #fda => 'file',
 776 #fdb => 'file',
 777 #mtdblock => 'file',
 778 #sd => 'file',
 779 #pflash => 'file',
 780 #snapshot => 'bool',
 781 #bootp => 'file',
 782 ##tftp => 'dir',
 783 ##smb => 'dir',
 784 #kernel => 'file',
 785 #append => 'string',
 786 #initrd => 'file',
 787 ##soundhw => 'string',
 788
 789 while (my ($k, $v) = each %$confdesc) {
 790     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 791 }
 792
 793 my $MAX_IDE_DISKS = 4;
 794 my $MAX_SCSI_DISKS = 14;
 795 my $MAX_VIRTIO_DISKS = 16;
 796 my $MAX_SATA_DISKS = 6;
 797 my $MAX_USB_DEVICES = 5;
 798 my $MAX_NETS = 32;
 799 my $MAX_UNUSED_DISKS = 256;
 800 my $MAX_HOSTPCI_DEVICES = 16;
 801 my $MAX_SERIAL_PORTS = 4;
 802 my $MAX_PARALLEL_PORTS = 3;
 803 my $MAX_NUMA = 8;
 804
 805 my $numa_fmt = {
 806     cpus => {
 807         type => "string",
 808         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 809         description => "CPUs accessing this NUMA node.",
 810         format_description => "id[-id];...",
 811     },
 812     memory => {
 813         type => "number",
 814         description => "Amount of memory this NUMA node provides.",
 815         optional => 1,
 816     },
 817     hostnodes => {
 818         type => "string",
 819         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 820         description => "Host NUMA nodes to use.",
 821         format_description => "id[-id];...",
 822         optional => 1,
 823     },
 824     policy => {
 825         type => 'string',
 826         enum => [qw(preferred bind interleave)],
 827         description => "NUMA allocation policy.",
 828         optional => 1,
 829     },
 830 };
 831 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 832 my $numadesc = {
 833     optional => 1,
 834     type => 'string', format => $numa_fmt,
 835     description => "NUMA topology.",
 836 };
 837 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 838
 839 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 840     $confdesc->{"numa$i"} = $numadesc;
 841 }
 842
 843 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 844                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 845                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 846 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 847
 848 my $net_fmt_bridge_descr = <<__EOD__;
 849 Bridge to attach the network device to. The Proxmox VE standard bridge
 850 is called 'vmbr0'.
 851
 852 If you do not specify a bridge, we create a kvm user (NATed) network
 853 device, which provides DHCP and DNS services. The following addresses
 854 are used:
 855
 856  10.0.2.2   Gateway
 857  10.0.2.3   DNS Server
 858  10.0.2.4   SMB Server
 859
 860 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 861 __EOD__
 862
 863 my $net_fmt = {
 864     macaddr  => get_standard_option('mac-addr', {
 865         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 866     }),
 867     model => {
 868         type => 'string',
 869         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 870         enum => $nic_model_list,
 871         default_key => 1,
 872     },
 873     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 874     bridge => {
 875         type => 'string',
 876         description => $net_fmt_bridge_descr,
 877         format_description => 'bridge',
 878         optional => 1,
 879     },
 880     queues => {
 881         type => 'integer',
 882         minimum => 0, maximum => 16,
 883         description => 'Number of packet queues to be used on the device.',
 884         optional => 1,
 885     },
 886     rate => {
 887         type => 'number',
 888         minimum => 0,
 889         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 890         optional => 1,
 891     },
 892     tag => {
 893         type => 'integer',
 894         minimum => 1, maximum => 4094,
 895         description => 'VLAN tag to apply to packets on this interface.',
 896         optional => 1,
 897     },
 898     trunks => {
 899         type => 'string',
 900         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 901         description => 'VLAN trunks to pass through this interface.',
 902         format_description => 'vlanid[;vlanid...]',
 903         optional => 1,
 904     },
 905     firewall => {
 906         type => 'boolean',
 907         description => 'Whether this interface should be protected by the firewall.',
 908         optional => 1,
 909     },
 910     link_down => {
 911         type => 'boolean',
 912         description => 'Whether this interface should be disconnected (like pulling the plug).',
 913         optional => 1,
 914     },
 915 };
 916
 917 my $netdesc = {
 918     optional => 1,
 919     type => 'string', format => $net_fmt,
 920     description => "Specify network devices.",
 921 };
 922
 923 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 924
 925 my $ipconfig_fmt = {
 926     ip => {
 927         type => 'string',
 928         format => 'pve-ipv4-config',
 929         format_description => 'IPv4Format/CIDR',
 930         description => 'IPv4 address in CIDR format.',
 931         optional => 1,
 932         default => 'dhcp',
 933     },
 934     gw => {
 935         type => 'string',
 936         format => 'ipv4',
 937         format_description => 'GatewayIPv4',
 938         description => 'Default gateway for IPv4 traffic.',
 939         optional => 1,
 940         requires => 'ip',
 941     },
 942     ip6 => {
 943         type => 'string',
 944         format => 'pve-ipv6-config',
 945         format_description => 'IPv6Format/CIDR',
 946         description => 'IPv6 address in CIDR format.',
 947         optional => 1,
 948         default => 'dhcp',
 949     },
 950     gw6 => {
 951         type => 'string',
 952         format => 'ipv6',
 953         format_description => 'GatewayIPv6',
 954         description => 'Default gateway for IPv6 traffic.',
 955         optional => 1,
 956         requires => 'ip6',
 957     },
 958 };
 959 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 960 my $ipconfigdesc = {
 961     optional => 1,
 962     type => 'string', format => 'pve-qm-ipconfig',
 963     description => <<'EODESCR',
 964 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 965
 966 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 967
 968 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 969 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 970
 971 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 972 EODESCR
 973 };
 974 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 975
 976 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 977     $confdesc->{"net$i"} = $netdesc;
 978     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 979 }
 980
 981 foreach my $key (keys %$confdesc_cloudinit) {
 982     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 983 }
 984
 985 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 986 sub verify_volume_id_or_qm_path {
 987     my ($volid, $noerr) = @_;
 988
 989     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 990         return $volid;
 991     }
 992
 993     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 994     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 995     if ($@) {
 996         return undef if $noerr;
 997         die $@;
 998     }
 999     return $volid;
1000 }
1001
1002 my $drivename_hash;
1003
1004 my %drivedesc_base = (
1005     volume => { alias => 'file' },
1006     file => {
1007         type => 'string',
1008         format => 'pve-volume-id-or-qm-path',
1009         default_key => 1,
1010         format_description => 'volume',
1011         description => "The drive's backing volume.",
1012     },
1013     media => {
1014         type => 'string',
1015         enum => [qw(cdrom disk)],
1016         description => "The drive's media type.",
1017         default => 'disk',
1018         optional => 1
1019     },
1020     cyls => {
1021         type => 'integer',
1022         description => "Force the drive's physical geometry to have a specific cylinder count.",
1023         optional => 1
1024     },
1025     heads => {
1026         type => 'integer',
1027         description => "Force the drive's physical geometry to have a specific head count.",
1028         optional => 1
1029     },
1030     secs => {
1031         type => 'integer',
1032         description => "Force the drive's physical geometry to have a specific sector count.",
1033         optional => 1
1034     },
1035     trans => {
1036         type => 'string',
1037         enum => [qw(none lba auto)],
1038         description => "Force disk geometry bios translation mode.",
1039         optional => 1,
1040     },
1041     snapshot => {
1042         type => 'boolean',
1043         description => "Controls qemu's snapshot mode feature."
1044             . " If activated, changes made to the disk are temporary and will"
1045             . " be discarded when the VM is shutdown.",
1046         optional => 1,
1047     },
1048     cache => {
1049         type => 'string',
1050         enum => [qw(none writethrough writeback unsafe directsync)],
1051         description => "The drive's cache mode",
1052         optional => 1,
1053     },
1054     format => get_standard_option('pve-qm-image-format'),
1055     size => {
1056         type => 'string',
1057         format => 'disk-size',
1058         format_description => 'DiskSize',
1059         description => "Disk size. This is purely informational and has no effect.",
1060         optional => 1,
1061     },
1062     backup => {
1063         type => 'boolean',
1064         description => "Whether the drive should be included when making backups.",
1065         optional => 1,
1066     },
1067     replicate => {
1068         type => 'boolean',
1069         description => 'Whether the drive should considered for replication jobs.',
1070         optional => 1,
1071         default => 1,
1072     },
1073     rerror => {
1074         type => 'string',
1075         enum => [qw(ignore report stop)],
1076         description => 'Read error action.',
1077         optional => 1,
1078     },
1079     werror => {
1080         type => 'string',
1081         enum => [qw(enospc ignore report stop)],
1082         description => 'Write error action.',
1083         optional => 1,
1084     },
1085     aio => {
1086         type => 'string',
1087         enum => [qw(native threads)],
1088         description => 'AIO type to use.',
1089         optional => 1,
1090     },
1091     discard => {
1092         type => 'string',
1093         enum => [qw(ignore on)],
1094         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
1095         optional => 1,
1096     },
1097     detect_zeroes => {
1098         type => 'boolean',
1099         description => 'Controls whether to detect and try to optimize writes of zeroes.',
1100         optional => 1,
1101     },
1102     serial => {
1103         type => 'string',
1104         format => 'urlencoded',
1105         format_description => 'serial',
1106         maxLength => 20*3, # *3 since it's %xx url enoded
1107         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
1108         optional => 1,
1109     },
1110     shared => {
1111         type => 'boolean',
1112         description => 'Mark this locally-managed volume as available on all nodes',
1113         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
1114         optional => 1,
1115         default => 0,
1116     }
1117 );
1118
1119 my %iothread_fmt = ( iothread => {
1120         type => 'boolean',
1121         description => "Whether to use iothreads for this drive",
1122         optional => 1,
1123 });
1124
1125 my %model_fmt = (
1126     model => {
1127         type => 'string',
1128         format => 'urlencoded',
1129         format_description => 'model',
1130         maxLength => 40*3, # *3 since it's %xx url enoded
1131         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
1132         optional => 1,
1133     },
1134 );
1135
1136 my %queues_fmt = (
1137     queues => {
1138         type => 'integer',
1139         description => "Number of queues.",
1140         minimum => 2,
1141         optional => 1
1142     }
1143 );
1144
1145 my %scsiblock_fmt = (
1146     scsiblock => {
1147         type => 'boolean',
1148         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
1149         optional => 1,
1150         default => 0,
1151     },
1152 );
1153
1154 my %ssd_fmt = (
1155     ssd => {
1156         type => 'boolean',
1157         description => "Whether to expose this drive as an SSD, rather than a rotational hard disk.",
1158         optional => 1,
1159     },
1160 );
1161
1162 my %wwn_fmt = (
1163     wwn => {
1164         type => 'string',
1165         pattern => qr/^(0x)[0-9a-fA-F]{16}/,
1166         format_description => 'wwn',
1167         description => "The drive's worldwide name, encoded as 16 bytes hex string, prefixed by '0x'.",
1168         optional => 1,
1169     },
1170 );
1171
1172 my $add_throttle_desc = sub {
1173     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
1174     my $d = {
1175         type => $type,
1176         format_description => $unit,
1177         description => "Maximum $what in $longunit.",
1178         optional => 1,
1179     };
1180     $d->{minimum} = $minimum if defined($minimum);
1181     $drivedesc_base{$key} = $d;
1182 };
1183 # throughput: (leaky bucket)
1184 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
1185 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
1186 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
1187 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
1188 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
1189 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
1190 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
1191 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
1192 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
1193
1194 # pools: (pool of IO before throttling starts taking effect)
1195 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
1196 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
1197 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
1198 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
1199 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
1200 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
1201
1202 # burst lengths
1203 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1204 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1205 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1206 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
1207 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
1208 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1209
1210 # legacy support
1211 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1212 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1213 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1214 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1215
1216 my $ide_fmt = {
1217     %drivedesc_base,
1218     %model_fmt,
1219     %ssd_fmt,
1220     %wwn_fmt,
1221 };
1222 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1223
1224 my $idedesc = {
1225     optional => 1,
1226     type => 'string', format => $ide_fmt,
1227     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1228 };
1229 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1230
1231 my $scsi_fmt = {
1232     %drivedesc_base,
1233     %iothread_fmt,
1234     %queues_fmt,
1235     %scsiblock_fmt,
1236     %ssd_fmt,
1237     %wwn_fmt,
1238 };
1239 my $scsidesc = {
1240     optional => 1,
1241     type => 'string', format => $scsi_fmt,
1242     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1243 };
1244 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1245
1246 my $sata_fmt = {
1247     %drivedesc_base,
1248     %ssd_fmt,
1249     %wwn_fmt,
1250 };
1251 my $satadesc = {
1252     optional => 1,
1253     type => 'string', format => $sata_fmt,
1254     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1255 };
1256 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1257
1258 my $virtio_fmt = {
1259     %drivedesc_base,
1260     %iothread_fmt,
1261 };
1262 my $virtiodesc = {
1263     optional => 1,
1264     type => 'string', format => $virtio_fmt,
1265     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1266 };
1267 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1268
1269 my $alldrive_fmt = {
1270     %drivedesc_base,
1271     %iothread_fmt,
1272     %model_fmt,
1273     %queues_fmt,
1274     %scsiblock_fmt,
1275     %ssd_fmt,
1276     %wwn_fmt,
1277 };
1278
1279 my $efidisk_fmt = {
1280     volume => { alias => 'file' },
1281     file => {
1282         type => 'string',
1283         format => 'pve-volume-id-or-qm-path',
1284         default_key => 1,
1285         format_description => 'volume',
1286         description => "The drive's backing volume.",
1287     },
1288     format => get_standard_option('pve-qm-image-format'),
1289     size => {
1290         type => 'string',
1291         format => 'disk-size',
1292         format_description => 'DiskSize',
1293         description => "Disk size. This is purely informational and has no effect.",
1294         optional => 1,
1295     },
1296 };
1297
1298 my $efidisk_desc = {
1299     optional => 1,
1300     type => 'string', format => $efidisk_fmt,
1301     description => "Configure a Disk for storing EFI vars",
1302 };
1303
1304 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1305
1306 my $usb_fmt = {
1307     host => {
1308         default_key => 1,
1309         type => 'string', format => 'pve-qm-usb-device',
1310         format_description => 'HOSTUSBDEVICE|spice',
1311         description => <<EODESCR,
1312 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1313
1314  'bus-port(.port)*' (decimal numbers) or
1315  'vendor_id:product_id' (hexadeciaml numbers) or
1316  'spice'
1317
1318 You can use the 'lsusb -t' command to list existing usb devices.
1319
1320 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1321
1322 The value 'spice' can be used to add a usb redirection devices for spice.
1323 EODESCR
1324     },
1325     usb3 => {
1326         optional => 1,
1327         type => 'boolean',
1328         description => "Specifies whether if given host option is a USB3 device or port.",
1329         default => 0,
1330     },
1331 };
1332
1333 my $usbdesc = {
1334     optional => 1,
1335     type => 'string', format => $usb_fmt,
1336     description => "Configure an USB device (n is 0 to 4).",
1337 };
1338 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1339
1340 my $PCIRE = qr/[a-f0-9]{2}:[a-f0-9]{2}(?:\.[a-f0-9])?/;
1341 my $hostpci_fmt = {
1342     host => {
1343         default_key => 1,
1344         type => 'string',
1345         pattern => qr/$PCIRE(;$PCIRE)*/,
1346         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1347         description => <<EODESCR,
1348 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1349 of PCI virtual functions of the host. HOSTPCIID syntax is:
1350
1351 'bus:dev.func' (hexadecimal numbers)
1352
1353 You can us the 'lspci' command to list existing PCI devices.
1354 EODESCR
1355     },
1356     rombar => {
1357         type => 'boolean',
1358         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1359         optional => 1,
1360         default => 1,
1361     },
1362     romfile => {
1363         type => 'string',
1364         pattern => '[^,;]+',
1365         format_description => 'string',
1366         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1367         optional => 1,
1368     },
1369     pcie => {
1370         type => 'boolean',
1371         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1372         optional => 1,
1373         default => 0,
1374     },
1375     'x-vga' => {
1376         type => 'boolean',
1377         description =>  "Enable vfio-vga device support.",
1378         optional => 1,
1379         default => 0,
1380     },
1381     'mdev' => {
1382         type => 'string',
1383         format_description => 'string',
1384         pattern => '[^/\.:]+',
1385         optional => 1,
1386         description => <<EODESCR
1387 The type of mediated device to use.
1388 An instance of this type will be created on startup of the VM and
1389 will be cleaned up when the VM stops.
1390 EODESCR
1391     }
1392 };
1393 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1394
1395 my $hostpcidesc = {
1396         optional => 1,
1397         type => 'string', format => 'pve-qm-hostpci',
1398         description => "Map host PCI devices into guest.",
1399         verbose_description =>  <<EODESCR,
1400 Map host PCI devices into guest.
1401
1402 NOTE: This option allows direct access to host hardware. So it is no longer
1403 possible to migrate such machines - use with special care.
1404
1405 CAUTION: Experimental! User reported problems with this option.
1406 EODESCR
1407 };
1408 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1409
1410 my $serialdesc = {
1411         optional => 1,
1412         type => 'string',
1413         pattern => '(/dev/.+|socket)',
1414         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1415         verbose_description =>  <<EODESCR,
1416 Create a serial device inside the VM (n is 0 to 3), and pass through a
1417 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1418 host side (use 'qm terminal' to open a terminal connection).
1419
1420 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1421
1422 CAUTION: Experimental! User reported problems with this option.
1423 EODESCR
1424 };
1425
1426 my $paralleldesc= {
1427         optional => 1,
1428         type => 'string',
1429         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1430         description =>  "Map host parallel devices (n is 0 to 2).",
1431         verbose_description =>  <<EODESCR,
1432 Map host parallel devices (n is 0 to 2).
1433
1434 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1435
1436 CAUTION: Experimental! User reported problems with this option.
1437 EODESCR
1438 };
1439
1440 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1441     $confdesc->{"parallel$i"} = $paralleldesc;
1442 }
1443
1444 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1445     $confdesc->{"serial$i"} = $serialdesc;
1446 }
1447
1448 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1449     $confdesc->{"hostpci$i"} = $hostpcidesc;
1450 }
1451
1452 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1453     $drivename_hash->{"ide$i"} = 1;
1454     $confdesc->{"ide$i"} = $idedesc;
1455 }
1456
1457 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1458     $drivename_hash->{"sata$i"} = 1;
1459     $confdesc->{"sata$i"} = $satadesc;
1460 }
1461
1462 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1463     $drivename_hash->{"scsi$i"} = 1;
1464     $confdesc->{"scsi$i"} = $scsidesc ;
1465 }
1466
1467 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1468     $drivename_hash->{"virtio$i"} = 1;
1469     $confdesc->{"virtio$i"} = $virtiodesc;
1470 }
1471
1472 $drivename_hash->{efidisk0} = 1;
1473 $confdesc->{efidisk0} = $efidisk_desc;
1474
1475 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1476     $confdesc->{"usb$i"} = $usbdesc;
1477 }
1478
1479 my $unuseddesc = {
1480     optional => 1,
1481     type => 'string', format => 'pve-volume-id',
1482     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1483 };
1484
1485 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1486     $confdesc->{"unused$i"} = $unuseddesc;
1487 }
1488
1489 my $kvm_api_version = 0;
1490
1491 sub kvm_version {
1492     return $kvm_api_version if $kvm_api_version;
1493
1494     open my $fh, '<', '/dev/kvm'
1495         or return undef;
1496
1497     # 0xae00 => KVM_GET_API_VERSION
1498     $kvm_api_version = ioctl($fh, 0xae00, 0);
1499
1500     return $kvm_api_version;
1501 }
1502
1503 my $kvm_user_version = {};
1504 my $kvm_mtime = {};
1505
1506 sub kvm_user_version {
1507     my ($binary) = @_;
1508
1509     $binary //= get_command_for_arch(get_host_arch()); # get the native arch by default
1510     my $st = stat($binary);
1511
1512     my $cachedmtime = $kvm_mtime->{$binary} // -1;
1513     return $kvm_user_version->{$binary} if $kvm_user_version->{$binary} &&
1514         $cachedmtime == $st->mtime;
1515
1516     $kvm_user_version->{$binary} = 'unknown';
1517     $kvm_mtime->{$binary} = $st->mtime;
1518
1519     my $code = sub {
1520         my $line = shift;
1521         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1522             $kvm_user_version->{$binary} = $2;
1523         }
1524     };
1525
1526     eval { run_command([$binary, '--version'], outfunc => $code); };
1527     warn $@ if $@;
1528
1529     return $kvm_user_version->{$binary};
1530
1531 }
1532
1533 sub kernel_has_vhost_net {
1534     return -c '/dev/vhost-net';
1535 }
1536
1537 sub valid_drive_names {
1538     # order is important - used to autoselect boot disk
1539     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1540             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1541             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1542             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1543             'efidisk0');
1544 }
1545
1546 sub is_valid_drivename {
1547     my $dev = shift;
1548
1549     return defined($drivename_hash->{$dev});
1550 }
1551
1552 sub option_exists {
1553     my $key = shift;
1554     return defined($confdesc->{$key});
1555 }
1556
1557 sub nic_models {
1558     return $nic_model_list;
1559 }
1560
1561 sub os_list_description {
1562
1563     return {
1564         other => 'Other',
1565         wxp => 'Windows XP',
1566         w2k => 'Windows 2000',
1567         w2k3 =>, 'Windows 2003',
1568         w2k8 => 'Windows 2008',
1569         wvista => 'Windows Vista',
1570         win7 => 'Windows 7',
1571         win8 => 'Windows 8/2012',
1572         win10 => 'Windows 10/2016',
1573         l24 => 'Linux 2.4',
1574         l26 => 'Linux 2.6',
1575     };
1576 }
1577
1578 my $cdrom_path;
1579
1580 sub get_cdrom_path {
1581
1582     return  $cdrom_path if $cdrom_path;
1583
1584     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1585     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1586     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1587 }
1588
1589 sub get_iso_path {
1590     my ($storecfg, $vmid, $cdrom) = @_;
1591
1592     if ($cdrom eq 'cdrom') {
1593         return get_cdrom_path();
1594     } elsif ($cdrom eq 'none') {
1595         return '';
1596     } elsif ($cdrom =~ m|^/|) {
1597         return $cdrom;
1598     } else {
1599         return PVE::Storage::path($storecfg, $cdrom);
1600     }
1601 }
1602
1603 # try to convert old style file names to volume IDs
1604 sub filename_to_volume_id {
1605     my ($vmid, $file, $media) = @_;
1606
1607      if (!($file eq 'none' || $file eq 'cdrom' ||
1608           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1609
1610         return undef if $file =~ m|/|;
1611
1612         if ($media && $media eq 'cdrom') {
1613             $file = "local:iso/$file";
1614         } else {
1615             $file = "local:$vmid/$file";
1616         }
1617     }
1618
1619     return $file;
1620 }
1621
1622 sub verify_media_type {
1623     my ($opt, $vtype, $media) = @_;
1624
1625     return if !$media;
1626
1627     my $etype;
1628     if ($media eq 'disk') {
1629         $etype = 'images';
1630     } elsif ($media eq 'cdrom') {
1631         $etype = 'iso';
1632     } else {
1633         die "internal error";
1634     }
1635
1636     return if ($vtype eq $etype);
1637
1638     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1639 }
1640
1641 sub cleanup_drive_path {
1642     my ($opt, $storecfg, $drive) = @_;
1643
1644     # try to convert filesystem paths to volume IDs
1645
1646     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1647         ($drive->{file} !~ m|^/dev/.+|) &&
1648         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1649         ($drive->{file} !~ m/^\d+$/)) {
1650         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1651         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1652         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1653         verify_media_type($opt, $vtype, $drive->{media});
1654         $drive->{file} = $volid;
1655     }
1656
1657     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1658 }
1659
1660 sub parse_hotplug_features {
1661     my ($data) = @_;
1662
1663     my $res = {};
1664
1665     return $res if $data eq '0';
1666
1667     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1668
1669     foreach my $feature (PVE::Tools::split_list($data)) {
1670         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1671             $res->{$1} = 1;
1672         } else {
1673             die "invalid hotplug feature '$feature'\n";
1674         }
1675     }
1676     return $res;
1677 }
1678
1679 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1680 sub pve_verify_hotplug_features {
1681     my ($value, $noerr) = @_;
1682
1683     return $value if parse_hotplug_features($value);
1684
1685     return undef if $noerr;
1686
1687     die "unable to parse hotplug option\n";
1688 }
1689
1690 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1691 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1692 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1693 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1694 #        [,iothread=on][,serial=serial][,model=model]
1695
1696 sub parse_drive {
1697     my ($key, $data) = @_;
1698
1699     my ($interface, $index);
1700
1701     if ($key =~ m/^([^\d]+)(\d+)$/) {
1702         $interface = $1;
1703         $index = $2;
1704     } else {
1705         return undef;
1706     }
1707
1708     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1709                                      : $confdesc->{$key}->{format};
1710     if (!$desc) {
1711         warn "invalid drive key: $key\n";
1712         return undef;
1713     }
1714     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1715     return undef if !$res;
1716     $res->{interface} = $interface;
1717     $res->{index} = $index;
1718
1719     my $error = 0;
1720     foreach my $opt (qw(bps bps_rd bps_wr)) {
1721         if (my $bps = defined(delete $res->{$opt})) {
1722             if (defined($res->{"m$opt"})) {
1723                 warn "both $opt and m$opt specified\n";
1724                 ++$error;
1725                 next;
1726             }
1727             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1728         }
1729     }
1730
1731     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1732     for my $requirement (
1733         [mbps_max => 'mbps'],
1734         [mbps_rd_max => 'mbps_rd'],
1735         [mbps_wr_max => 'mbps_wr'],
1736         [miops_max => 'miops'],
1737         [miops_rd_max => 'miops_rd'],
1738         [miops_wr_max => 'miops_wr'],
1739         [bps_max_length => 'mbps_max'],
1740         [bps_rd_max_length => 'mbps_rd_max'],
1741         [bps_wr_max_length => 'mbps_wr_max'],
1742         [iops_max_length => 'iops_max'],
1743         [iops_rd_max_length => 'iops_rd_max'],
1744         [iops_wr_max_length => 'iops_wr_max']) {
1745         my ($option, $requires) = @$requirement;
1746         if ($res->{$option} && !$res->{$requires}) {
1747             warn "$option requires $requires\n";
1748             ++$error;
1749         }
1750     }
1751
1752     return undef if $error;
1753
1754     return undef if $res->{mbps_rd} && $res->{mbps};
1755     return undef if $res->{mbps_wr} && $res->{mbps};
1756     return undef if $res->{iops_rd} && $res->{iops};
1757     return undef if $res->{iops_wr} && $res->{iops};
1758
1759     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1760         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1761         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1762         return undef if $res->{interface} eq 'virtio';
1763     }
1764
1765     if (my $size = $res->{size}) {
1766         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1767     }
1768
1769     return $res;
1770 }
1771
1772 sub print_drive {
1773     my ($vmid, $drive) = @_;
1774     my $data = { %$drive };
1775     delete $data->{$_} for qw(index interface);
1776     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1777 }
1778
1779 sub scsi_inquiry {
1780     my($fh, $noerr) = @_;
1781
1782     my $SG_IO = 0x2285;
1783     my $SG_GET_VERSION_NUM = 0x2282;
1784
1785     my $versionbuf = "\x00" x 8;
1786     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1787     if (!$ret) {
1788         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1789         return undef;
1790     }
1791     my $version = unpack("I", $versionbuf);
1792     if ($version < 30000) {
1793         die "scsi generic interface too old\n"  if !$noerr;
1794         return undef;
1795     }
1796
1797     my $buf = "\x00" x 36;
1798     my $sensebuf = "\x00" x 8;
1799     my $cmd = pack("C x3 C x1", 0x12, 36);
1800
1801     # see /usr/include/scsi/sg.h
1802     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1803
1804     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1805                       length($sensebuf), 0, length($buf), $buf,
1806                       $cmd, $sensebuf, 6000);
1807
1808     $ret = ioctl($fh, $SG_IO, $packet);
1809     if (!$ret) {
1810         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1811         return undef;
1812     }
1813
1814     my @res = unpack($sg_io_hdr_t, $packet);
1815     if ($res[17] || $res[18]) {
1816         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1817         return undef;
1818     }
1819
1820     my $res = {};
1821     (my $byte0, my $byte1, $res->{vendor},
1822      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1823
1824     $res->{removable} = $byte1 & 128 ? 1 : 0;
1825     $res->{type} = $byte0 & 31;
1826
1827     return $res;
1828 }
1829
1830 sub path_is_scsi {
1831     my ($path) = @_;
1832
1833     my $fh = IO::File->new("+<$path") || return undef;
1834     my $res = scsi_inquiry($fh, 1);
1835     close($fh);
1836
1837     return $res;
1838 }
1839
1840 sub machine_type_is_q35 {
1841     my ($conf) = @_;
1842
1843     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1844 }
1845
1846 sub print_tabletdevice_full {
1847     my ($conf, $arch) = @_;
1848
1849     my $q35 = machine_type_is_q35($conf);
1850
1851     # we use uhci for old VMs because tablet driver was buggy in older qemu
1852     my $usbbus;
1853     if (machine_type_is_q35($conf) || $arch eq 'aarch64') {
1854         $usbbus = 'ehci';
1855     } else {
1856         $usbbus = 'uhci';
1857     }
1858
1859     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1860 }
1861
1862 sub print_keyboarddevice_full {
1863     my ($conf, $arch, $machine) = @_;
1864
1865     return undef if $arch ne 'aarch64';
1866
1867     return "usb-kbd,id=keyboard,bus=ehci.0,port=2";
1868 }
1869
1870 sub print_drivedevice_full {
1871     my ($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type) = @_;
1872
1873     my $device = '';
1874     my $maxdev = 0;
1875
1876     if ($drive->{interface} eq 'virtio') {
1877         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges, $arch, $machine_type);
1878         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1879         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1880     } elsif ($drive->{interface} eq 'scsi') {
1881
1882         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1883         my $unit = $drive->{index} % $maxdev;
1884         my $devicetype = 'hd';
1885         my $path = '';
1886         if (drive_is_cdrom($drive)) {
1887             $devicetype = 'cd';
1888         } else {
1889             if ($drive->{file} =~ m|^/|) {
1890                 $path = $drive->{file};
1891                 if (my $info = path_is_scsi($path)) {
1892                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1893                         $devicetype = 'block';
1894                     } elsif ($info->{type} == 1) { # tape
1895                         $devicetype = 'generic';
1896                     }
1897                 }
1898             } else {
1899                  $path = PVE::Storage::path($storecfg, $drive->{file});
1900             }
1901
1902             # for compatibility only, we prefer scsi-hd (#2408, #2355, #2380)
1903             if ($path =~ m/^iscsi\:\/\// &&
1904                !qemu_machine_feature_enabled($machine_type, undef, 4, 1)) {
1905                 $devicetype = 'generic';
1906             }
1907         }
1908
1909         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1910             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1911         } else {
1912             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1913         }
1914
1915         if ($drive->{ssd} && ($devicetype eq 'block' || $devicetype eq 'hd')) {
1916             $device .= ",rotation_rate=1";
1917         }
1918         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1919
1920     } elsif ($drive->{interface} eq 'ide' || $drive->{interface} eq 'sata') {
1921         my $maxdev = ($drive->{interface} eq 'sata') ? $MAX_SATA_DISKS : 2;
1922         my $controller = int($drive->{index} / $maxdev);
1923         my $unit = $drive->{index} % $maxdev;
1924         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1925
1926         $device = "ide-$devicetype";
1927         if ($drive->{interface} eq 'ide') {
1928             $device .= ",bus=ide.$controller,unit=$unit";
1929         } else {
1930             $device .= ",bus=ahci$controller.$unit";
1931         }
1932         $device .= ",drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1933
1934         if ($devicetype eq 'hd') {
1935             if (my $model = $drive->{model}) {
1936                 $model = URI::Escape::uri_unescape($model);
1937                 $device .= ",model=$model";
1938             }
1939             if ($drive->{ssd}) {
1940                 $device .= ",rotation_rate=1";
1941             }
1942         }
1943         $device .= ",wwn=$drive->{wwn}" if $drive->{wwn};
1944     } elsif ($drive->{interface} eq 'usb') {
1945         die "implement me";
1946         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1947     } else {
1948         die "unsupported interface type";
1949     }
1950
1951     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1952
1953     if (my $serial = $drive->{serial}) {
1954         $serial = URI::Escape::uri_unescape($serial);
1955         $device .= ",serial=$serial";
1956     }
1957
1958
1959     return $device;
1960 }
1961
1962 sub get_initiator_name {
1963     my $initiator;
1964
1965     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1966     while (defined(my $line = <$fh>)) {
1967         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1968         $initiator = $1;
1969         last;
1970     }
1971     $fh->close();
1972
1973     return $initiator;
1974 }
1975
1976 sub print_drive_full {
1977     my ($storecfg, $vmid, $drive) = @_;
1978
1979     my $path;
1980     my $volid = $drive->{file};
1981     my $format;
1982
1983     if (drive_is_cdrom($drive)) {
1984         $path = get_iso_path($storecfg, $vmid, $volid);
1985     } else {
1986         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1987         if ($storeid) {
1988             $path = PVE::Storage::path($storecfg, $volid);
1989             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1990             $format = qemu_img_format($scfg, $volname);
1991         } else {
1992             $path = $volid;
1993             $format = "raw";
1994         }
1995    }
1996
1997     my $opts = '';
1998     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1999     foreach my $o (@qemu_drive_options) {
2000         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
2001     }
2002
2003     # snapshot only accepts on|off
2004     if (defined($drive->{snapshot})) {
2005         my $v = $drive->{snapshot} ? 'on' : 'off';
2006         $opts .= ",snapshot=$v";
2007     }
2008
2009     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
2010         my ($dir, $qmpname) = @$type;
2011         if (my $v = $drive->{"mbps$dir"}) {
2012             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
2013         }
2014         if (my $v = $drive->{"mbps${dir}_max"}) {
2015             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
2016         }
2017         if (my $v = $drive->{"bps${dir}_max_length"}) {
2018             $opts .= ",throttling.bps$qmpname-max-length=$v";
2019         }
2020         if (my $v = $drive->{"iops${dir}"}) {
2021             $opts .= ",throttling.iops$qmpname=$v";
2022         }
2023         if (my $v = $drive->{"iops${dir}_max"}) {
2024             $opts .= ",throttling.iops$qmpname-max=$v";
2025         }
2026         if (my $v = $drive->{"iops${dir}_max_length"}) {
2027             $opts .= ",throttling.iops$qmpname-max-length=$v";
2028         }
2029     }
2030
2031     $opts .= ",format=$format" if $format && !$drive->{format};
2032
2033     my $cache_direct = 0;
2034
2035     if (my $cache = $drive->{cache}) {
2036         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
2037     } elsif (!drive_is_cdrom($drive)) {
2038         $opts .= ",cache=none";
2039         $cache_direct = 1;
2040     }
2041
2042     # aio native works only with O_DIRECT
2043     if (!$drive->{aio}) {
2044         if($cache_direct) {
2045             $opts .= ",aio=native";
2046         } else {
2047             $opts .= ",aio=threads";
2048         }
2049     }
2050
2051     if (!drive_is_cdrom($drive)) {
2052         my $detectzeroes;
2053         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
2054             $detectzeroes = 'off';
2055         } elsif ($drive->{discard}) {
2056             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
2057         } else {
2058             # This used to be our default with discard not being specified:
2059             $detectzeroes = 'on';
2060         }
2061         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
2062     }
2063
2064     my $pathinfo = $path ? "file=$path," : '';
2065
2066     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
2067 }
2068
2069 sub print_netdevice_full {
2070     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files, $arch, $machine_type) = @_;
2071
2072     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
2073
2074     my $device = $net->{model};
2075     if ($net->{model} eq 'virtio') {
2076          $device = 'virtio-net-pci';
2077      };
2078
2079     my $pciaddr = print_pci_addr("$netid", $bridges, $arch, $machine_type);
2080     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
2081     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
2082         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
2083         my $vectors = $net->{queues} * 2 + 2;
2084         $tmpstr .= ",vectors=$vectors,mq=on";
2085     }
2086     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
2087
2088     if ($use_old_bios_files) {
2089         my $romfile;
2090         if ($device eq 'virtio-net-pci') {
2091             $romfile = 'pxe-virtio.rom';
2092         } elsif ($device eq 'e1000') {
2093             $romfile = 'pxe-e1000.rom';
2094         } elsif ($device eq 'ne2k') {
2095             $romfile = 'pxe-ne2k_pci.rom';
2096         } elsif ($device eq 'pcnet') {
2097             $romfile = 'pxe-pcnet.rom';
2098         } elsif ($device eq 'rtl8139') {
2099             $romfile = 'pxe-rtl8139.rom';
2100         }
2101         $tmpstr .= ",romfile=$romfile" if $romfile;
2102     }
2103
2104     return $tmpstr;
2105 }
2106
2107 sub print_netdev_full {
2108     my ($vmid, $conf, $arch, $net, $netid, $hotplug) = @_;
2109
2110     my $i = '';
2111     if ($netid =~ m/^net(\d+)$/) {
2112         $i = int($1);
2113     }
2114
2115     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
2116
2117     my $ifname = "tap${vmid}i$i";
2118
2119     # kvm uses TUNSETIFF ioctl, and that limits ifname length
2120     die "interface name '$ifname' is too long (max 15 character)\n"
2121         if length($ifname) >= 16;
2122
2123     my $vhostparam = '';
2124     if (is_native($arch)) {
2125         $vhostparam = ',vhost=on' if kernel_has_vhost_net() && $net->{model} eq 'virtio';
2126     }
2127
2128     my $vmname = $conf->{name} || "vm$vmid";
2129
2130     my $netdev = "";
2131     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
2132
2133     if ($net->{bridge}) {
2134         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
2135     } else {
2136         $netdev = "type=user,id=$netid,hostname=$vmname";
2137     }
2138
2139     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
2140
2141     return $netdev;
2142 }
2143
2144
2145 sub print_cpu_device {
2146     my ($conf, $id) = @_;
2147
2148     my $kvm = $conf->{kvm} // 1;
2149     my $cpu = $kvm ? "kvm64" : "qemu64";
2150     if (my $cputype = $conf->{cpu}) {
2151         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
2152             or die "Cannot parse cpu description: $cputype\n";
2153         $cpu = $cpuconf->{cputype};
2154     }
2155
2156     my $cores = $conf->{cores} || 1;
2157
2158     my $current_core = ($id - 1) % $cores;
2159     my $current_socket = int(($id - 1 - $current_core)/$cores);
2160
2161     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
2162 }
2163
2164 my $vga_map = {
2165     'cirrus' => 'cirrus-vga',
2166     'std' => 'VGA',
2167     'vmware' => 'vmware-svga',
2168     'virtio' => 'virtio-vga',
2169 };
2170
2171 sub print_vga_device {
2172     my ($conf, $vga, $arch, $machine, $id, $qxlnum, $bridges) = @_;
2173
2174     my $type = $vga_map->{$vga->{type}};
2175     if ($arch eq 'aarch64' && defined($type) && $type eq 'virtio-vga') {
2176         $type = 'virtio-gpu';
2177     }
2178     my $vgamem_mb = $vga->{memory};
2179     if ($qxlnum) {
2180         $type = $id ? 'qxl' : 'qxl-vga';
2181     }
2182     die "no devicetype for $vga->{type}\n" if !$type;
2183
2184     my $memory = "";
2185     if ($vgamem_mb) {
2186         if ($vga->{type} eq 'virtio') {
2187             my $bytes = PVE::Tools::convert_size($vgamem_mb, "mb" => "b");
2188             $memory = ",max_hostmem=$bytes";
2189         } elsif ($qxlnum) {
2190             # from https://www.spice-space.org/multiple-monitors.html
2191             $memory = ",vgamem_mb=$vga->{memory}";
2192             my $ram = $vgamem_mb * 4;
2193             my $vram = $vgamem_mb * 2;
2194             $memory .= ",ram_size_mb=$ram,vram_size_mb=$vram";
2195         } else {
2196             $memory = ",vgamem_mb=$vga->{memory}";
2197         }
2198     } elsif ($qxlnum && $id) {
2199         $memory = ",ram_size=67108864,vram_size=33554432";
2200     }
2201
2202     my $q35 = machine_type_is_q35($conf);
2203     my $vgaid = "vga" . ($id // '');
2204     my $pciaddr;
2205
2206     if ($q35 && $vgaid eq 'vga') {
2207         # the first display uses pcie.0 bus on q35 machines
2208         $pciaddr = print_pcie_addr($vgaid, $bridges, $arch, $machine);
2209     } else {
2210         $pciaddr = print_pci_addr($vgaid, $bridges, $arch, $machine);
2211     }
2212
2213     return "$type,id=${vgaid}${memory}${pciaddr}";
2214 }
2215
2216 sub drive_is_cloudinit {
2217     my ($drive) = @_;
2218     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
2219 }
2220
2221 sub drive_is_cdrom {
2222     my ($drive, $exclude_cloudinit) = @_;
2223
2224     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
2225
2226     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
2227
2228 }
2229
2230 sub parse_number_sets {
2231     my ($set) = @_;
2232     my $res = [];
2233     foreach my $part (split(/;/, $set)) {
2234         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
2235             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
2236             push @$res, [ $1, $2 ];
2237         } else {
2238             die "invalid range: $part\n";
2239         }
2240     }
2241     return $res;
2242 }
2243
2244 sub parse_numa {
2245     my ($data) = @_;
2246
2247     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
2248     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
2249     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
2250     return $res;
2251 }
2252
2253 sub parse_hostpci {
2254     my ($value) = @_;
2255
2256     return undef if !$value;
2257
2258     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
2259
2260     my @idlist = split(/;/, $res->{host});
2261     delete $res->{host};
2262     foreach my $id (@idlist) {
2263         if ($id =~ m/\./) { # full id 00:00.1
2264             push @{$res->{pciid}}, {
2265                 id => $id,
2266             };
2267         } else { # partial id 00:00
2268             $res->{pciid} = PVE::SysFSTools::lspci($id);
2269         }
2270     }
2271     return $res;
2272 }
2273
2274 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
2275 sub parse_net {
2276     my ($data) = @_;
2277
2278     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
2279     if ($@) {
2280         warn $@;
2281         return undef;
2282     }
2283     if (!defined($res->{macaddr})) {
2284         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
2285         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
2286     }
2287     return $res;
2288 }
2289
2290 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
2291 sub parse_ipconfig {
2292     my ($data) = @_;
2293
2294     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
2295     if ($@) {
2296         warn $@;
2297         return undef;
2298     }
2299
2300     if ($res->{gw} && !$res->{ip}) {
2301         warn 'gateway specified without specifying an IP address';
2302         return undef;
2303     }
2304     if ($res->{gw6} && !$res->{ip6}) {
2305         warn 'IPv6 gateway specified without specifying an IPv6 address';
2306         return undef;
2307     }
2308     if ($res->{gw} && $res->{ip} eq 'dhcp') {
2309         warn 'gateway specified together with DHCP';
2310         return undef;
2311     }
2312     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2313         # gw6 + auto/dhcp
2314         warn "IPv6 gateway specified together with $res->{ip6} address";
2315         return undef;
2316     }
2317
2318     if (!$res->{ip} && !$res->{ip6}) {
2319         return { ip => 'dhcp', ip6 => 'dhcp' };
2320     }
2321
2322     return $res;
2323 }
2324
2325 sub print_net {
2326     my $net = shift;
2327
2328     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2329 }
2330
2331 sub add_random_macs {
2332     my ($settings) = @_;
2333
2334     foreach my $opt (keys %$settings) {
2335         next if $opt !~ m/^net(\d+)$/;
2336         my $net = parse_net($settings->{$opt});
2337         next if !$net;
2338         $settings->{$opt} = print_net($net);
2339     }
2340 }
2341
2342 sub vm_is_volid_owner {
2343     my ($storecfg, $vmid, $volid) = @_;
2344
2345     if ($volid !~  m|^/|) {
2346         my ($path, $owner);
2347         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2348         if ($owner && ($owner == $vmid)) {
2349             return 1;
2350         }
2351     }
2352
2353     return undef;
2354 }
2355
2356 sub vmconfig_register_unused_drive {
2357     my ($storecfg, $vmid, $conf, $drive) = @_;
2358
2359     if (drive_is_cloudinit($drive)) {
2360         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2361         warn $@ if $@;
2362     } elsif (!drive_is_cdrom($drive)) {
2363         my $volid = $drive->{file};
2364         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2365             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2366         }
2367     }
2368 }
2369
2370 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str][,base64=bool]
2371 my $smbios1_fmt = {
2372     uuid => {
2373         type => 'string',
2374         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2375         format_description => 'UUID',
2376         description => "Set SMBIOS1 UUID.",
2377         optional => 1,
2378     },
2379     version => {
2380         type => 'string',
2381         pattern => '[A-Za-z0-9+\/]+={0,2}',
2382         format_description => 'Base64 encoded string',
2383         description => "Set SMBIOS1 version.",
2384         optional => 1,
2385     },
2386     serial => {
2387         type => 'string',
2388         pattern => '[A-Za-z0-9+\/]+={0,2}',
2389         format_description => 'Base64 encoded string',
2390         description => "Set SMBIOS1 serial number.",
2391         optional => 1,
2392     },
2393     manufacturer => {
2394         type => 'string',
2395         pattern => '[A-Za-z0-9+\/]+={0,2}',
2396         format_description => 'Base64 encoded string',
2397         description => "Set SMBIOS1 manufacturer.",
2398         optional => 1,
2399     },
2400     product => {
2401         type => 'string',
2402         pattern => '[A-Za-z0-9+\/]+={0,2}',
2403         format_description => 'Base64 encoded string',
2404         description => "Set SMBIOS1 product ID.",
2405         optional => 1,
2406     },
2407     sku => {
2408         type => 'string',
2409         pattern => '[A-Za-z0-9+\/]+={0,2}',
2410         format_description => 'Base64 encoded string',
2411         description => "Set SMBIOS1 SKU string.",
2412         optional => 1,
2413     },
2414     family => {
2415         type => 'string',
2416         pattern => '[A-Za-z0-9+\/]+={0,2}',
2417         format_description => 'Base64 encoded string',
2418         description => "Set SMBIOS1 family string.",
2419         optional => 1,
2420     },
2421     base64 => {
2422         type => 'boolean',
2423         description => 'Flag to indicate that the SMBIOS values are base64 encoded',
2424         optional => 1,
2425     },
2426 };
2427
2428 sub parse_smbios1 {
2429     my ($data) = @_;
2430
2431     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2432     warn $@ if $@;
2433     return $res;
2434 }
2435
2436 sub print_smbios1 {
2437     my ($smbios1) = @_;
2438     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2439 }
2440
2441 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2442
2443 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2444 sub verify_bootdisk {
2445     my ($value, $noerr) = @_;
2446
2447     return $value if is_valid_drivename($value);
2448
2449     return undef if $noerr;
2450
2451     die "invalid boot disk '$value'\n";
2452 }
2453
2454 sub parse_watchdog {
2455     my ($value) = @_;
2456
2457     return undef if !$value;
2458
2459     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2460     warn $@ if $@;
2461     return $res;
2462 }
2463
2464 sub parse_guest_agent {
2465     my ($value) = @_;
2466
2467     return {} if !defined($value->{agent});
2468
2469     my $res = eval { PVE::JSONSchema::parse_property_string($agent_fmt, $value->{agent}) };
2470     warn $@ if $@;
2471
2472     # if the agent is disabled ignore the other potentially set properties
2473     return {} if !$res->{enabled};
2474     return $res;
2475 }
2476
2477 sub parse_vga {
2478     my ($value) = @_;
2479
2480     return {} if !$value;
2481     my $res = eval { PVE::JSONSchema::parse_property_string($vga_fmt, $value) };
2482     warn $@ if $@;
2483     return $res;
2484 }
2485
2486 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2487 sub verify_usb_device {
2488     my ($value, $noerr) = @_;
2489
2490     return $value if parse_usb_device($value);
2491
2492     return undef if $noerr;
2493
2494     die "unable to parse usb device\n";
2495 }
2496
2497 # add JSON properties for create and set function
2498 sub json_config_properties {
2499     my $prop = shift;
2500
2501     foreach my $opt (keys %$confdesc) {
2502         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate' || $opt eq 'runningmachine';
2503         $prop->{$opt} = $confdesc->{$opt};
2504     }
2505
2506     return $prop;
2507 }
2508
2509 # return copy of $confdesc_cloudinit to generate documentation
2510 sub cloudinit_config_properties {
2511
2512     return dclone($confdesc_cloudinit);
2513 }
2514
2515 sub check_type {
2516     my ($key, $value) = @_;
2517
2518     die "unknown setting '$key'\n" if !$confdesc->{$key};
2519
2520     my $type = $confdesc->{$key}->{type};
2521
2522     if (!defined($value)) {
2523         die "got undefined value\n";
2524     }
2525
2526     if ($value =~ m/[\n\r]/) {
2527         die "property contains a line feed\n";
2528     }
2529
2530     if ($type eq 'boolean') {
2531         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2532         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2533         die "type check ('boolean') failed - got '$value'\n";
2534     } elsif ($type eq 'integer') {
2535         return int($1) if $value =~ m/^(\d+)$/;
2536         die "type check ('integer') failed - got '$value'\n";
2537     } elsif ($type eq 'number') {
2538         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2539         die "type check ('number') failed - got '$value'\n";
2540     } elsif ($type eq 'string') {
2541         if (my $fmt = $confdesc->{$key}->{format}) {
2542             PVE::JSONSchema::check_format($fmt, $value);
2543             return $value;
2544         }
2545         $value =~ s/^\"(.*)\"$/$1/;
2546         return $value;
2547     } else {
2548         die "internal error"
2549     }
2550 }
2551
2552 sub destroy_vm {
2553     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2554
2555     my $conf = PVE::QemuConfig->load_config($vmid);
2556
2557     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2558
2559     if ($conf->{template}) {
2560         # check if any base image is still used by a linked clone
2561         foreach_drive($conf, sub {
2562                 my ($ds, $drive) = @_;
2563
2564                 return if drive_is_cdrom($drive);
2565
2566                 my $volid = $drive->{file};
2567
2568                 return if !$volid || $volid =~ m|^/|;
2569
2570                 die "base volume '$volid' is still in use by linked cloned\n"
2571                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2572
2573         });
2574     }
2575
2576     # only remove disks owned by this VM
2577     foreach_drive($conf, sub {
2578         my ($ds, $drive) = @_;
2579
2580         return if drive_is_cdrom($drive, 1);
2581
2582         my $volid = $drive->{file};
2583
2584         return if !$volid || $volid =~ m|^/|;
2585
2586         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2587         return if !$path || !$owner || ($owner != $vmid);
2588
2589         eval {
2590             PVE::Storage::vdisk_free($storecfg, $volid);
2591         };
2592         warn "Could not remove disk '$volid', check manually: $@" if $@;
2593
2594     });
2595
2596     # also remove unused disk
2597     eval {
2598         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2599
2600         eval {
2601             PVE::Storage::foreach_volid($dl, sub {
2602                 my ($volid, $sid, $volname, $d) = @_;
2603                 PVE::Storage::vdisk_free($storecfg, $volid);
2604             });
2605         };
2606         warn $@ if $@;
2607
2608     };
2609     warn $@ if $@;
2610
2611     if ($keep_empty_config) {
2612         PVE::QemuConfig->write_config($vmid, { memory => 128 });
2613     } else {
2614         PVE::QemuConfig->destroy_config($vmid);
2615     }
2616 }
2617
2618 sub parse_vm_config {
2619     my ($filename, $raw) = @_;
2620
2621     return undef if !defined($raw);
2622
2623     my $res = {
2624         digest => Digest::SHA::sha1_hex($raw),
2625         snapshots => {},
2626         pending => {},
2627     };
2628
2629     $filename =~ m|/qemu-server/(\d+)\.conf$|
2630         || die "got strange filename '$filename'";
2631
2632     my $vmid = $1;
2633
2634     my $conf = $res;
2635     my $descr;
2636     my $section = '';
2637
2638     my @lines = split(/\n/, $raw);
2639     foreach my $line (@lines) {
2640         next if $line =~ m/^\s*$/;
2641
2642         if ($line =~ m/^\[PENDING\]\s*$/i) {
2643             $section = 'pending';
2644             if (defined($descr)) {
2645                 $descr =~ s/\s+$//;
2646                 $conf->{description} = $descr;
2647             }
2648             $descr = undef;
2649             $conf = $res->{$section} = {};
2650             next;
2651
2652         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2653             $section = $1;
2654             if (defined($descr)) {
2655                 $descr =~ s/\s+$//;
2656                 $conf->{description} = $descr;
2657             }
2658             $descr = undef;
2659             $conf = $res->{snapshots}->{$section} = {};
2660             next;
2661         }
2662
2663         if ($line =~ m/^\#(.*)\s*$/) {
2664             $descr = '' if !defined($descr);
2665             $descr .= PVE::Tools::decode_text($1) . "\n";
2666             next;
2667         }
2668
2669         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2670             $descr = '' if !defined($descr);
2671             $descr .= PVE::Tools::decode_text($2);
2672         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2673             $conf->{snapstate} = $1;
2674         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2675             my $key = $1;
2676             my $value = $2;
2677             $conf->{$key} = $value;
2678         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2679             my $value = $1;
2680             if ($section eq 'pending') {
2681                 $conf->{delete} = $value; # we parse this later
2682             } else {
2683                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2684             }
2685         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2686             my $key = $1;
2687             my $value = $2;
2688             eval { $value = check_type($key, $value); };
2689             if ($@) {
2690                 warn "vm $vmid - unable to parse value of '$key' - $@";
2691             } else {
2692                 $key = 'ide2' if $key eq 'cdrom';
2693                 my $fmt = $confdesc->{$key}->{format};
2694                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2695                     my $v = parse_drive($key, $value);
2696                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2697                         $v->{file} = $volid;
2698                         $value = print_drive($vmid, $v);
2699                     } else {
2700                         warn "vm $vmid - unable to parse value of '$key'\n";
2701                         next;
2702                     }
2703                 }
2704
2705                 $conf->{$key} = $value;
2706             }
2707         }
2708     }
2709
2710     if (defined($descr)) {
2711         $descr =~ s/\s+$//;
2712         $conf->{description} = $descr;
2713     }
2714     delete $res->{snapstate}; # just to be sure
2715
2716     return $res;
2717 }
2718
2719 sub write_vm_config {
2720     my ($filename, $conf) = @_;
2721
2722     delete $conf->{snapstate}; # just to be sure
2723
2724     if ($conf->{cdrom}) {
2725         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2726         $conf->{ide2} = $conf->{cdrom};
2727         delete $conf->{cdrom};
2728     }
2729
2730     # we do not use 'smp' any longer
2731     if ($conf->{sockets}) {
2732         delete $conf->{smp};
2733     } elsif ($conf->{smp}) {
2734         $conf->{sockets} = $conf->{smp};
2735         delete $conf->{cores};
2736         delete $conf->{smp};
2737     }
2738
2739     my $used_volids = {};
2740
2741     my $cleanup_config = sub {
2742         my ($cref, $pending, $snapname) = @_;
2743
2744         foreach my $key (keys %$cref) {
2745             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2746                 $key eq 'snapstate' || $key eq 'pending';
2747             my $value = $cref->{$key};
2748             if ($key eq 'delete') {
2749                 die "propertry 'delete' is only allowed in [PENDING]\n"
2750                     if !$pending;
2751                 # fixme: check syntax?
2752                 next;
2753             }
2754             eval { $value = check_type($key, $value); };
2755             die "unable to parse value of '$key' - $@" if $@;
2756
2757             $cref->{$key} = $value;
2758
2759             if (!$snapname && is_valid_drivename($key)) {
2760                 my $drive = parse_drive($key, $value);
2761                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2762             }
2763         }
2764     };
2765
2766     &$cleanup_config($conf);
2767
2768     &$cleanup_config($conf->{pending}, 1);
2769
2770     foreach my $snapname (keys %{$conf->{snapshots}}) {
2771         die "internal error" if $snapname eq 'pending';
2772         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2773     }
2774
2775     # remove 'unusedX' settings if we re-add a volume
2776     foreach my $key (keys %$conf) {
2777         my $value = $conf->{$key};
2778         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2779             delete $conf->{$key};
2780         }
2781     }
2782
2783     my $generate_raw_config = sub {
2784         my ($conf, $pending) = @_;
2785
2786         my $raw = '';
2787
2788         # add description as comment to top of file
2789         if (defined(my $descr = $conf->{description})) {
2790             if ($descr) {
2791                 foreach my $cl (split(/\n/, $descr)) {
2792                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2793                 }
2794             } else {
2795                 $raw .= "#\n" if $pending;
2796             }
2797         }
2798
2799         foreach my $key (sort keys %$conf) {
2800             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2801             $raw .= "$key: $conf->{$key}\n";
2802         }
2803         return $raw;
2804     };
2805
2806     my $raw = &$generate_raw_config($conf);
2807
2808     if (scalar(keys %{$conf->{pending}})){
2809         $raw .= "\n[PENDING]\n";
2810         $raw .= &$generate_raw_config($conf->{pending}, 1);
2811     }
2812
2813     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2814         $raw .= "\n[$snapname]\n";
2815         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2816     }
2817
2818     return $raw;
2819 }
2820
2821 sub load_defaults {
2822
2823     my $res = {};
2824
2825     # we use static defaults from our JSON schema configuration
2826     foreach my $key (keys %$confdesc) {
2827         if (defined(my $default = $confdesc->{$key}->{default})) {
2828             $res->{$key} = $default;
2829         }
2830     }
2831
2832     return $res;
2833 }
2834
2835 sub config_list {
2836     my $vmlist = PVE::Cluster::get_vmlist();
2837     my $res = {};
2838     return $res if !$vmlist || !$vmlist->{ids};
2839     my $ids = $vmlist->{ids};
2840
2841     foreach my $vmid (keys %$ids) {
2842         my $d = $ids->{$vmid};
2843         next if !$d->{node} || $d->{node} ne $nodename;
2844         next if !$d->{type} || $d->{type} ne 'qemu';
2845         $res->{$vmid}->{exists} = 1;
2846     }
2847     return $res;
2848 }
2849
2850 # test if VM uses local resources (to prevent migration)
2851 sub check_local_resources {
2852     my ($conf, $noerr) = @_;
2853
2854     my @loc_res = ();
2855
2856     push @loc_res, "hostusb" if $conf->{hostusb}; # old syntax
2857     push @loc_res, "hostpci" if $conf->{hostpci}; # old syntax
2858
2859     push @loc_res, "ivshmem" if $conf->{ivshmem};
2860
2861     foreach my $k (keys %$conf) {
2862         next if $k =~ m/^usb/ && ($conf->{$k} =~ m/^spice(?![^,])/);
2863         # sockets are safe: they will recreated be on the target side post-migrate
2864         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2865         push @loc_res, $k if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2866     }
2867
2868     die "VM uses local resources\n" if scalar @loc_res && !$noerr;
2869
2870     return \@loc_res;
2871 }
2872
2873 # check if used storages are available on all nodes (use by migrate)
2874 sub check_storage_availability {
2875     my ($storecfg, $conf, $node) = @_;
2876
2877     foreach_drive($conf, sub {
2878         my ($ds, $drive) = @_;
2879
2880         my $volid = $drive->{file};
2881         return if !$volid;
2882
2883         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2884         return if !$sid;
2885
2886         # check if storage is available on both nodes
2887         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2888         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2889    });
2890 }
2891
2892 # list nodes where all VM images are available (used by has_feature API)
2893 sub shared_nodes {
2894     my ($conf, $storecfg) = @_;
2895
2896     my $nodelist = PVE::Cluster::get_nodelist();
2897     my $nodehash = { map { $_ => 1 } @$nodelist };
2898     my $nodename = PVE::INotify::nodename();
2899
2900     foreach_drive($conf, sub {
2901         my ($ds, $drive) = @_;
2902
2903         my $volid = $drive->{file};
2904         return if !$volid;
2905
2906         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2907         if ($storeid) {
2908             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2909             if ($scfg->{disable}) {
2910                 $nodehash = {};
2911             } elsif (my $avail = $scfg->{nodes}) {
2912                 foreach my $node (keys %$nodehash) {
2913                     delete $nodehash->{$node} if !$avail->{$node};
2914                 }
2915             } elsif (!$scfg->{shared}) {
2916                 foreach my $node (keys %$nodehash) {
2917                     delete $nodehash->{$node} if $node ne $nodename
2918                 }
2919             }
2920         }
2921     });
2922
2923     return $nodehash
2924 }
2925
2926 sub check_local_storage_availability {
2927     my ($conf, $storecfg) = @_;
2928
2929     my $nodelist = PVE::Cluster::get_nodelist();
2930     my $nodehash = { map { $_ => {} } @$nodelist };
2931
2932     foreach_drive($conf, sub {
2933         my ($ds, $drive) = @_;
2934
2935         my $volid = $drive->{file};
2936         return if !$volid;
2937
2938         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2939         if ($storeid) {
2940             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2941
2942             if ($scfg->{disable}) {
2943                 foreach my $node (keys %$nodehash) {
2944                     $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2945                 }
2946             } elsif (my $avail = $scfg->{nodes}) {
2947                 foreach my $node (keys %$nodehash) {
2948                     if (!$avail->{$node}) {
2949                         $nodehash->{$node}->{unavailable_storages}->{$storeid} = 1;
2950                     }
2951                 }
2952             }
2953         }
2954     });
2955
2956     foreach my $node (values %$nodehash) {
2957         if (my $unavail = $node->{unavailable_storages}) {
2958             $node->{unavailable_storages} = [ sort keys %$unavail ];
2959         }
2960     }
2961
2962     return $nodehash
2963 }
2964
2965 sub check_cmdline {
2966     my ($pidfile, $pid) = @_;
2967
2968     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2969     if (defined($fh)) {
2970         my $line = <$fh>;
2971         $fh->close;
2972         return undef if !$line;
2973         my @param = split(/\0/, $line);
2974
2975         my $cmd = $param[0];
2976         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m@(?:^|/)qemu-system-[^/]+$@);
2977
2978         for (my $i = 0; $i < scalar (@param); $i++) {
2979             my $p = $param[$i];
2980             next if !$p;
2981             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2982                 my $p = $param[$i+1];
2983                 return 1 if $p && ($p eq $pidfile);
2984                 return undef;
2985             }
2986         }
2987     }
2988     return undef;
2989 }
2990
2991 sub check_running {
2992     my ($vmid, $nocheck, $node) = @_;
2993
2994     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2995
2996     die "unable to find configuration file for VM $vmid - no such machine\n"
2997         if !$nocheck && ! -f $filename;
2998
2999     my $pidfile = pidfile_name($vmid);
3000
3001     if (my $fd = IO::File->new("<$pidfile")) {
3002         my $st = stat($fd);
3003         my $line = <$fd>;
3004         close($fd);
3005
3006         my $mtime = $st->mtime;
3007         if ($mtime > time()) {
3008             warn "file '$filename' modified in future\n";
3009         }
3010
3011         if ($line =~ m/^(\d+)$/) {
3012             my $pid = $1;
3013             if (check_cmdline($pidfile, $pid)) {
3014                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
3015                     return $pid;
3016                 }
3017             }
3018         }
3019     }
3020
3021     return undef;
3022 }
3023
3024 sub vzlist {
3025
3026     my $vzlist = config_list();
3027
3028     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
3029
3030     while (defined(my $de = $fd->read)) {
3031         next if $de !~ m/^(\d+)\.pid$/;
3032         my $vmid = $1;
3033         next if !defined($vzlist->{$vmid});
3034         if (my $pid = check_running($vmid)) {
3035             $vzlist->{$vmid}->{pid} = $pid;
3036         }
3037     }
3038
3039     return $vzlist;
3040 }
3041
3042 sub disksize {
3043     my ($storecfg, $conf) = @_;
3044
3045     my $bootdisk = $conf->{bootdisk};
3046     return undef if !$bootdisk;
3047     return undef if !is_valid_drivename($bootdisk);
3048
3049     return undef if !$conf->{$bootdisk};
3050
3051     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
3052     return undef if !defined($drive);
3053
3054     return undef if drive_is_cdrom($drive);
3055
3056     my $volid = $drive->{file};
3057     return undef if !$volid;
3058
3059     return $drive->{size};
3060 }
3061
3062 our $vmstatus_return_properties = {
3063     vmid => get_standard_option('pve-vmid'),
3064     status => {
3065         description => "Qemu process status.",
3066         type => 'string',
3067         enum => ['stopped', 'running'],
3068     },
3069     maxmem => {
3070         description => "Maximum memory in bytes.",
3071         type => 'integer',
3072         optional => 1,
3073         renderer => 'bytes',
3074     },
3075     maxdisk => {
3076         description => "Root disk size in bytes.",
3077         type => 'integer',
3078         optional => 1,
3079         renderer => 'bytes',
3080     },
3081     name => {
3082         description => "VM name.",
3083         type => 'string',
3084         optional => 1,
3085     },
3086     qmpstatus => {
3087         description => "Qemu QMP agent status.",
3088         type => 'string',
3089         optional => 1,
3090     },
3091     pid => {
3092         description => "PID of running qemu process.",
3093         type => 'integer',
3094         optional => 1,
3095     },
3096     uptime => {
3097         description => "Uptime.",
3098         type => 'integer',
3099         optional => 1,
3100         renderer => 'duration',
3101     },
3102     cpus => {
3103         description => "Maximum usable CPUs.",
3104         type => 'number',
3105         optional => 1,
3106     },
3107     lock => {
3108         description => "The current config lock, if any.",
3109         type => 'string',
3110         optional => 1,
3111     }
3112 };
3113
3114 my $last_proc_pid_stat;
3115
3116 # get VM status information
3117 # This must be fast and should not block ($full == false)
3118 # We only query KVM using QMP if $full == true (this can be slow)
3119 sub vmstatus {
3120     my ($opt_vmid, $full) = @_;
3121
3122     my $res = {};
3123
3124     my $storecfg = PVE::Storage::config();
3125
3126     my $list = vzlist();
3127     my $defaults = load_defaults();
3128
3129     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
3130
3131     my $cpucount = $cpuinfo->{cpus} || 1;
3132
3133     foreach my $vmid (keys %$list) {
3134         next if $opt_vmid && ($vmid ne $opt_vmid);
3135
3136         my $conf = PVE::QemuConfig->load_config($vmid);
3137
3138         my $d = { vmid => $vmid };
3139         $d->{pid} = $list->{$vmid}->{pid};
3140
3141         # fixme: better status?
3142         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
3143
3144         my $size = disksize($storecfg, $conf);
3145         if (defined($size)) {
3146             $d->{disk} = 0; # no info available
3147             $d->{maxdisk} = $size;
3148         } else {
3149             $d->{disk} = 0;
3150             $d->{maxdisk} = 0;
3151         }
3152
3153         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
3154             * ($conf->{cores} || $defaults->{cores});
3155         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
3156         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
3157
3158         $d->{name} = $conf->{name} || "VM $vmid";
3159         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
3160             : $defaults->{memory}*(1024*1024);
3161
3162         if ($conf->{balloon}) {
3163             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
3164             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
3165                 : $defaults->{shares};
3166         }
3167
3168         $d->{uptime} = 0;
3169         $d->{cpu} = 0;
3170         $d->{mem} = 0;
3171
3172         $d->{netout} = 0;
3173         $d->{netin} = 0;
3174
3175         $d->{diskread} = 0;
3176         $d->{diskwrite} = 0;
3177
3178         $d->{template} = PVE::QemuConfig->is_template($conf);
3179
3180         $d->{serial} = 1 if conf_has_serial($conf);
3181         $d->{lock} = $conf->{lock} if $conf->{lock};
3182
3183         $res->{$vmid} = $d;
3184     }
3185
3186     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
3187     foreach my $dev (keys %$netdev) {
3188         next if $dev !~ m/^tap([1-9]\d*)i/;
3189         my $vmid = $1;
3190         my $d = $res->{$vmid};
3191         next if !$d;
3192
3193         $d->{netout} += $netdev->{$dev}->{receive};
3194         $d->{netin} += $netdev->{$dev}->{transmit};
3195
3196         if ($full) {
3197             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
3198             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
3199         }
3200
3201     }
3202
3203     my $ctime = gettimeofday;
3204
3205     foreach my $vmid (keys %$list) {
3206
3207         my $d = $res->{$vmid};
3208         my $pid = $d->{pid};
3209         next if !$pid;
3210
3211         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
3212         next if !$pstat; # not running
3213
3214         my $used = $pstat->{utime} + $pstat->{stime};
3215
3216         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
3217
3218         if ($pstat->{vsize}) {
3219             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
3220         }
3221
3222         my $old = $last_proc_pid_stat->{$pid};
3223         if (!$old) {
3224             $last_proc_pid_stat->{$pid} = {
3225                 time => $ctime,
3226                 used => $used,
3227                 cpu => 0,
3228             };
3229             next;
3230         }
3231
3232         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
3233
3234         if ($dtime > 1000) {
3235             my $dutime = $used -  $old->{used};
3236
3237             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
3238             $last_proc_pid_stat->{$pid} = {
3239                 time => $ctime,
3240                 used => $used,
3241                 cpu => $d->{cpu},
3242             };
3243         } else {
3244             $d->{cpu} = $old->{cpu};
3245         }
3246     }
3247
3248     return $res if !$full;
3249
3250     my $qmpclient = PVE::QMPClient->new();
3251
3252     my $ballooncb = sub {
3253         my ($vmid, $resp) = @_;
3254
3255         my $info = $resp->{'return'};
3256         return if !$info->{max_mem};
3257
3258         my $d = $res->{$vmid};
3259
3260         # use memory assigned to VM
3261         $d->{maxmem} = $info->{max_mem};
3262         $d->{balloon} = $info->{actual};
3263
3264         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
3265             $d->{mem} = $info->{total_mem} - $info->{free_mem};
3266             $d->{freemem} = $info->{free_mem};
3267         }
3268
3269         $d->{ballooninfo} = $info;
3270     };
3271
3272     my $blockstatscb = sub {
3273         my ($vmid, $resp) = @_;
3274         my $data = $resp->{'return'} || [];
3275         my $totalrdbytes = 0;
3276         my $totalwrbytes = 0;
3277
3278         for my $blockstat (@$data) {
3279             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
3280             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
3281
3282             $blockstat->{device} =~ s/drive-//;
3283             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
3284         }
3285         $res->{$vmid}->{diskread} = $totalrdbytes;
3286         $res->{$vmid}->{diskwrite} = $totalwrbytes;
3287     };
3288
3289     my $statuscb = sub {
3290         my ($vmid, $resp) = @_;
3291
3292         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
3293         # this fails if ballon driver is not loaded, so this must be
3294         # the last commnand (following command are aborted if this fails).
3295         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
3296
3297         my $status = 'unknown';
3298         if (!defined($status = $resp->{'return'}->{status})) {
3299             warn "unable to get VM status\n";
3300             return;
3301         }
3302
3303         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
3304     };
3305
3306     foreach my $vmid (keys %$list) {
3307         next if $opt_vmid && ($vmid ne $opt_vmid);
3308         next if !$res->{$vmid}->{pid}; # not running
3309         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
3310     }
3311
3312     $qmpclient->queue_execute(undef, 2);
3313
3314     foreach my $vmid (keys %$list) {
3315         next if $opt_vmid && ($vmid ne $opt_vmid);
3316         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
3317     }
3318
3319     return $res;
3320 }
3321
3322 sub foreach_drive {
3323     my ($conf, $func, @param) = @_;
3324
3325     foreach my $ds (valid_drive_names()) {
3326         next if !defined($conf->{$ds});
3327
3328         my $drive = parse_drive($ds, $conf->{$ds});
3329         next if !$drive;
3330
3331         &$func($ds, $drive, @param);
3332     }
3333 }
3334
3335 sub foreach_volid {
3336     my ($conf, $func, @param) = @_;
3337
3338     my $volhash = {};
3339
3340     my $test_volid = sub {
3341         my ($volid, $is_cdrom, $replicate, $shared, $snapname, $size) = @_;
3342
3343         return if !$volid;
3344
3345         $volhash->{$volid}->{cdrom} //= 1;
3346         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3347
3348         $volhash->{$volid}->{replicate} //= 0;
3349         $volhash->{$volid}->{replicate} = 1 if $replicate;
3350
3351         $volhash->{$volid}->{shared} //= 0;
3352         $volhash->{$volid}->{shared} = 1 if $shared;
3353
3354         $volhash->{$volid}->{referenced_in_config} //= 0;
3355         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3356
3357         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3358             if defined($snapname);
3359         $volhash->{$volid}->{size} = $size if $size;
3360     };
3361
3362     foreach_drive($conf, sub {
3363         my ($ds, $drive) = @_;
3364         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef, $drive->{size});
3365     });
3366
3367     foreach my $snapname (keys %{$conf->{snapshots}}) {
3368         my $snap = $conf->{snapshots}->{$snapname};
3369         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3370         foreach_drive($snap, sub {
3371             my ($ds, $drive) = @_;
3372             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3373         });
3374     }
3375
3376     foreach my $volid (keys %$volhash) {
3377         &$func($volid, $volhash->{$volid}, @param);
3378     }
3379 }
3380
3381 sub conf_has_serial {
3382     my ($conf) = @_;
3383
3384     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3385         if ($conf->{"serial$i"}) {
3386             return 1;
3387         }
3388     }
3389
3390     return 0;
3391 }
3392
3393 sub conf_has_audio {
3394     my ($conf, $id) = @_;
3395
3396     $id //= 0;
3397     my $audio = $conf->{"audio$id"};
3398     return undef if !defined($audio);
3399
3400     my $audioproperties = PVE::JSONSchema::parse_property_string($audio_fmt, $audio);
3401     my $audiodriver = $audioproperties->{driver} // 'spice';
3402
3403     return {
3404         dev => $audioproperties->{device},
3405         dev_id => "audiodev$id",
3406         backend => $audiodriver,
3407         backend_id => "$audiodriver-backend${id}",
3408     };
3409 }
3410
3411 sub vga_conf_has_spice {
3412     my ($vga) = @_;
3413
3414     my $vgaconf = parse_vga($vga);
3415     my $vgatype = $vgaconf->{type};
3416     return 0 if !$vgatype || $vgatype !~ m/^qxl([234])?$/;
3417
3418     return $1 || 1;
3419 }
3420
3421 sub is_native($) {
3422     my ($arch) = @_;
3423     return get_host_arch() eq $arch;
3424 }
3425
3426 my $default_machines = {
3427     x86_64 => 'pc',
3428     aarch64 => 'virt',
3429 };
3430
3431 sub get_basic_machine_info {
3432     my ($conf, $forcemachine) = @_;
3433
3434     my $arch = $conf->{arch} // get_host_arch();
3435     my $machine = $forcemachine || $conf->{machine} || $default_machines->{$arch};
3436     return ($arch, $machine);
3437 }
3438
3439 sub get_ovmf_files($) {
3440     my ($arch) = @_;
3441
3442     my $ovmf = $OVMF->{$arch}
3443         or die "no OVMF images known for architecture '$arch'\n";
3444
3445     return @$ovmf;
3446 }
3447
3448 my $Arch2Qemu = {
3449     aarch64 => '/usr/bin/qemu-system-aarch64',
3450     x86_64 => '/usr/bin/qemu-system-x86_64',
3451 };
3452 sub get_command_for_arch($) {
3453     my ($arch) = @_;
3454     return '/usr/bin/kvm' if is_native($arch);
3455
3456     my $cmd = $Arch2Qemu->{$arch}
3457         or die "don't know how to emulate architecture '$arch'\n";
3458     return $cmd;
3459 }
3460
3461 sub get_cpu_options {
3462     my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough) = @_;
3463
3464     my $cpuFlags = [];
3465     my $ostype = $conf->{ostype};
3466
3467     my $cpu = $kvm ? "kvm64" : "qemu64";
3468     if ($arch eq 'aarch64') {
3469         $cpu = 'cortex-a57';
3470     }
3471     my $hv_vendor_id;
3472     if (my $cputype = $conf->{cpu}) {
3473         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3474             or die "Cannot parse cpu description: $cputype\n";
3475         $cpu = $cpuconf->{cputype};
3476         $kvm_off = 1 if $cpuconf->{hidden};
3477         $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
3478
3479         if (defined(my $flags = $cpuconf->{flags})) {
3480             push @$cpuFlags, split(";", $flags);
3481         }
3482     }
3483
3484     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
3485
3486     push @$cpuFlags , '-x2apic'
3487         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3488
3489     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3490
3491     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3492
3493     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch eq 'x86_64') {
3494
3495         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3496         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3497     }
3498
3499     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
3500
3501     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq 'x86_64';
3502
3503     push @$cpuFlags, 'kvm=off' if $kvm_off;
3504
3505     if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
3506         push @$cpuFlags, "vendor=${cpu_vendor}"
3507             if $cpu_vendor ne 'default';
3508     } elsif ($arch ne 'aarch64') {
3509         die "internal error"; # should not happen
3510     }
3511
3512     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3513
3514     return ('-cpu', $cpu);
3515 }
3516
3517 sub config_to_command {
3518     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3519
3520     my $cmd = [];
3521     my $globalFlags = [];
3522     my $machineFlags = [];
3523     my $rtcFlags = [];
3524     my $devices = [];
3525     my $pciaddr = '';
3526     my $bridges = {};
3527     my $vernum = 0; # unknown
3528     my $ostype = $conf->{ostype};
3529     my $winversion = windows_version($ostype);
3530     my $kvm = $conf->{kvm};
3531
3532     my ($arch, $machine_type) = get_basic_machine_info($conf, $forcemachine);
3533     my $kvm_binary = get_command_for_arch($arch);
3534     my $kvmver = kvm_user_version($kvm_binary);
3535     $kvm //= 1 if is_native($arch);
3536
3537     if ($kvm) {
3538         die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n"
3539             if !defined kvm_version();
3540     }
3541
3542     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3543         $vernum = $1*1000000+$2*1000;
3544     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3545         $vernum = $1*1000000+$2*1000+$3;
3546     }
3547
3548     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3549
3550     my $q35 = machine_type_is_q35($conf);
3551     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3552     my $use_old_bios_files = undef;
3553     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3554
3555     my $cpuunits = defined($conf->{cpuunits}) ?
3556             $conf->{cpuunits} : $defaults->{cpuunits};
3557
3558     push @$cmd, $kvm_binary;
3559
3560     push @$cmd, '-id', $vmid;
3561
3562     my $vmname = $conf->{name} || "vm$vmid";
3563
3564     push @$cmd, '-name', $vmname;
3565
3566     my $use_virtio = 0;
3567
3568     my $qmpsocket = qmp_socket($vmid);
3569     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3570     push @$cmd, '-mon', "chardev=qmp,mode=control";
3571
3572     if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 12)) {
3573         push @$cmd, '-chardev', "socket,id=qmp-event,path=/var/run/qmeventd.sock,reconnect=5";
3574         push @$cmd, '-mon', "chardev=qmp-event,mode=control";
3575     }
3576
3577     push @$cmd, '-pidfile' , pidfile_name($vmid);
3578
3579     push @$cmd, '-daemonize';
3580
3581     if ($conf->{smbios1}) {
3582         my $smbios_conf = parse_smbios1($conf->{smbios1});
3583         if ($smbios_conf->{base64}) {
3584             # Do not pass base64 flag to qemu
3585             delete $smbios_conf->{base64};
3586             my $smbios_string = "";
3587             foreach my $key (keys %$smbios_conf) {
3588                 my $value;
3589                 if ($key eq "uuid") {
3590                     $value = $smbios_conf->{uuid}
3591                 } else {
3592                     $value = decode_base64($smbios_conf->{$key});
3593                 }
3594                 # qemu accepts any binary data, only commas need escaping by double comma
3595                 $value =~ s/,/,,/g;
3596                 $smbios_string .= "," . $key . "=" . $value if $value;
3597             }
3598             push @$cmd, '-smbios', "type=1" . $smbios_string;
3599         } else {
3600             push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3601         }
3602     }
3603
3604     if ($conf->{vmgenid}) {
3605         push @$devices, '-device', 'vmgenid,guid='.$conf->{vmgenid};
3606     }
3607
3608     my ($ovmf_code, $ovmf_vars) = get_ovmf_files($arch);
3609     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3610         die "uefi base image not found\n" if ! -f $ovmf_code;
3611
3612         my $path;
3613         my $format;
3614         if (my $efidisk = $conf->{efidisk0}) {
3615             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3616             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3617             $format = $d->{format};
3618             if ($storeid) {
3619                 $path = PVE::Storage::path($storecfg, $d->{file});
3620                 if (!defined($format)) {
3621                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3622                     $format = qemu_img_format($scfg, $volname);
3623                 }
3624             } else {
3625                 $path = $d->{file};
3626                 die "efidisk format must be specified\n"
3627                     if !defined($format);
3628             }
3629         } else {
3630             warn "no efidisk configured! Using temporary efivars disk.\n";
3631             $path = "/tmp/$vmid-ovmf.fd";
3632             PVE::Tools::file_copy($ovmf_vars, $path, -s $ovmf_vars);
3633             $format = 'raw';
3634         }
3635
3636         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmf_code";
3637         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3638     }
3639
3640     # load q35 config
3641     if ($q35) {
3642         # we use different pcie-port hardware for qemu >= 4.0 for passthrough
3643         if (qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0)) {
3644             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35-4.0.cfg';
3645         } else {
3646             push @$devices, '-readconfig', '/usr/share/qemu-server/pve-q35.cfg';
3647         }
3648     }
3649
3650     # add usb controllers
3651     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $arch, $machine_type, $usbdesc->{format}, $MAX_USB_DEVICES);
3652     push @$devices, @usbcontrollers if @usbcontrollers;
3653     my $vga = parse_vga($conf->{vga});
3654
3655     my $qxlnum = vga_conf_has_spice($conf->{vga});
3656     $vga->{type} = 'qxl' if $qxlnum;
3657
3658     if (!$vga->{type}) {
3659         if ($arch eq 'aarch64') {
3660             $vga->{type} = 'virtio';
3661         } elsif (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3662             $vga->{type} = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3663         } else {
3664             $vga->{type} = ($winversion >= 6) ? 'std' : 'cirrus';
3665         }
3666     }
3667
3668     # enable absolute mouse coordinates (needed by vnc)
3669     my $tablet;
3670     if (defined($conf->{tablet})) {
3671         $tablet = $conf->{tablet};
3672     } else {
3673         $tablet = $defaults->{tablet};
3674         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3675         $tablet = 0 if $vga->{type} =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3676     }
3677
3678     if ($tablet) {
3679         push @$devices, '-device', print_tabletdevice_full($conf, $arch) if $tablet;
3680         my $kbd = print_keyboarddevice_full($conf, $arch);
3681         push @$devices, '-device', $kbd if defined($kbd);
3682     }
3683
3684     my $kvm_off = 0;
3685     my $gpu_passthrough;
3686
3687     # host pci devices
3688     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3689         my $id = "hostpci$i";
3690         my $d = parse_hostpci($conf->{$id});
3691         next if !$d;
3692
3693         if (my $pcie = $d->{pcie}) {
3694             die "q35 machine model is not enabled" if !$q35;
3695             # win7 wants to have the pcie devices directly on the pcie bus
3696             # instead of in the root port
3697             if ($winversion == 7) {
3698                 $pciaddr = print_pcie_addr("${id}bus0");
3699             } else {
3700                 # add more root ports if needed, 4 are present by default
3701                 # by pve-q35 cfgs, rest added here on demand.
3702                 if ($i > 3) {
3703                     push @$devices, '-device', print_pcie_root_port($i);
3704                 }
3705                 $pciaddr = print_pcie_addr($id);
3706             }
3707         } else {
3708             $pciaddr = print_pci_addr($id, $bridges, $arch, $machine_type);
3709         }
3710
3711         my $xvga = '';
3712         if ($d->{'x-vga'}) {
3713             $xvga = ',x-vga=on' if !($conf->{bios} && $conf->{bios} eq 'ovmf');
3714             $kvm_off = 1;
3715             $vga->{type} = 'none' if !defined($conf->{vga});
3716             $gpu_passthrough = 1;
3717         }
3718
3719         my $pcidevices = $d->{pciid};
3720         my $multifunction = 1 if @$pcidevices > 1;
3721
3722         my $sysfspath;
3723         if ($d->{mdev} && scalar(@$pcidevices) == 1) {
3724             my $pci_id = $pcidevices->[0]->{id};
3725             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
3726             $sysfspath = "/sys/bus/pci/devices/0000:$pci_id/$uuid";
3727         } elsif ($d->{mdev}) {
3728             warn "ignoring mediated device '$id' with multifunction device\n";
3729         }
3730
3731         my $j=0;
3732         foreach my $pcidevice (@$pcidevices) {
3733             my $devicestr = "vfio-pci";
3734
3735             if ($sysfspath) {
3736                 $devicestr .= ",sysfsdev=$sysfspath";
3737             } else {
3738                 $devicestr .= ",host=$pcidevice->{id}";
3739             }
3740
3741             my $mf_addr = $multifunction ? ".$j" : '';
3742             $devicestr .= ",id=${id}${mf_addr}${pciaddr}${mf_addr}";
3743
3744             if ($j == 0) {
3745                 $devicestr .= ',rombar=0' if defined($d->{rombar}) && !$d->{rombar};
3746                 $devicestr .= "$xvga";
3747                 $devicestr .= ",multifunction=on" if $multifunction;
3748                 $devicestr .= ",romfile=/usr/share/kvm/$d->{romfile}" if $d->{romfile};
3749             }
3750
3751             push @$devices, '-device', $devicestr;
3752             $j++;
3753         }
3754     }
3755
3756     # usb devices
3757     my $usb_dev_features = {};
3758     $usb_dev_features->{spice_usb3} = 1 if qemu_machine_feature_enabled($machine_type, $kvmver, 4, 0);
3759
3760     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES, $usb_dev_features);
3761     push @$devices, @usbdevices if @usbdevices;
3762     # serial devices
3763     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3764         if (my $path = $conf->{"serial$i"}) {
3765             if ($path eq 'socket') {
3766                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3767                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3768                 # On aarch64, serial0 is the UART device. Qemu only allows
3769                 # connecting UART devices via the '-serial' command line, as
3770                 # the device has a fixed slot on the hardware...
3771                 if ($arch eq 'aarch64' && $i == 0) {
3772                     push @$devices, '-serial', "chardev:serial$i";
3773                 } else {
3774                     push @$devices, '-device', "isa-serial,chardev=serial$i";
3775                 }
3776             } else {
3777                 die "no such serial device\n" if ! -c $path;
3778                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3779                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3780             }
3781         }
3782     }
3783
3784     # parallel devices
3785     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3786         if (my $path = $conf->{"parallel$i"}) {
3787             die "no such parallel device\n" if ! -c $path;
3788             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3789             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3790             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3791         }
3792     }
3793
3794     if (my $audio = conf_has_audio($conf)) {
3795
3796         my $audiopciaddr = print_pci_addr("audio0", $bridges, $arch, $machine_type);
3797
3798         my $id = $audio->{dev_id};
3799         if ($audio->{dev} eq 'AC97') {
3800             push @$devices, '-device', "AC97,id=${id}${audiopciaddr}";
3801         } elsif ($audio->{dev} =~ /intel\-hda$/) {
3802             push @$devices, '-device', "$audio->{dev},id=${id}${audiopciaddr}";
3803             push @$devices, '-device', "hda-micro,id=${id}-codec0,bus=${id}.0,cad=0";
3804             push @$devices, '-device', "hda-duplex,id=${id}-codec1,bus=${id}.0,cad=1";
3805         } else {
3806             die "unkown audio device '$audio->{dev}', implement me!";
3807         }
3808
3809         push @$devices, '-audiodev', "$audio->{backend},id=$audio->{backend_id}";
3810     }
3811
3812     my $sockets = 1;
3813     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3814     $sockets = $conf->{sockets} if  $conf->{sockets};
3815
3816     my $cores = $conf->{cores} || 1;
3817
3818     my $maxcpus = $sockets * $cores;
3819
3820     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3821
3822     my $allowed_vcpus = $cpuinfo->{cpus};
3823
3824     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3825         if ($allowed_vcpus < $maxcpus);
3826
3827     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3828
3829         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3830         for (my $i = 2; $i <= $vcpus; $i++)  {
3831             my $cpustr = print_cpu_device($conf,$i);
3832             push @$cmd, '-device', $cpustr;
3833         }
3834
3835     } else {
3836
3837         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3838     }
3839     push @$cmd, '-nodefaults';
3840
3841     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3842
3843     my $bootindex_hash = {};
3844     my $i = 1;
3845     foreach my $o (split(//, $bootorder)) {
3846         $bootindex_hash->{$o} = $i*100;
3847         $i++;
3848     }
3849
3850     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3851
3852     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3853
3854     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3855
3856     if ($vga->{type} && $vga->{type} !~ m/^serial\d+$/ && $vga->{type} ne 'none'){
3857         push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, undef, $qxlnum, $bridges);
3858         my $socket = vnc_socket($vmid);
3859         push @$cmd,  '-vnc', "unix:$socket,password";
3860     } else {
3861         push @$cmd, '-vga', 'none' if $vga->{type} eq 'none';
3862         push @$cmd, '-nographic';
3863     }
3864
3865     # time drift fix
3866     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3867
3868     my $useLocaltime = $conf->{localtime};
3869
3870     if ($winversion >= 5) { # windows
3871         $useLocaltime = 1 if !defined($conf->{localtime});
3872
3873         # use time drift fix when acpi is enabled
3874         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3875             $tdf = 1 if !defined($conf->{tdf});
3876         }
3877     }
3878
3879     if ($winversion >= 6) {
3880         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3881         push @$cmd, '-no-hpet';
3882     }
3883
3884     push @$rtcFlags, 'driftfix=slew' if $tdf;
3885
3886     if (!$kvm) {
3887         push @$machineFlags, 'accel=tcg';
3888     }
3889
3890     if ($machine_type) {
3891         push @$machineFlags, "type=${machine_type}";
3892     }
3893
3894     if (($conf->{startdate}) && ($conf->{startdate} ne 'now')) {
3895         push @$rtcFlags, "base=$conf->{startdate}";
3896     } elsif ($useLocaltime) {
3897         push @$rtcFlags, 'base=localtime';
3898     }
3899
3900     push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
3901
3902     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3903
3904     push @$cmd, '-S' if $conf->{freeze};
3905
3906     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3907
3908     if (parse_guest_agent($conf)->{enabled}) {
3909         my $qgasocket = qmp_socket($vmid, 1);
3910         my $pciaddr = print_pci_addr("qga0", $bridges, $arch, $machine_type);
3911         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3912         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3913         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3914     }
3915
3916     my $spice_port;
3917
3918     if ($qxlnum) {
3919         if ($qxlnum > 1) {
3920             if ($winversion){
3921                 for(my $i = 1; $i < $qxlnum; $i++){
3922                     push @$devices, '-device', print_vga_device($conf, $vga, $arch, $machine_type, $i, $qxlnum, $bridges);
3923                 }
3924             } else {
3925                 # assume other OS works like Linux
3926                 my ($ram, $vram) = ("134217728", "67108864");
3927                 if ($vga->{memory}) {
3928                     $ram = PVE::Tools::convert_size($qxlnum*4*$vga->{memory}, 'mb' => 'b');
3929                     $vram = PVE::Tools::convert_size($qxlnum*2*$vga->{memory}, 'mb' => 'b');
3930                 }
3931                 push @$cmd, '-global', "qxl-vga.ram_size=$ram";
3932                 push @$cmd, '-global', "qxl-vga.vram_size=$vram";
3933             }
3934         }
3935
3936         my $pciaddr = print_pci_addr("spice", $bridges, $arch, $machine_type);
3937
3938         my $nodename = PVE::INotify::nodename();
3939         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3940         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3941         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3942
3943         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3944         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3945         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3946
3947         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3948         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3949
3950         my $spice_enhancement = PVE::JSONSchema::parse_property_string($spice_enhancements_fmt, $conf->{spice_enhancements} // '');
3951         if ($spice_enhancement->{foldersharing}) {
3952             push @$devices, '-chardev', "spiceport,id=foldershare,name=org.spice-space.webdav.0";
3953             push @$devices, '-device', "virtserialport,chardev=foldershare,name=org.spice-space.webdav.0";
3954         }
3955
3956         my $spice_opts = "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3957         $spice_opts .= ",streaming-video=$spice_enhancement->{videostreaming}" if $spice_enhancement->{videostreaming};
3958         push @$devices, '-spice', "$spice_opts";
3959     }
3960
3961     # enable balloon by default, unless explicitly disabled
3962     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3963         $pciaddr = print_pci_addr("balloon0", $bridges, $arch, $machine_type);
3964         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3965     }
3966
3967     if ($conf->{watchdog}) {
3968         my $wdopts = parse_watchdog($conf->{watchdog});
3969         $pciaddr = print_pci_addr("watchdog", $bridges, $arch, $machine_type);
3970         my $watchdog = $wdopts->{model} || 'i6300esb';
3971         push @$devices, '-device', "$watchdog$pciaddr";
3972         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3973     }
3974
3975     my $vollist = [];
3976     my $scsicontroller = {};
3977     my $ahcicontroller = {};
3978     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3979
3980     # Add iscsi initiator name if available
3981     if (my $initiator = get_initiator_name()) {
3982         push @$devices, '-iscsi', "initiator-name=$initiator";
3983     }
3984
3985     foreach_drive($conf, sub {
3986         my ($ds, $drive) = @_;
3987
3988         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3989             push @$vollist, $drive->{file};
3990         }
3991
3992         # ignore efidisk here, already added in bios/fw handling code above
3993         return if $drive->{interface} eq 'efidisk';
3994
3995         $use_virtio = 1 if $ds =~ m/^virtio/;
3996
3997         if (drive_is_cdrom ($drive)) {
3998             if ($bootindex_hash->{d}) {
3999                 $drive->{bootindex} = $bootindex_hash->{d};
4000                 $bootindex_hash->{d} += 1;
4001             }
4002         } else {
4003             if ($bootindex_hash->{c}) {
4004                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
4005                 $bootindex_hash->{c} += 1;
4006             }
4007         }
4008
4009         if($drive->{interface} eq 'virtio'){
4010            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
4011         }
4012
4013         if ($drive->{interface} eq 'scsi') {
4014
4015             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
4016
4017             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges, $arch, $machine_type);
4018             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
4019
4020             my $iothread = '';
4021             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
4022                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
4023                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
4024             } elsif ($drive->{iothread}) {
4025                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
4026             }
4027
4028             my $queues = '';
4029             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
4030                 $queues = ",num_queues=$drive->{queues}";
4031             }
4032
4033             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
4034             $scsicontroller->{$controller}=1;
4035         }
4036
4037         if ($drive->{interface} eq 'sata') {
4038            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
4039            $pciaddr = print_pci_addr("ahci$controller", $bridges, $arch, $machine_type);
4040            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
4041            $ahcicontroller->{$controller}=1;
4042         }
4043
4044         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
4045         push @$devices, '-drive',$drive_cmd;
4046         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges, $arch, $machine_type);
4047     });
4048
4049     for (my $i = 0; $i < $MAX_NETS; $i++) {
4050          next if !$conf->{"net$i"};
4051          my $d = parse_net($conf->{"net$i"});
4052          next if !$d;
4053
4054          $use_virtio = 1 if $d->{model} eq 'virtio';
4055
4056          if ($bootindex_hash->{n}) {
4057             $d->{bootindex} = $bootindex_hash->{n};
4058             $bootindex_hash->{n} += 1;
4059          }
4060
4061          my $netdevfull = print_netdev_full($vmid, $conf, $arch, $d, "net$i");
4062          push @$devices, '-netdev', $netdevfull;
4063
4064          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files, $arch, $machine_type);
4065          push @$devices, '-device', $netdevicefull;
4066     }
4067
4068     if ($conf->{ivshmem}) {
4069         my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
4070
4071         my $bus;
4072         if ($q35) {
4073             $bus = print_pcie_addr("ivshmem");
4074         } else {
4075             $bus = print_pci_addr("ivshmem", $bridges, $arch, $machine_type);
4076         }
4077
4078         my $ivshmem_name = $ivshmem->{name} // $vmid;
4079         my $path = '/dev/shm/pve-shm-' . $ivshmem_name;
4080
4081         push @$devices, '-device', "ivshmem-plain,memdev=ivshmem$bus,";
4082         push @$devices, '-object', "memory-backend-file,id=ivshmem,share=on,mem-path=$path,size=$ivshmem->{size}M";
4083     }
4084
4085     if (!$q35) {
4086         # add pci bridges
4087         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
4088            $bridges->{1} = 1;
4089            $bridges->{2} = 1;
4090         }
4091
4092         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
4093
4094         for my $k (sort {$b cmp $a} keys %$bridges) {
4095             $pciaddr = print_pci_addr("pci.$k", undef, $arch, $machine_type);
4096             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
4097         }
4098     }
4099
4100     push @$cmd, @$devices;
4101     push @$cmd, '-rtc', join(',', @$rtcFlags)
4102         if scalar(@$rtcFlags);
4103     push @$cmd, '-machine', join(',', @$machineFlags)
4104         if scalar(@$machineFlags);
4105     push @$cmd, '-global', join(',', @$globalFlags)
4106         if scalar(@$globalFlags);
4107
4108     if (my $vmstate = $conf->{vmstate}) {
4109         my $statepath = PVE::Storage::path($storecfg, $vmstate);
4110         push @$vollist, $vmstate;
4111         push @$cmd, '-loadstate', $statepath;
4112     }
4113
4114     # add custom args
4115     if ($conf->{args}) {
4116         my $aa = PVE::Tools::split_args($conf->{args});
4117         push @$cmd, @$aa;
4118     }
4119
4120     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
4121 }
4122
4123 sub vnc_socket {
4124     my ($vmid) = @_;
4125     return "${var_run_tmpdir}/$vmid.vnc";
4126 }
4127
4128 sub spice_port {
4129     my ($vmid) = @_;
4130
4131     my $res = vm_mon_cmd($vmid, 'query-spice');
4132
4133     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
4134 }
4135
4136 sub qmp_socket {
4137     my ($vmid, $qga) = @_;
4138     my $sockettype = $qga ? 'qga' : 'qmp';
4139     return "${var_run_tmpdir}/$vmid.$sockettype";
4140 }
4141
4142 sub pidfile_name {
4143     my ($vmid) = @_;
4144     return "${var_run_tmpdir}/$vmid.pid";
4145 }
4146
4147 sub vm_devices_list {
4148     my ($vmid) = @_;
4149
4150     my $res = vm_mon_cmd($vmid, 'query-pci');
4151     my $devices_to_check = [];
4152     my $devices = {};
4153     foreach my $pcibus (@$res) {
4154         push @$devices_to_check, @{$pcibus->{devices}},
4155     }
4156
4157     while (@$devices_to_check) {
4158         my $to_check = [];
4159         for my $d (@$devices_to_check) {
4160             $devices->{$d->{'qdev_id'}} = 1 if $d->{'qdev_id'};
4161             next if !$d->{'pci_bridge'};
4162
4163             $devices->{$d->{'qdev_id'}} += scalar(@{$d->{'pci_bridge'}->{devices}});
4164             push @$to_check, @{$d->{'pci_bridge'}->{devices}};
4165         }
4166         $devices_to_check = $to_check;
4167     }
4168
4169     my $resblock = vm_mon_cmd($vmid, 'query-block');
4170     foreach my $block (@$resblock) {
4171         if($block->{device} =~ m/^drive-(\S+)/){
4172                 $devices->{$1} = 1;
4173         }
4174     }
4175
4176     my $resmice = vm_mon_cmd($vmid, 'query-mice');
4177     foreach my $mice (@$resmice) {
4178         if ($mice->{name} eq 'QEMU HID Tablet') {
4179             $devices->{tablet} = 1;
4180             last;
4181         }
4182     }
4183
4184     # for usb devices there is no query-usb
4185     # but we can iterate over the entries in
4186     # qom-list path=/machine/peripheral
4187     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
4188     foreach my $per (@$resperipheral) {
4189         if ($per->{name} =~ m/^usb\d+$/) {
4190             $devices->{$per->{name}} = 1;
4191         }
4192     }
4193
4194     return $devices;
4195 }
4196
4197 sub vm_deviceplug {
4198     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4199
4200     my $q35 = machine_type_is_q35($conf);
4201
4202     my $devices_list = vm_devices_list($vmid);
4203     return 1 if defined($devices_list->{$deviceid});
4204
4205     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid, $arch, $machine_type); # add PCI bridge if we need it for the device
4206
4207     if ($deviceid eq 'tablet') {
4208
4209         qemu_deviceadd($vmid, print_tabletdevice_full($conf, $arch));
4210
4211     } elsif ($deviceid eq 'keyboard') {
4212
4213         qemu_deviceadd($vmid, print_keyboarddevice_full($conf, $arch));
4214
4215     } elsif ($deviceid =~ m/^usb(\d+)$/) {
4216
4217         die "usb hotplug currently not reliable\n";
4218         # since we can't reliably hot unplug all added usb devices
4219         # and usb passthrough disables live migration
4220         # we disable usb hotplugging for now
4221         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
4222
4223     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4224
4225         qemu_iothread_add($vmid, $deviceid, $device);
4226
4227         qemu_driveadd($storecfg, $vmid, $device);
4228         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4229
4230         qemu_deviceadd($vmid, $devicefull);
4231         eval { qemu_deviceaddverify($vmid, $deviceid); };
4232         if (my $err = $@) {
4233             eval { qemu_drivedel($vmid, $deviceid); };
4234             warn $@ if $@;
4235             die $err;
4236         }
4237
4238     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4239
4240
4241         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
4242         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4243         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
4244
4245         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
4246
4247         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
4248             qemu_iothread_add($vmid, $deviceid, $device);
4249             $devicefull .= ",iothread=iothread-$deviceid";
4250         }
4251
4252         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
4253             $devicefull .= ",num_queues=$device->{queues}";
4254         }
4255
4256         qemu_deviceadd($vmid, $devicefull);
4257         qemu_deviceaddverify($vmid, $deviceid);
4258
4259     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4260
4261         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device, $arch, $machine_type);
4262         qemu_driveadd($storecfg, $vmid, $device);
4263
4264         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device, $arch, $machine_type);
4265         eval { qemu_deviceadd($vmid, $devicefull); };
4266         if (my $err = $@) {
4267             eval { qemu_drivedel($vmid, $deviceid); };
4268             warn $@ if $@;
4269             die $err;
4270         }
4271
4272     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4273
4274         return undef if !qemu_netdevadd($vmid, $conf, $arch, $device, $deviceid);
4275
4276         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
4277         my $use_old_bios_files = undef;
4278         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
4279
4280         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files, $arch, $machine_type);
4281         qemu_deviceadd($vmid, $netdevicefull);
4282         eval {
4283             qemu_deviceaddverify($vmid, $deviceid);
4284             qemu_set_link_status($vmid, $deviceid, !$device->{link_down});
4285         };
4286         if (my $err = $@) {
4287             eval { qemu_netdevdel($vmid, $deviceid); };
4288             warn $@ if $@;
4289             die $err;
4290         }
4291
4292     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
4293
4294         my $bridgeid = $2;
4295         my $pciaddr = print_pci_addr($deviceid, undef, $arch, $machine_type);
4296         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
4297
4298         qemu_deviceadd($vmid, $devicefull);
4299         qemu_deviceaddverify($vmid, $deviceid);
4300
4301     } else {
4302         die "can't hotplug device '$deviceid'\n";
4303     }
4304
4305     return 1;
4306 }
4307
4308 # fixme: this should raise exceptions on error!
4309 sub vm_deviceunplug {
4310     my ($vmid, $conf, $deviceid) = @_;
4311
4312     my $devices_list = vm_devices_list($vmid);
4313     return 1 if !defined($devices_list->{$deviceid});
4314
4315     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
4316
4317     if ($deviceid eq 'tablet' || $deviceid eq 'keyboard') {
4318
4319         qemu_devicedel($vmid, $deviceid);
4320
4321     } elsif ($deviceid =~ m/^usb\d+$/) {
4322
4323         die "usb hotplug currently not reliable\n";
4324         # when unplugging usb devices this way,
4325         # there may be remaining usb controllers/hubs
4326         # so we disable it for now
4327         qemu_devicedel($vmid, $deviceid);
4328         qemu_devicedelverify($vmid, $deviceid);
4329
4330     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
4331
4332         qemu_devicedel($vmid, $deviceid);
4333         qemu_devicedelverify($vmid, $deviceid);
4334         qemu_drivedel($vmid, $deviceid);
4335         qemu_iothread_del($conf, $vmid, $deviceid);
4336
4337     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
4338
4339         qemu_devicedel($vmid, $deviceid);
4340         qemu_devicedelverify($vmid, $deviceid);
4341         qemu_iothread_del($conf, $vmid, $deviceid);
4342
4343     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
4344
4345         qemu_devicedel($vmid, $deviceid);
4346         qemu_drivedel($vmid, $deviceid);
4347         qemu_deletescsihw($conf, $vmid, $deviceid);
4348
4349     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
4350
4351         qemu_devicedel($vmid, $deviceid);
4352         qemu_devicedelverify($vmid, $deviceid);
4353         qemu_netdevdel($vmid, $deviceid);
4354
4355     } else {
4356         die "can't unplug device '$deviceid'\n";
4357     }
4358
4359     return 1;
4360 }
4361
4362 sub qemu_deviceadd {
4363     my ($vmid, $devicefull) = @_;
4364
4365     $devicefull = "driver=".$devicefull;
4366     my %options =  split(/[=,]/, $devicefull);
4367
4368     vm_mon_cmd($vmid, "device_add" , %options);
4369 }
4370
4371 sub qemu_devicedel {
4372     my ($vmid, $deviceid) = @_;
4373
4374     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
4375 }
4376
4377 sub qemu_iothread_add {
4378     my($vmid, $deviceid, $device) = @_;
4379
4380     if ($device->{iothread}) {
4381         my $iothreads = vm_iothreads_list($vmid);
4382         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
4383     }
4384 }
4385
4386 sub qemu_iothread_del {
4387     my($conf, $vmid, $deviceid) = @_;
4388
4389     my $confid = $deviceid;
4390     if ($deviceid =~ m/^(?:virtioscsi|scsihw)(\d+)$/) {
4391         $confid = 'scsi' . $1;
4392     }
4393     my $device = parse_drive($confid, $conf->{$confid});
4394     if ($device->{iothread}) {
4395         my $iothreads = vm_iothreads_list($vmid);
4396         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
4397     }
4398 }
4399
4400 sub qemu_objectadd {
4401     my($vmid, $objectid, $qomtype) = @_;
4402
4403     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
4404
4405     return 1;
4406 }
4407
4408 sub qemu_objectdel {
4409     my($vmid, $objectid) = @_;
4410
4411     vm_mon_cmd($vmid, "object-del", id => $objectid);
4412
4413     return 1;
4414 }
4415
4416 sub qemu_driveadd {
4417     my ($storecfg, $vmid, $device) = @_;
4418
4419     my $drive = print_drive_full($storecfg, $vmid, $device);
4420     $drive =~ s/\\/\\\\/g;
4421     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
4422
4423     # If the command succeeds qemu prints: "OK"
4424     return 1 if $ret =~ m/OK/s;
4425
4426     die "adding drive failed: $ret\n";
4427 }
4428
4429 sub qemu_drivedel {
4430     my($vmid, $deviceid) = @_;
4431
4432     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
4433     $ret =~ s/^\s+//;
4434
4435     return 1 if $ret eq "";
4436
4437     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
4438     return 1 if $ret =~ m/Device \'.*?\' not found/s;
4439
4440     die "deleting drive $deviceid failed : $ret\n";
4441 }
4442
4443 sub qemu_deviceaddverify {
4444     my ($vmid, $deviceid) = @_;
4445
4446     for (my $i = 0; $i <= 5; $i++) {
4447          my $devices_list = vm_devices_list($vmid);
4448          return 1 if defined($devices_list->{$deviceid});
4449          sleep 1;
4450     }
4451
4452     die "error on hotplug device '$deviceid'\n";
4453 }
4454
4455
4456 sub qemu_devicedelverify {
4457     my ($vmid, $deviceid) = @_;
4458
4459     # need to verify that the device is correctly removed as device_del
4460     # is async and empty return is not reliable
4461
4462     for (my $i = 0; $i <= 5; $i++) {
4463          my $devices_list = vm_devices_list($vmid);
4464          return 1 if !defined($devices_list->{$deviceid});
4465          sleep 1;
4466     }
4467
4468     die "error on hot-unplugging device '$deviceid'\n";
4469 }
4470
4471 sub qemu_findorcreatescsihw {
4472     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4473
4474     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4475
4476     my $scsihwid="$controller_prefix$controller";
4477     my $devices_list = vm_devices_list($vmid);
4478
4479     if(!defined($devices_list->{$scsihwid})) {
4480         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device, $arch, $machine_type);
4481     }
4482
4483     return 1;
4484 }
4485
4486 sub qemu_deletescsihw {
4487     my ($conf, $vmid, $opt) = @_;
4488
4489     my $device = parse_drive($opt, $conf->{$opt});
4490
4491     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
4492         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
4493         return 1;
4494     }
4495
4496     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
4497
4498     my $devices_list = vm_devices_list($vmid);
4499     foreach my $opt (keys %{$devices_list}) {
4500         if (PVE::QemuServer::is_valid_drivename($opt)) {
4501             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
4502             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
4503                 return 1;
4504             }
4505         }
4506     }
4507
4508     my $scsihwid="scsihw$controller";
4509
4510     vm_deviceunplug($vmid, $conf, $scsihwid);
4511
4512     return 1;
4513 }
4514
4515 sub qemu_add_pci_bridge {
4516     my ($storecfg, $conf, $vmid, $device, $arch, $machine_type) = @_;
4517
4518     my $bridges = {};
4519
4520     my $bridgeid;
4521
4522     print_pci_addr($device, $bridges, $arch, $machine_type);
4523
4524     while (my ($k, $v) = each %$bridges) {
4525         $bridgeid = $k;
4526     }
4527     return 1 if !defined($bridgeid) || $bridgeid < 1;
4528
4529     my $bridge = "pci.$bridgeid";
4530     my $devices_list = vm_devices_list($vmid);
4531
4532     if (!defined($devices_list->{$bridge})) {
4533         vm_deviceplug($storecfg, $conf, $vmid, $bridge, $arch, $machine_type);
4534     }
4535
4536     return 1;
4537 }
4538
4539 sub qemu_set_link_status {
4540     my ($vmid, $device, $up) = @_;
4541
4542     vm_mon_cmd($vmid, "set_link", name => $device,
4543                up => $up ? JSON::true : JSON::false);
4544 }
4545
4546 sub qemu_netdevadd {
4547     my ($vmid, $conf, $arch, $device, $deviceid) = @_;
4548
4549     my $netdev = print_netdev_full($vmid, $conf, $arch, $device, $deviceid, 1);
4550     my %options =  split(/[=,]/, $netdev);
4551
4552     vm_mon_cmd($vmid, "netdev_add",  %options);
4553     return 1;
4554 }
4555
4556 sub qemu_netdevdel {
4557     my ($vmid, $deviceid) = @_;
4558
4559     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4560 }
4561
4562 sub qemu_usb_hotplug {
4563     my ($storecfg, $conf, $vmid, $deviceid, $device, $arch, $machine_type) = @_;
4564
4565     return if !$device;
4566
4567     # remove the old one first
4568     vm_deviceunplug($vmid, $conf, $deviceid);
4569
4570     # check if xhci controller is necessary and available
4571     if ($device->{usb3}) {
4572
4573         my $devicelist = vm_devices_list($vmid);
4574
4575         if (!$devicelist->{xhci}) {
4576             my $pciaddr = print_pci_addr("xhci", undef, $arch, $machine_type);
4577             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4578         }
4579     }
4580     my $d = parse_usb_device($device->{host});
4581     $d->{usb3} = $device->{usb3};
4582
4583     # add the new one
4584     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d, $arch, $machine_type);
4585 }
4586
4587 sub qemu_cpu_hotplug {
4588     my ($vmid, $conf, $vcpus) = @_;
4589
4590     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4591
4592     my $sockets = 1;
4593     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4594     $sockets = $conf->{sockets} if  $conf->{sockets};
4595     my $cores = $conf->{cores} || 1;
4596     my $maxcpus = $sockets * $cores;
4597
4598     $vcpus = $maxcpus if !$vcpus;
4599
4600     die "you can't add more vcpus than maxcpus\n"
4601         if $vcpus > $maxcpus;
4602
4603     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4604
4605     if ($vcpus < $currentvcpus) {
4606
4607         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4608
4609             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4610                 qemu_devicedel($vmid, "cpu$i");
4611                 my $retry = 0;
4612                 my $currentrunningvcpus = undef;
4613                 while (1) {
4614                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4615                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4616                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4617                     $retry++;
4618                     sleep 1;
4619                 }
4620                 #update conf after each succesfull cpu unplug
4621                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4622                 PVE::QemuConfig->write_config($vmid, $conf);
4623             }
4624         } else {
4625             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4626         }
4627
4628         return;
4629     }
4630
4631     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4632     die "vcpus in running vm does not match its configuration\n"
4633         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4634
4635     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4636
4637         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4638             my $cpustr = print_cpu_device($conf, $i);
4639             qemu_deviceadd($vmid, $cpustr);
4640
4641             my $retry = 0;
4642             my $currentrunningvcpus = undef;
4643             while (1) {
4644                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4645                 last if scalar(@{$currentrunningvcpus}) == $i;
4646                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4647                 sleep 1;
4648                 $retry++;
4649             }
4650             #update conf after each succesfull cpu hotplug
4651             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4652             PVE::QemuConfig->write_config($vmid, $conf);
4653         }
4654     } else {
4655
4656         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4657             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4658         }
4659     }
4660 }
4661
4662 sub qemu_block_set_io_throttle {
4663     my ($vmid, $deviceid,
4664         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4665         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4666         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4667         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4668
4669     return if !check_running($vmid) ;
4670
4671     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4672         bps => int($bps),
4673         bps_rd => int($bps_rd),
4674         bps_wr => int($bps_wr),
4675         iops => int($iops),
4676         iops_rd => int($iops_rd),
4677         iops_wr => int($iops_wr),
4678         bps_max => int($bps_max),
4679         bps_rd_max => int($bps_rd_max),
4680         bps_wr_max => int($bps_wr_max),
4681         iops_max => int($iops_max),
4682         iops_rd_max => int($iops_rd_max),
4683         iops_wr_max => int($iops_wr_max),
4684         bps_max_length => int($bps_max_length),
4685         bps_rd_max_length => int($bps_rd_max_length),
4686         bps_wr_max_length => int($bps_wr_max_length),
4687         iops_max_length => int($iops_max_length),
4688         iops_rd_max_length => int($iops_rd_max_length),
4689         iops_wr_max_length => int($iops_wr_max_length),
4690     );
4691
4692 }
4693
4694 # old code, only used to shutdown old VM after update
4695 sub __read_avail {
4696     my ($fh, $timeout) = @_;
4697
4698     my $sel = new IO::Select;
4699     $sel->add($fh);
4700
4701     my $res = '';
4702     my $buf;
4703
4704     my @ready;
4705     while (scalar (@ready = $sel->can_read($timeout))) {
4706         my $count;
4707         if ($count = $fh->sysread($buf, 8192)) {
4708             if ($buf =~ /^(.*)\(qemu\) $/s) {
4709                 $res .= $1;
4710                 last;
4711             } else {
4712                 $res .= $buf;
4713             }
4714         } else {
4715             if (!defined($count)) {
4716                 die "$!\n";
4717             }
4718             last;
4719         }
4720     }
4721
4722     die "monitor read timeout\n" if !scalar(@ready);
4723
4724     return $res;
4725 }
4726
4727 sub qemu_block_resize {
4728     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4729
4730     my $running = check_running($vmid);
4731
4732     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4733
4734     return if !$running;
4735
4736     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4737
4738 }
4739
4740 sub qemu_volume_snapshot {
4741     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4742
4743     my $running = check_running($vmid);
4744
4745     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4746         vm_mon_cmd($vmid, 'blockdev-snapshot-internal-sync', device => $deviceid, name => $snap);
4747     } else {
4748         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4749     }
4750 }
4751
4752 sub qemu_volume_snapshot_delete {
4753     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4754
4755     my $running = check_running($vmid);
4756
4757     if($running) {
4758
4759         $running = undef;
4760         my $conf = PVE::QemuConfig->load_config($vmid);
4761         foreach_drive($conf, sub {
4762             my ($ds, $drive) = @_;
4763             $running = 1 if $drive->{file} eq $volid;
4764         });
4765     }
4766
4767     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4768         vm_mon_cmd($vmid, 'blockdev-snapshot-delete-internal-sync', device => $deviceid, name => $snap);
4769     } else {
4770         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4771     }
4772 }
4773
4774 sub set_migration_caps {
4775     my ($vmid) = @_;
4776
4777     my $cap_ref = [];
4778
4779     my $enabled_cap = {
4780         "auto-converge" => 1,
4781         "xbzrle" => 1,
4782         "x-rdma-pin-all" => 0,
4783         "zero-blocks" => 0,
4784         "compress" => 0
4785     };
4786
4787     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4788
4789     for my $supported_capability (@$supported_capabilities) {
4790         push @$cap_ref, {
4791             capability => $supported_capability->{capability},
4792             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4793         };
4794     }
4795
4796     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4797 }
4798
4799 my $fast_plug_option = {
4800     'lock' => 1,
4801     'name' => 1,
4802     'onboot' => 1,
4803     'shares' => 1,
4804     'startup' => 1,
4805     'description' => 1,
4806     'protection' => 1,
4807     'vmstatestorage' => 1,
4808     'hookscript' => 1,
4809 };
4810
4811 # hotplug changes in [PENDING]
4812 # $selection hash can be used to only apply specified options, for
4813 # example: { cores => 1 } (only apply changed 'cores')
4814 # $errors ref is used to return error messages
4815 sub vmconfig_hotplug_pending {
4816     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4817
4818     my $defaults = load_defaults();
4819     my ($arch, $machine_type) = get_basic_machine_info($conf, undef);
4820
4821     # commit values which do not have any impact on running VM first
4822     # Note: those option cannot raise errors, we we do not care about
4823     # $selection and always apply them.
4824
4825     my $add_error = sub {
4826         my ($opt, $msg) = @_;
4827         $errors->{$opt} = "hotplug problem - $msg";
4828     };
4829
4830     my $changes = 0;
4831     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4832         if ($fast_plug_option->{$opt}) {
4833             $conf->{$opt} = $conf->{pending}->{$opt};
4834             delete $conf->{pending}->{$opt};
4835             $changes = 1;
4836         }
4837     }
4838
4839     if ($changes) {
4840         PVE::QemuConfig->write_config($vmid, $conf);
4841         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4842     }
4843
4844     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4845
4846     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
4847     foreach my $opt (sort keys %$pending_delete_hash) {
4848         next if $selection && !$selection->{$opt};
4849         my $force = $pending_delete_hash->{$opt}->{force};
4850         eval {
4851             if ($opt eq 'hotplug') {
4852                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4853             } elsif ($opt eq 'tablet') {
4854                 die "skip\n" if !$hotplug_features->{usb};
4855                 if ($defaults->{tablet}) {
4856                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4857                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4858                         if $arch eq 'aarch64';
4859                 } else {
4860                     vm_deviceunplug($vmid, $conf, 'tablet');
4861                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4862                 }
4863             } elsif ($opt =~ m/^usb\d+/) {
4864                 die "skip\n";
4865                 # since we cannot reliably hot unplug usb devices
4866                 # we are disabling it
4867                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4868                 vm_deviceunplug($vmid, $conf, $opt);
4869             } elsif ($opt eq 'vcpus') {
4870                 die "skip\n" if !$hotplug_features->{cpu};
4871                 qemu_cpu_hotplug($vmid, $conf, undef);
4872             } elsif ($opt eq 'balloon') {
4873                 # enable balloon device is not hotpluggable
4874                 die "skip\n" if defined($conf->{balloon}) && $conf->{balloon} == 0;
4875                 # here we reset the ballooning value to memory
4876                 my $balloon = $conf->{memory} || $defaults->{memory};
4877                 vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4878             } elsif ($fast_plug_option->{$opt}) {
4879                 # do nothing
4880             } elsif ($opt =~ m/^net(\d+)$/) {
4881                 die "skip\n" if !$hotplug_features->{network};
4882                 vm_deviceunplug($vmid, $conf, $opt);
4883             } elsif (is_valid_drivename($opt)) {
4884                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4885                 vm_deviceunplug($vmid, $conf, $opt);
4886                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4887             } elsif ($opt =~ m/^memory$/) {
4888                 die "skip\n" if !$hotplug_features->{memory};
4889                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4890             } elsif ($opt eq 'cpuunits') {
4891                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4892             } elsif ($opt eq 'cpulimit') {
4893                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4894             } else {
4895                 die "skip\n";
4896             }
4897         };
4898         if (my $err = $@) {
4899             &$add_error($opt, $err) if $err ne "skip\n";
4900         } else {
4901             # save new config if hotplug was successful
4902             delete $conf->{$opt};
4903             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
4904             PVE::QemuConfig->write_config($vmid, $conf);
4905             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4906         }
4907     }
4908
4909     my $apply_pending_cloudinit;
4910     $apply_pending_cloudinit = sub {
4911         my ($key, $value) = @_;
4912         $apply_pending_cloudinit = sub {}; # once is enough
4913
4914         my @cloudinit_opts = keys %$confdesc_cloudinit;
4915         foreach my $opt (keys %{$conf->{pending}}) {
4916             next if !grep { $_ eq $opt } @cloudinit_opts;
4917             $conf->{$opt} = delete $conf->{pending}->{$opt};
4918         }
4919
4920         my $new_conf = { %$conf };
4921         $new_conf->{$key} = $value;
4922         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4923     };
4924
4925     foreach my $opt (keys %{$conf->{pending}}) {
4926         next if $selection && !$selection->{$opt};
4927         my $value = $conf->{pending}->{$opt};
4928         eval {
4929             if ($opt eq 'hotplug') {
4930                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4931             } elsif ($opt eq 'tablet') {
4932                 die "skip\n" if !$hotplug_features->{usb};
4933                 if ($value == 1) {
4934                     vm_deviceplug($storecfg, $conf, $vmid, 'tablet', $arch, $machine_type);
4935                     vm_deviceplug($storecfg, $conf, $vmid, 'keyboard', $arch, $machine_type)
4936                         if $arch eq 'aarch64';
4937                 } elsif ($value == 0) {
4938                     vm_deviceunplug($vmid, $conf, 'tablet');
4939                     vm_deviceunplug($vmid, $conf, 'keyboard') if $arch eq 'aarch64';
4940                 }
4941             } elsif ($opt =~ m/^usb\d+$/) {
4942                 die "skip\n";
4943                 # since we cannot reliably hot unplug usb devices
4944                 # we are disabling it
4945                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4946                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4947                 die "skip\n" if !$d;
4948                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d, $arch, $machine_type);
4949             } elsif ($opt eq 'vcpus') {
4950                 die "skip\n" if !$hotplug_features->{cpu};
4951                 qemu_cpu_hotplug($vmid, $conf, $value);
4952             } elsif ($opt eq 'balloon') {
4953                 # enable/disable balloning device is not hotpluggable
4954                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4955                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4956                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4957
4958                 # allow manual ballooning if shares is set to zero
4959                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4960                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4961                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4962                 }
4963             } elsif ($opt =~ m/^net(\d+)$/) {
4964                 # some changes can be done without hotplug
4965                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4966                                     $vmid, $opt, $value, $arch, $machine_type);
4967             } elsif (is_valid_drivename($opt)) {
4968                 # some changes can be done without hotplug
4969                 my $drive = parse_drive($opt, $value);
4970                 if (drive_is_cloudinit($drive)) {
4971                     &$apply_pending_cloudinit($opt, $value);
4972                 }
4973                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4974                                      $vmid, $opt, $value, 1, $arch, $machine_type);
4975             } elsif ($opt =~ m/^memory$/) { #dimms
4976                 die "skip\n" if !$hotplug_features->{memory};
4977                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4978             } elsif ($opt eq 'cpuunits') {
4979                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4980             } elsif ($opt eq 'cpulimit') {
4981                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4982                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4983             } else {
4984                 die "skip\n";  # skip non-hot-pluggable options
4985             }
4986         };
4987         if (my $err = $@) {
4988             &$add_error($opt, $err) if $err ne "skip\n";
4989         } else {
4990             # save new config if hotplug was successful
4991             $conf->{$opt} = $value;
4992             delete $conf->{pending}->{$opt};
4993             PVE::QemuConfig->write_config($vmid, $conf);
4994             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4995         }
4996     }
4997 }
4998
4999 sub try_deallocate_drive {
5000     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
5001
5002     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
5003         my $volid = $drive->{file};
5004         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
5005             my $sid = PVE::Storage::parse_volume_id($volid);
5006             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
5007
5008             # check if the disk is really unused
5009             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
5010                 if is_volume_in_use($storecfg, $conf, $key, $volid);
5011             PVE::Storage::vdisk_free($storecfg, $volid);
5012             return 1;
5013         } else {
5014             # If vm is not owner of this disk remove from config
5015             return 1;
5016         }
5017     }
5018
5019     return undef;
5020 }
5021
5022 sub vmconfig_delete_or_detach_drive {
5023     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
5024
5025     my $drive = parse_drive($opt, $conf->{$opt});
5026
5027     my $rpcenv = PVE::RPCEnvironment::get();
5028     my $authuser = $rpcenv->get_user();
5029
5030     if ($force) {
5031         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
5032         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
5033     } else {
5034         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
5035     }
5036 }
5037
5038
5039
5040 sub vmconfig_apply_pending {
5041     my ($vmid, $conf, $storecfg) = @_;
5042
5043     # cold plug
5044
5045     my $pending_delete_hash = PVE::QemuConfig->parse_pending_delete($conf->{pending}->{delete});
5046     foreach my $opt (sort keys %$pending_delete_hash) {
5047         die "internal error" if $opt =~ m/^unused/;
5048         my $force = $pending_delete_hash->{$opt}->{force};
5049         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5050         if (!defined($conf->{$opt})) {
5051             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5052             PVE::QemuConfig->write_config($vmid, $conf);
5053         } elsif (is_valid_drivename($opt)) {
5054             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
5055             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5056             delete $conf->{$opt};
5057             PVE::QemuConfig->write_config($vmid, $conf);
5058         } else {
5059             PVE::QemuConfig->remove_from_pending_delete($conf, $opt);
5060             delete $conf->{$opt};
5061             PVE::QemuConfig->write_config($vmid, $conf);
5062         }
5063     }
5064
5065     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5066
5067     foreach my $opt (keys %{$conf->{pending}}) { # add/change
5068         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5069
5070         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
5071             # skip if nothing changed
5072         } elsif (is_valid_drivename($opt)) {
5073             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
5074                 if defined($conf->{$opt});
5075             $conf->{$opt} = $conf->{pending}->{$opt};
5076         } else {
5077             $conf->{$opt} = $conf->{pending}->{$opt};
5078         }
5079
5080         delete $conf->{pending}->{$opt};
5081         PVE::QemuConfig->write_config($vmid, $conf);
5082     }
5083 }
5084
5085 my $safe_num_ne = sub {
5086     my ($a, $b) = @_;
5087
5088     return 0 if !defined($a) && !defined($b);
5089     return 1 if !defined($a);
5090     return 1 if !defined($b);
5091
5092     return $a != $b;
5093 };
5094
5095 my $safe_string_ne = sub {
5096     my ($a, $b) = @_;
5097
5098     return 0 if !defined($a) && !defined($b);
5099     return 1 if !defined($a);
5100     return 1 if !defined($b);
5101
5102     return $a ne $b;
5103 };
5104
5105 sub vmconfig_update_net {
5106     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $arch, $machine_type) = @_;
5107
5108     my $newnet = parse_net($value);
5109
5110     if ($conf->{$opt}) {
5111         my $oldnet = parse_net($conf->{$opt});
5112
5113         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
5114             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
5115             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
5116             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
5117
5118             # for non online change, we try to hot-unplug
5119             die "skip\n" if !$hotplug;
5120             vm_deviceunplug($vmid, $conf, $opt);
5121         } else {
5122
5123             die "internal error" if $opt !~ m/net(\d+)/;
5124             my $iface = "tap${vmid}i$1";
5125
5126             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
5127                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
5128                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
5129                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
5130                 PVE::Network::tap_unplug($iface);
5131                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
5132             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
5133                 # Rate can be applied on its own but any change above needs to
5134                 # include the rate in tap_plug since OVS resets everything.
5135                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
5136             }
5137
5138             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
5139                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
5140             }
5141
5142             return 1;
5143         }
5144     }
5145
5146     if ($hotplug) {
5147         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet, $arch, $machine_type);
5148     } else {
5149         die "skip\n";
5150     }
5151 }
5152
5153 sub vmconfig_update_disk {
5154     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force, $arch, $machine_type) = @_;
5155
5156     # fixme: do we need force?
5157
5158     my $drive = parse_drive($opt, $value);
5159
5160     if ($conf->{$opt}) {
5161
5162         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
5163
5164             my $media = $drive->{media} || 'disk';
5165             my $oldmedia = $old_drive->{media} || 'disk';
5166             die "unable to change media type\n" if $media ne $oldmedia;
5167
5168             if (!drive_is_cdrom($old_drive)) {
5169
5170                 if ($drive->{file} ne $old_drive->{file}) {
5171
5172                     die "skip\n" if !$hotplug;
5173
5174                     # unplug and register as unused
5175                     vm_deviceunplug($vmid, $conf, $opt);
5176                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
5177
5178                 } else {
5179                     # update existing disk
5180
5181                     # skip non hotpluggable value
5182                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
5183                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
5184                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
5185                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
5186                         die "skip\n";
5187                     }
5188
5189                     # apply throttle
5190                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
5191                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
5192                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
5193                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
5194                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
5195                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
5196                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
5197                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
5198                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
5199                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
5200                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
5201                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
5202                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
5203                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
5204                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
5205                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
5206                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
5207                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
5208
5209                         qemu_block_set_io_throttle($vmid,"drive-$opt",
5210                                                    ($drive->{mbps} || 0)*1024*1024,
5211                                                    ($drive->{mbps_rd} || 0)*1024*1024,
5212                                                    ($drive->{mbps_wr} || 0)*1024*1024,
5213                                                    $drive->{iops} || 0,
5214                                                    $drive->{iops_rd} || 0,
5215                                                    $drive->{iops_wr} || 0,
5216                                                    ($drive->{mbps_max} || 0)*1024*1024,
5217                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
5218                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
5219                                                    $drive->{iops_max} || 0,
5220                                                    $drive->{iops_rd_max} || 0,
5221                                                    $drive->{iops_wr_max} || 0,
5222                                                    $drive->{bps_max_length} || 1,
5223                                                    $drive->{bps_rd_max_length} || 1,
5224                                                    $drive->{bps_wr_max_length} || 1,
5225                                                    $drive->{iops_max_length} || 1,
5226                                                    $drive->{iops_rd_max_length} || 1,
5227                                                    $drive->{iops_wr_max_length} || 1);
5228
5229                     }
5230
5231                     return 1;
5232                 }
5233
5234             } else { # cdrom
5235
5236                 if ($drive->{file} eq 'none') {
5237                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
5238                     if (drive_is_cloudinit($old_drive)) {
5239                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
5240                     }
5241                 } else {
5242                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
5243                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
5244                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
5245                 }
5246
5247                 return 1;
5248             }
5249         }
5250     }
5251
5252     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
5253     # hotplug new disks
5254     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
5255     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive, $arch, $machine_type);
5256 }
5257
5258 sub vm_start {
5259     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
5260         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
5261
5262     PVE::QemuConfig->lock_config($vmid, sub {
5263         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5264
5265         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
5266
5267         my $is_suspended = PVE::QemuConfig->has_lock($conf, 'suspended');
5268
5269         PVE::QemuConfig->check_lock($conf)
5270             if !($skiplock || $is_suspended);
5271
5272         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
5273
5274         # clean up leftover reboot request files
5275         eval { clear_reboot_request($vmid); };
5276         warn $@ if $@;
5277
5278         if (!$statefile && scalar(keys %{$conf->{pending}})) {
5279             vmconfig_apply_pending($vmid, $conf, $storecfg);
5280             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
5281         }
5282
5283         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
5284
5285         my $defaults = load_defaults();
5286
5287         # set environment variable useful inside network script
5288         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
5289
5290         my $local_volumes = {};
5291
5292         if ($targetstorage) {
5293             foreach_drive($conf, sub {
5294                 my ($ds, $drive) = @_;
5295
5296                 return if drive_is_cdrom($drive);
5297
5298                 my $volid = $drive->{file};
5299
5300                 return if !$volid;
5301
5302                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
5303
5304                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5305                 return if $scfg->{shared};
5306                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
5307             });
5308
5309             my $format = undef;
5310
5311             foreach my $opt (sort keys %$local_volumes) {
5312
5313                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
5314                 my $drive = parse_drive($opt, $conf->{$opt});
5315
5316                 #if remote storage is specified, use default format
5317                 if ($targetstorage && $targetstorage ne "1") {
5318                     $storeid = $targetstorage;
5319                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5320                     $format = $defFormat;
5321                 } else {
5322                     #else we use same format than original
5323                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5324                     $format = qemu_img_format($scfg, $volid);
5325                 }
5326
5327                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
5328                 my $newdrive = $drive;
5329                 $newdrive->{format} = $format;
5330                 $newdrive->{file} = $newvolid;
5331                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
5332                 $local_volumes->{$opt} = $drivestr;
5333                 #pass drive to conf for command line
5334                 $conf->{$opt} = $drivestr;
5335             }
5336         }
5337
5338         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-start', 1);
5339
5340         if ($is_suspended) {
5341             # enforce machine type on suspended vm to ensure HW compatibility
5342             $forcemachine = $conf->{runningmachine};
5343             print "Resuming suspended VM\n";
5344         }
5345
5346         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
5347
5348         my $migrate_uri;
5349         if ($statefile) {
5350             if ($statefile eq 'tcp') {
5351                 my $localip = "localhost";
5352                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
5353                 my $nodename = PVE::INotify::nodename();
5354
5355                 if (!defined($migration_type)) {
5356                     if (defined($datacenterconf->{migration}->{type})) {
5357                         $migration_type = $datacenterconf->{migration}->{type};
5358                     } else {
5359                         $migration_type = 'secure';
5360                     }
5361                 }
5362
5363                 if ($migration_type eq 'insecure') {
5364                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5365                     if ($migrate_network_addr) {
5366                         $localip = $migrate_network_addr;
5367                     } else {
5368                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
5369                     }
5370
5371                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5372                 }
5373
5374                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
5375                 my $migrate_port = PVE::Tools::next_migrate_port($pfamily);
5376                 $migrate_uri = "tcp:${localip}:${migrate_port}";
5377                 push @$cmd, '-incoming', $migrate_uri;
5378                 push @$cmd, '-S';
5379
5380             } elsif ($statefile eq 'unix') {
5381                 # should be default for secure migrations as a ssh TCP forward
5382                 # tunnel is not deterministic reliable ready and fails regurarly
5383                 # to set up in time, so use UNIX socket forwards
5384                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
5385                 unlink $socket_addr;
5386
5387                 $migrate_uri = "unix:$socket_addr";
5388
5389                 push @$cmd, '-incoming', $migrate_uri;
5390                 push @$cmd, '-S';
5391
5392             } elsif (-e $statefile) {
5393                 push @$cmd, '-loadstate', $statefile;
5394             } else {
5395                 my $statepath = PVE::Storage::path($storecfg, $statefile);
5396                 push @$vollist, $statefile;
5397                 push @$cmd, '-loadstate', $statepath;
5398             }
5399         } elsif ($paused) {
5400             push @$cmd, '-S';
5401         }
5402
5403         # host pci devices
5404         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
5405           my $d = parse_hostpci($conf->{"hostpci$i"});
5406           next if !$d;
5407           my $pcidevices = $d->{pciid};
5408           foreach my $pcidevice (@$pcidevices) {
5409                 my $pciid = $pcidevice->{id};
5410
5411                 my $info = PVE::SysFSTools::pci_device_info("0000:$pciid");
5412                 die "IOMMU not present\n" if !PVE::SysFSTools::check_iommu_support();
5413                 die "no pci device info for device '$pciid'\n" if !$info;
5414
5415                 if ($d->{mdev}) {
5416                     my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $i);
5417                     PVE::SysFSTools::pci_create_mdev_device($pciid, $uuid, $d->{mdev});
5418                 } else {
5419                     die "can't unbind/bind pci group to vfio '$pciid'\n"
5420                         if !PVE::SysFSTools::pci_dev_group_bind_to_vfio($pciid);
5421                     die "can't reset pci device '$pciid'\n"
5422                         if $info->{has_fl_reset} and !PVE::SysFSTools::pci_dev_reset($info);
5423                 }
5424           }
5425         }
5426
5427         PVE::Storage::activate_volumes($storecfg, $vollist);
5428
5429         eval {
5430             run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
5431                 outfunc => sub {}, errfunc => sub {});
5432         };
5433         # Issues with the above 'stop' not being fully completed are extremely rare, a very low
5434         # timeout should be more than enough here...
5435         PVE::Systemd::wait_for_unit_removed("$vmid.scope", 5);
5436
5437         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
5438                                                   : $defaults->{cpuunits};
5439
5440         my $start_timeout = ($conf->{hugepages} || $is_suspended) ? 300 : 30;
5441         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
5442
5443         my %properties = (
5444             Slice => 'qemu.slice',
5445             KillMode => 'none',
5446             CPUShares => $cpuunits
5447         );
5448
5449         if (my $cpulimit = $conf->{cpulimit}) {
5450             $properties{CPUQuota} = int($cpulimit * 100);
5451         }
5452         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
5453
5454         my $run_qemu = sub {
5455             PVE::Tools::run_fork sub {
5456                 PVE::Systemd::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
5457                 run_command($cmd, %run_params);
5458             };
5459         };
5460
5461         if ($conf->{hugepages}) {
5462
5463             my $code = sub {
5464                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
5465                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
5466
5467                 PVE::QemuServer::Memory::hugepages_mount();
5468                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
5469
5470                 eval { $run_qemu->() };
5471                 if (my $err = $@) {
5472                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
5473                     die $err;
5474                 }
5475
5476                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
5477             };
5478             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
5479
5480         } else {
5481             eval { $run_qemu->() };
5482         }
5483
5484         if (my $err = $@) {
5485             # deactivate volumes if start fails
5486             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
5487             die "start failed: $err";
5488         }
5489
5490         print "migration listens on $migrate_uri\n" if $migrate_uri;
5491
5492         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
5493             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
5494             warn $@ if $@;
5495         }
5496
5497         #start nbd server for storage migration
5498         if ($targetstorage) {
5499             my $nodename = PVE::INotify::nodename();
5500             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
5501             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
5502             my $pfamily = PVE::Tools::get_host_address_family($nodename);
5503             my $storage_migrate_port = PVE::Tools::next_migrate_port($pfamily);
5504
5505             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${storage_migrate_port}" } } );
5506
5507             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
5508
5509             foreach my $opt (sort keys %$local_volumes) {
5510                 my $volid = $local_volumes->{$opt};
5511                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
5512                 my $migrate_storage_uri = "nbd:${localip}:${storage_migrate_port}:exportname=drive-$opt";
5513                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
5514             }
5515         }
5516
5517         if ($migratedfrom) {
5518             eval {
5519                 set_migration_caps($vmid);
5520             };
5521             warn $@ if $@;
5522
5523             if ($spice_port) {
5524                 print "spice listens on port $spice_port\n";
5525                 if ($spice_ticket) {
5526                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
5527                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
5528                 }
5529             }
5530
5531         } else {
5532             vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
5533                 if !$statefile && $conf->{balloon};
5534
5535             foreach my $opt (keys %$conf) {
5536                 next if $opt !~  m/^net\d+$/;
5537                 my $nicconf = parse_net($conf->{$opt});
5538                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
5539             }
5540         }
5541
5542         vm_mon_cmd_nocheck($vmid, 'qom-set',
5543                     path => "machine/peripheral/balloon0",
5544                     property => "guest-stats-polling-interval",
5545                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
5546
5547         if ($is_suspended && (my $vmstate = $conf->{vmstate})) {
5548             print "Resumed VM, removing state\n";
5549             delete $conf->@{qw(lock vmstate runningmachine)};
5550             PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5551             PVE::Storage::vdisk_free($storecfg, $vmstate);
5552             PVE::QemuConfig->write_config($vmid, $conf);
5553         }
5554
5555         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'post-start');
5556     });
5557 }
5558
5559 sub vm_mon_cmd {
5560     my ($vmid, $execute, %params) = @_;
5561
5562     my $cmd = { execute => $execute, arguments => \%params };
5563     vm_qmp_command($vmid, $cmd);
5564 }
5565
5566 sub vm_mon_cmd_nocheck {
5567     my ($vmid, $execute, %params) = @_;
5568
5569     my $cmd = { execute => $execute, arguments => \%params };
5570     vm_qmp_command($vmid, $cmd, 1);
5571 }
5572
5573 sub vm_qmp_command {
5574     my ($vmid, $cmd, $nocheck) = @_;
5575
5576     my $res;
5577
5578     my $timeout;
5579     if ($cmd->{arguments}) {
5580         $timeout = delete $cmd->{arguments}->{timeout};
5581     }
5582
5583     eval {
5584         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
5585         my $sname = qmp_socket($vmid);
5586         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
5587             my $qmpclient = PVE::QMPClient->new();
5588
5589             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
5590         } else {
5591             die "unable to open monitor socket\n";
5592         }
5593     };
5594     if (my $err = $@) {
5595         syslog("err", "VM $vmid qmp command failed - $err");
5596         die $err;
5597     }
5598
5599     return $res;
5600 }
5601
5602 sub vm_human_monitor_command {
5603     my ($vmid, $cmdline) = @_;
5604
5605     my $cmd = {
5606         execute => 'human-monitor-command',
5607         arguments => { 'command-line' => $cmdline},
5608     };
5609
5610     return vm_qmp_command($vmid, $cmd);
5611 }
5612
5613 sub vm_commandline {
5614     my ($storecfg, $vmid, $snapname) = @_;
5615
5616     my $conf = PVE::QemuConfig->load_config($vmid);
5617
5618     if ($snapname) {
5619         my $snapshot = $conf->{snapshots}->{$snapname};
5620         die "snapshot '$snapname' does not exist\n" if !defined($snapshot);
5621
5622         $snapshot->{digest} = $conf->{digest}; # keep file digest for API
5623
5624         $conf = $snapshot;
5625     }
5626
5627     my $defaults = load_defaults();
5628
5629     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5630
5631     return PVE::Tools::cmd2string($cmd);
5632 }
5633
5634 sub vm_reset {
5635     my ($vmid, $skiplock) = @_;
5636
5637     PVE::QemuConfig->lock_config($vmid, sub {
5638
5639         my $conf = PVE::QemuConfig->load_config($vmid);
5640
5641         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5642
5643         vm_mon_cmd($vmid, "system_reset");
5644     });
5645 }
5646
5647 sub get_vm_volumes {
5648     my ($conf) = @_;
5649
5650     my $vollist = [];
5651     foreach_volid($conf, sub {
5652         my ($volid, $attr) = @_;
5653
5654         return if $volid =~ m|^/|;
5655
5656         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5657         return if !$sid;
5658
5659         push @$vollist, $volid;
5660     });
5661
5662     return $vollist;
5663 }
5664
5665 sub vm_stop_cleanup {
5666     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5667
5668     eval {
5669
5670         if (!$keepActive) {
5671             my $vollist = get_vm_volumes($conf);
5672             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5673         }
5674
5675         foreach my $ext (qw(mon qmp pid vnc qga)) {
5676             unlink "/var/run/qemu-server/${vmid}.$ext";
5677         }
5678
5679         if ($conf->{ivshmem}) {
5680             my $ivshmem = PVE::JSONSchema::parse_property_string($ivshmem_fmt, $conf->{ivshmem});
5681             # just delete it for now, VMs which have this already open do not
5682             # are affected, but new VMs will get a separated one. If this
5683             # becomes an issue we either add some sort of ref-counting or just
5684             # add a "don't delete on stop" flag to the ivshmem format.
5685             unlink '/dev/shm/pve-shm-' . ($ivshmem->{name} // $vmid);
5686         }
5687
5688         foreach my $key (keys %$conf) {
5689             next if $key !~ m/^hostpci(\d+)$/;
5690             my $hostpciindex = $1;
5691             my $d = parse_hostpci($conf->{$key});
5692             my $uuid = PVE::SysFSTools::generate_mdev_uuid($vmid, $hostpciindex);
5693
5694             foreach my $pci (@{$d->{pciid}}) {
5695                 my $pciid = $pci->{id};
5696                 PVE::SysFSTools::pci_cleanup_mdev_device($pciid, $uuid);
5697             }
5698         }
5699
5700         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5701     };
5702     warn $@ if $@; # avoid errors - just warn
5703 }
5704
5705 # call only in locked context
5706 sub _do_vm_stop {
5707     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive) = @_;
5708
5709     my $pid = check_running($vmid, $nocheck);
5710     return if !$pid;
5711
5712     my $conf;
5713     if (!$nocheck) {
5714         $conf = PVE::QemuConfig->load_config($vmid);
5715         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5716         if (!defined($timeout) && $shutdown && $conf->{startup}) {
5717             my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5718             $timeout = $opts->{down} if $opts->{down};
5719         }
5720         PVE::GuestHelpers::exec_hookscript($conf, $vmid, 'pre-stop');
5721     }
5722
5723     eval {
5724         if ($shutdown) {
5725             if (defined($conf) && parse_guest_agent($conf)->{enabled}) {
5726                 vm_qmp_command($vmid, {
5727                         execute => "guest-shutdown",
5728                         arguments => { timeout => $timeout }
5729                     }, $nocheck);
5730             } else {
5731                 vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5732             }
5733         } else {
5734             vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5735         }
5736     };
5737     my $err = $@;
5738
5739     if (!$err) {
5740         $timeout = 60 if !defined($timeout);
5741
5742         my $count = 0;
5743         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5744             $count++;
5745             sleep 1;
5746         }
5747
5748         if ($count >= $timeout) {
5749             if ($force) {
5750                 warn "VM still running - terminating now with SIGTERM\n";
5751                 kill 15, $pid;
5752             } else {
5753                 die "VM quit/powerdown failed - got timeout\n";
5754             }
5755         } else {
5756             vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5757             return;
5758         }
5759     } else {
5760         if ($force) {
5761             warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5762             kill 15, $pid;
5763         } else {
5764             die "VM quit/powerdown failed\n";
5765         }
5766     }
5767
5768     # wait again
5769     $timeout = 10;
5770
5771     my $count = 0;
5772     while (($count < $timeout) && check_running($vmid, $nocheck)) {
5773         $count++;
5774         sleep 1;
5775     }
5776
5777     if ($count >= $timeout) {
5778         warn "VM still running - terminating now with SIGKILL\n";
5779         kill 9, $pid;
5780         sleep 1;
5781     }
5782
5783     vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5784 }
5785
5786 # Note: use $nocheck to skip tests if VM configuration file exists.
5787 # We need that when migration VMs to other nodes (files already moved)
5788 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5789 sub vm_stop {
5790     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5791
5792     $force = 1 if !defined($force) && !$shutdown;
5793
5794     if ($migratedfrom){
5795         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5796         kill 15, $pid if $pid;
5797         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5798         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5799         return;
5800     }
5801
5802     PVE::QemuConfig->lock_config($vmid, sub {
5803         _do_vm_stop($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive);
5804    });
5805 }
5806
5807 sub vm_reboot {
5808     my ($vmid, $timeout) = @_;
5809
5810     PVE::QemuConfig->lock_config($vmid, sub {
5811
5812         # only reboot if running, as qmeventd starts it again on a stop event
5813         return if !check_running($vmid);
5814
5815         create_reboot_request($vmid);
5816
5817         my $storecfg = PVE::Storage::config();
5818         _do_vm_stop($storecfg, $vmid, undef, undef, $timeout, 1);
5819
5820    });
5821 }
5822
5823 sub vm_suspend {
5824     my ($vmid, $skiplock, $includestate, $statestorage) = @_;
5825
5826     my $conf;
5827     my $path;
5828     my $storecfg;
5829     my $vmstate;
5830
5831     PVE::QemuConfig->lock_config($vmid, sub {
5832
5833         $conf = PVE::QemuConfig->load_config($vmid);
5834
5835         my $is_backing_up = PVE::QemuConfig->has_lock($conf, 'backup');
5836         PVE::QemuConfig->check_lock($conf)
5837             if !($skiplock || $is_backing_up);
5838
5839         die "cannot suspend to disk during backup\n"
5840             if $is_backing_up && $includestate;
5841
5842         if ($includestate) {
5843             $conf->{lock} = 'suspending';
5844             my $date = strftime("%Y-%m-%d", localtime(time()));
5845             $storecfg = PVE::Storage::config();
5846             $vmstate = PVE::QemuConfig->__snapshot_save_vmstate($vmid, $conf, "suspend-$date", $storecfg, $statestorage, 1);
5847             $path = PVE::Storage::path($storecfg, $vmstate);
5848             PVE::QemuConfig->write_config($vmid, $conf);
5849         } else {
5850             vm_mon_cmd($vmid, "stop");
5851         }
5852     });
5853
5854     if ($includestate) {
5855         # save vm state
5856         PVE::Storage::activate_volumes($storecfg, [$vmstate]);
5857
5858         eval {
5859             vm_mon_cmd($vmid, "savevm-start", statefile => $path);
5860             for(;;) {
5861                 my $state = vm_mon_cmd_nocheck($vmid, "query-savevm");
5862                 if (!$state->{status}) {
5863                     die "savevm not active\n";
5864                 } elsif ($state->{status} eq 'active') {
5865                     sleep(1);
5866                     next;
5867                 } elsif ($state->{status} eq 'completed') {
5868                     print "State saved, quitting\n";
5869                     last;
5870                 } elsif ($state->{status} eq 'failed' && $state->{error}) {
5871                     die "query-savevm failed with error '$state->{error}'\n"
5872                 } else {
5873                     die "query-savevm returned status '$state->{status}'\n";
5874                 }
5875             }
5876         };
5877         my $err = $@;
5878
5879         PVE::QemuConfig->lock_config($vmid, sub {
5880             $conf = PVE::QemuConfig->load_config($vmid);
5881             if ($err) {
5882                 # cleanup, but leave suspending lock, to indicate something went wrong
5883                 eval {
5884                     vm_mon_cmd($vmid, "savevm-end");
5885                     PVE::Storage::deactivate_volumes($storecfg, [$vmstate]);
5886                     PVE::Storage::vdisk_free($storecfg, $vmstate);
5887                     delete $conf->@{qw(vmstate runningmachine)};
5888                     PVE::QemuConfig->write_config($vmid, $conf);
5889                 };
5890                 warn $@ if $@;
5891                 die $err;
5892             }
5893
5894             die "lock changed unexpectedly\n"
5895                 if !PVE::QemuConfig->has_lock($conf, 'suspending');
5896
5897             vm_qmp_command($vmid, { execute => "quit" });
5898             $conf->{lock} = 'suspended';
5899             PVE::QemuConfig->write_config($vmid, $conf);
5900         });
5901     }
5902 }
5903
5904 sub vm_resume {
5905     my ($vmid, $skiplock, $nocheck) = @_;
5906
5907     PVE::QemuConfig->lock_config($vmid, sub {
5908         my $vm_mon_cmd = $nocheck ? \&vm_mon_cmd_nocheck : \&vm_mon_cmd;
5909         my $res = $vm_mon_cmd->($vmid, 'query-status');
5910         my $resume_cmd = 'cont';
5911
5912         if ($res->{status} && $res->{status} eq 'suspended') {
5913             $resume_cmd = 'system_wakeup';
5914         }
5915
5916         if (!$nocheck) {
5917
5918             my $conf = PVE::QemuConfig->load_config($vmid);
5919
5920             PVE::QemuConfig->check_lock($conf)
5921                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5922         }
5923
5924         $vm_mon_cmd->($vmid, $resume_cmd);
5925     });
5926 }
5927
5928 sub vm_sendkey {
5929     my ($vmid, $skiplock, $key) = @_;
5930
5931     PVE::QemuConfig->lock_config($vmid, sub {
5932
5933         my $conf = PVE::QemuConfig->load_config($vmid);
5934
5935         # there is no qmp command, so we use the human monitor command
5936         my $res = vm_human_monitor_command($vmid, "sendkey $key");
5937         die $res if $res ne '';
5938     });
5939 }
5940
5941 # vzdump restore implementaion
5942
5943 sub tar_archive_read_firstfile {
5944     my $archive = shift;
5945
5946     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5947
5948     # try to detect archive type first
5949     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5950         die "unable to open file '$archive'\n";
5951     my $firstfile = <$fh>;
5952     kill 15, $pid;
5953     close $fh;
5954
5955     die "ERROR: archive contaions no data\n" if !$firstfile;
5956     chomp $firstfile;
5957
5958     return $firstfile;
5959 }
5960
5961 sub tar_restore_cleanup {
5962     my ($storecfg, $statfile) = @_;
5963
5964     print STDERR "starting cleanup\n";
5965
5966     if (my $fd = IO::File->new($statfile, "r")) {
5967         while (defined(my $line = <$fd>)) {
5968             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5969                 my $volid = $2;
5970                 eval {
5971                     if ($volid =~ m|^/|) {
5972                         unlink $volid || die 'unlink failed\n';
5973                     } else {
5974                         PVE::Storage::vdisk_free($storecfg, $volid);
5975                     }
5976                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5977                 };
5978                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5979             } else {
5980                 print STDERR "unable to parse line in statfile - $line";
5981             }
5982         }
5983         $fd->close();
5984     }
5985 }
5986
5987 sub restore_archive {
5988     my ($archive, $vmid, $user, $opts) = @_;
5989
5990     my $format = $opts->{format};
5991     my $comp;
5992
5993     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5994         $format = 'tar' if !$format;
5995         $comp = 'gzip';
5996     } elsif ($archive =~ m/\.tar$/) {
5997         $format = 'tar' if !$format;
5998     } elsif ($archive =~ m/.tar.lzo$/) {
5999         $format = 'tar' if !$format;
6000         $comp = 'lzop';
6001     } elsif ($archive =~ m/\.vma$/) {
6002         $format = 'vma' if !$format;
6003     } elsif ($archive =~ m/\.vma\.gz$/) {
6004         $format = 'vma' if !$format;
6005         $comp = 'gzip';
6006     } elsif ($archive =~ m/\.vma\.lzo$/) {
6007         $format = 'vma' if !$format;
6008         $comp = 'lzop';
6009     } else {
6010         $format = 'vma' if !$format; # default
6011     }
6012
6013     # try to detect archive format
6014     if ($format eq 'tar') {
6015         return restore_tar_archive($archive, $vmid, $user, $opts);
6016     } else {
6017         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
6018     }
6019 }
6020
6021 sub restore_update_config_line {
6022     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
6023
6024     return if $line =~ m/^\#qmdump\#/;
6025     return if $line =~ m/^\#vzdump\#/;
6026     return if $line =~ m/^lock:/;
6027     return if $line =~ m/^unused\d+:/;
6028     return if $line =~ m/^parent:/;
6029
6030     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
6031     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
6032         # try to convert old 1.X settings
6033         my ($id, $ind, $ethcfg) = ($1, $2, $3);
6034         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
6035             my ($model, $macaddr) = split(/\=/, $devconfig);
6036             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
6037             my $net = {
6038                 model => $model,
6039                 bridge => "vmbr$ind",
6040                 macaddr => $macaddr,
6041             };
6042             my $netstr = print_net($net);
6043
6044             print $outfd "net$cookie->{netcount}: $netstr\n";
6045             $cookie->{netcount}++;
6046         }
6047     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
6048         my ($id, $netstr) = ($1, $2);
6049         my $net = parse_net($netstr);
6050         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
6051         $netstr = print_net($net);
6052         print $outfd "$id: $netstr\n";
6053     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
6054         my $virtdev = $1;
6055         my $value = $3;
6056         my $di = parse_drive($virtdev, $value);
6057         if (defined($di->{backup}) && !$di->{backup}) {
6058             print $outfd "#$line";
6059         } elsif ($map->{$virtdev}) {
6060             delete $di->{format}; # format can change on restore
6061             $di->{file} = $map->{$virtdev};
6062             $value = print_drive($vmid, $di);
6063             print $outfd "$virtdev: $value\n";
6064         } else {
6065             print $outfd $line;
6066         }
6067     } elsif (($line =~ m/^vmgenid: (.*)/)) {
6068         my $vmgenid = $1;
6069         if ($vmgenid ne '0') {
6070             # always generate a new vmgenid if there was a valid one setup
6071             $vmgenid = generate_uuid();
6072         }
6073         print $outfd "vmgenid: $vmgenid\n";
6074     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
6075         my ($uuid, $uuid_str);
6076         UUID::generate($uuid);
6077         UUID::unparse($uuid, $uuid_str);
6078         my $smbios1 = parse_smbios1($2);
6079         $smbios1->{uuid} = $uuid_str;
6080         print $outfd $1.print_smbios1($smbios1)."\n";
6081     } else {
6082         print $outfd $line;
6083     }
6084 }
6085
6086 sub scan_volids {
6087     my ($cfg, $vmid) = @_;
6088
6089     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
6090
6091     my $volid_hash = {};
6092     foreach my $storeid (keys %$info) {
6093         foreach my $item (@{$info->{$storeid}}) {
6094             next if !($item->{volid} && $item->{size});
6095             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
6096             $volid_hash->{$item->{volid}} = $item;
6097         }
6098     }
6099
6100     return $volid_hash;
6101 }
6102
6103 sub is_volume_in_use {
6104     my ($storecfg, $conf, $skip_drive, $volid) = @_;
6105
6106     my $path = PVE::Storage::path($storecfg, $volid);
6107
6108     my $scan_config = sub {
6109         my ($cref, $snapname) = @_;
6110
6111         foreach my $key (keys %$cref) {
6112             my $value = $cref->{$key};
6113             if (is_valid_drivename($key)) {
6114                 next if $skip_drive && $key eq $skip_drive;
6115                 my $drive = parse_drive($key, $value);
6116                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
6117                 return 1 if $volid eq $drive->{file};
6118                 if ($drive->{file} =~ m!^/!) {
6119                     return 1 if $drive->{file} eq $path;
6120                 } else {
6121                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
6122                     next if !$storeid;
6123                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
6124                     next if !$scfg;
6125                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
6126                 }
6127             }
6128         }
6129
6130         return 0;
6131     };
6132
6133     return 1 if &$scan_config($conf);
6134
6135     undef $skip_drive;
6136
6137     foreach my $snapname (keys %{$conf->{snapshots}}) {
6138         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
6139     }
6140
6141     return 0;
6142 }
6143
6144 sub update_disksize {
6145     my ($vmid, $conf, $volid_hash) = @_;
6146
6147     my $changes;
6148     my $prefix = "VM $vmid:";
6149
6150     # used and unused disks
6151     my $referenced = {};
6152
6153     # Note: it is allowed to define multiple storages with same path (alias), so
6154     # we need to check both 'volid' and real 'path' (two different volid can point
6155     # to the same path).
6156
6157     my $referencedpath = {};
6158
6159     # update size info
6160     foreach my $opt (keys %$conf) {
6161         if (is_valid_drivename($opt)) {
6162             my $drive = parse_drive($opt, $conf->{$opt});
6163             my $volid = $drive->{file};
6164             next if !$volid;
6165
6166             $referenced->{$volid} = 1;
6167             if ($volid_hash->{$volid} &&
6168                 (my $path = $volid_hash->{$volid}->{path})) {
6169                 $referencedpath->{$path} = 1;
6170             }
6171
6172             next if drive_is_cdrom($drive);
6173             next if !$volid_hash->{$volid};
6174
6175             $drive->{size} = $volid_hash->{$volid}->{size};
6176             my $new = print_drive($vmid, $drive);
6177             if ($new ne $conf->{$opt}) {
6178                 $changes = 1;
6179                 $conf->{$opt} = $new;
6180                 print "$prefix update disk '$opt' information.\n";
6181             }
6182         }
6183     }
6184
6185     # remove 'unusedX' entry if volume is used
6186     foreach my $opt (keys %$conf) {
6187         next if $opt !~ m/^unused\d+$/;
6188         my $volid = $conf->{$opt};
6189         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
6190         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
6191             print "$prefix remove entry '$opt', its volume '$volid' is in use.\n";
6192             $changes = 1;
6193             delete $conf->{$opt};
6194         }
6195
6196         $referenced->{$volid} = 1;
6197         $referencedpath->{$path} = 1 if $path;
6198     }
6199
6200     foreach my $volid (sort keys %$volid_hash) {
6201         next if $volid =~ m/vm-$vmid-state-/;
6202         next if $referenced->{$volid};
6203         my $path = $volid_hash->{$volid}->{path};
6204         next if !$path; # just to be sure
6205         next if $referencedpath->{$path};
6206         $changes = 1;
6207         my $key = PVE::QemuConfig->add_unused_volume($conf, $volid);
6208         print "$prefix add unreferenced volume '$volid' as '$key' to config.\n";
6209         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
6210     }
6211
6212     return $changes;
6213 }
6214
6215 sub rescan {
6216     my ($vmid, $nolock, $dryrun) = @_;
6217
6218     my $cfg = PVE::Storage::config();
6219
6220     # FIXME: Remove once our RBD plugin can handle CT and VM on a single storage
6221     # see: https://pve.proxmox.com/pipermail/pve-devel/2018-July/032900.html
6222     foreach my $stor (keys %{$cfg->{ids}}) {
6223         delete($cfg->{ids}->{$stor}) if ! $cfg->{ids}->{$stor}->{content}->{images};
6224     }
6225
6226     print "rescan volumes...\n";
6227     my $volid_hash = scan_volids($cfg, $vmid);
6228
6229     my $updatefn =  sub {
6230         my ($vmid) = @_;
6231
6232         my $conf = PVE::QemuConfig->load_config($vmid);
6233
6234         PVE::QemuConfig->check_lock($conf);
6235
6236         my $vm_volids = {};
6237         foreach my $volid (keys %$volid_hash) {
6238             my $info = $volid_hash->{$volid};
6239             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
6240         }
6241
6242         my $changes = update_disksize($vmid, $conf, $vm_volids);
6243
6244         PVE::QemuConfig->write_config($vmid, $conf) if $changes && !$dryrun;
6245     };
6246
6247     if (defined($vmid)) {
6248         if ($nolock) {
6249             &$updatefn($vmid);
6250         } else {
6251             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6252         }
6253     } else {
6254         my $vmlist = config_list();
6255         foreach my $vmid (keys %$vmlist) {
6256             if ($nolock) {
6257                 &$updatefn($vmid);
6258             } else {
6259                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
6260             }
6261         }
6262     }
6263 }
6264
6265 sub restore_vma_archive {
6266     my ($archive, $vmid, $user, $opts, $comp) = @_;
6267
6268     my $readfrom = $archive;
6269
6270     my $cfg = PVE::Storage::config();
6271     my $commands = [];
6272     my $bwlimit = $opts->{bwlimit};
6273
6274     my $dbg_cmdstring = '';
6275     my $add_pipe = sub {
6276         my ($cmd) = @_;
6277         push @$commands, $cmd;
6278         $dbg_cmdstring .= ' | ' if length($dbg_cmdstring);
6279         $dbg_cmdstring .= PVE::Tools::cmd2string($cmd);
6280         $readfrom = '-';
6281     };
6282
6283     my $input = undef;
6284     if ($archive eq '-') {
6285         $input = '<&STDIN';
6286     } else {
6287         # If we use a backup from a PVE defined storage we also consider that
6288         # storage's rate limit:
6289         my (undef, $volid) = PVE::Storage::path_to_volume_id($cfg, $archive);
6290         if (defined($volid)) {
6291             my ($sid, undef) = PVE::Storage::parse_volume_id($volid);
6292             my $readlimit = PVE::Storage::get_bandwidth_limit('restore', [$sid], $bwlimit);
6293             if ($readlimit) {
6294                 print STDERR "applying read rate limit: $readlimit\n";
6295                 my $cstream = ['cstream', '-t', $readlimit*1024, '--', $readfrom];
6296                 $add_pipe->($cstream);
6297             }
6298         }
6299     }
6300
6301     if ($comp) {
6302         my $cmd;
6303         if ($comp eq 'gzip') {
6304             $cmd = ['zcat', $readfrom];
6305         } elsif ($comp eq 'lzop') {
6306             $cmd = ['lzop', '-d', '-c', $readfrom];
6307         } else {
6308             die "unknown compression method '$comp'\n";
6309         }
6310         $add_pipe->($cmd);
6311     }
6312
6313     my $tmpdir = "/var/tmp/vzdumptmp$$";
6314     rmtree $tmpdir;
6315
6316     # disable interrupts (always do cleanups)
6317     local $SIG{INT} =
6318         local $SIG{TERM} =
6319         local $SIG{QUIT} =
6320         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
6321
6322     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
6323     POSIX::mkfifo($mapfifo, 0600);
6324     my $fifofh;
6325
6326     my $openfifo = sub {
6327         open($fifofh, '>', $mapfifo) || die $!;
6328     };
6329
6330     $add_pipe->(['vma', 'extract', '-v', '-r', $mapfifo, $readfrom, $tmpdir]);
6331
6332     my $oldtimeout;
6333     my $timeout = 5;
6334
6335     my $devinfo = {};
6336
6337     my $rpcenv = PVE::RPCEnvironment::get();
6338
6339     my $conffile = PVE::QemuConfig->config_file($vmid);
6340     my $tmpfn = "$conffile.$$.tmp";
6341
6342     # Note: $oldconf is undef if VM does not exists
6343     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
6344     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
6345
6346     my %storage_limits;
6347
6348     my $print_devmap = sub {
6349         my $virtdev_hash = {};
6350
6351         my $cfgfn = "$tmpdir/qemu-server.conf";
6352
6353         # we can read the config - that is already extracted
6354         my $fh = IO::File->new($cfgfn, "r") ||
6355             "unable to read qemu-server.conf - $!\n";
6356
6357         my $fwcfgfn = "$tmpdir/qemu-server.fw";
6358         if (-f $fwcfgfn) {
6359             my $pve_firewall_dir = '/etc/pve/firewall';
6360             mkdir $pve_firewall_dir; # make sure the dir exists
6361             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
6362         }
6363
6364         while (defined(my $line = <$fh>)) {
6365             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
6366                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
6367                 die "archive does not contain data for drive '$virtdev'\n"
6368                     if !$devinfo->{$devname};
6369                 if (defined($opts->{storage})) {
6370                     $storeid = $opts->{storage} || 'local';
6371                 } elsif (!$storeid) {
6372                     $storeid = 'local';
6373                 }
6374                 $format = 'raw' if !$format;
6375                 $devinfo->{$devname}->{devname} = $devname;
6376                 $devinfo->{$devname}->{virtdev} = $virtdev;
6377                 $devinfo->{$devname}->{format} = $format;
6378                 $devinfo->{$devname}->{storeid} = $storeid;
6379
6380                 # check permission on storage
6381                 my $pool = $opts->{pool}; # todo: do we need that?
6382                 if ($user ne 'root@pam') {
6383                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
6384                 }
6385
6386                 $storage_limits{$storeid} = $bwlimit;
6387
6388                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
6389             } elsif ($line =~ m/^((?:ide|sata|scsi)\d+):\s*(.*)\s*$/) {
6390                 my $virtdev = $1;
6391                 my $drive = parse_drive($virtdev, $2);
6392                 if (drive_is_cloudinit($drive)) {
6393                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6394                     my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6395                     my $format = qemu_img_format($scfg, $volname); # has 'raw' fallback
6396
6397                     my $d = {
6398                         format => $format,
6399                         storeid => $opts->{storage} // $storeid,
6400                         size => PVE::QemuServer::Cloudinit::CLOUDINIT_DISK_SIZE,
6401                         file => $drive->{file}, # to make drive_is_cloudinit check possible
6402                         name => "vm-$vmid-cloudinit",
6403                         is_cloudinit => 1,
6404                     };
6405                     $virtdev_hash->{$virtdev} = $d;
6406                 }
6407             }
6408         }
6409
6410         foreach my $key (keys %storage_limits) {
6411             my $limit = PVE::Storage::get_bandwidth_limit('restore', [$key], $bwlimit);
6412             next if !$limit;
6413             print STDERR "rate limit for storage $key: $limit KiB/s\n";
6414             $storage_limits{$key} = $limit * 1024;
6415         }
6416
6417         foreach my $devname (keys %$devinfo) {
6418             die "found no device mapping information for device '$devname'\n"
6419                 if !$devinfo->{$devname}->{virtdev};
6420         }
6421
6422         # create empty/temp config
6423         if ($oldconf) {
6424             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
6425             foreach_drive($oldconf, sub {
6426                 my ($ds, $drive) = @_;
6427
6428                 return if drive_is_cdrom($drive, 1);
6429
6430                 my $volid = $drive->{file};
6431                 return if !$volid || $volid =~ m|^/|;
6432
6433                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
6434                 return if !$path || !$owner || ($owner != $vmid);
6435
6436                 # Note: only delete disk we want to restore
6437                 # other volumes will become unused
6438                 if ($virtdev_hash->{$ds}) {
6439                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
6440                     if (my $err = $@) {
6441                         warn $err;
6442                     }
6443                 }
6444             });
6445
6446             # delete vmstate files, after the restore we have no snapshots anymore
6447             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
6448                 my $snap = $oldconf->{snapshots}->{$snapname};
6449                 if ($snap->{vmstate}) {
6450                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
6451                     if (my $err = $@) {
6452                         warn $err;
6453                     }
6454                 }
6455             }
6456         }
6457
6458         my $map = {};
6459         foreach my $virtdev (sort keys %$virtdev_hash) {
6460             my $d = $virtdev_hash->{$virtdev};
6461             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
6462             my $storeid = $d->{storeid};
6463             my $scfg = PVE::Storage::storage_config($cfg, $storeid);
6464
6465             my $map_opts = '';
6466             if (my $limit = $storage_limits{$storeid}) {
6467                 $map_opts .= "throttling.bps=$limit:throttling.group=$storeid:";
6468             }
6469
6470             # test if requested format is supported
6471             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $storeid);
6472             my $supported = grep { $_ eq $d->{format} } @$validFormats;
6473             $d->{format} = $defFormat if !$supported;
6474
6475             my $name;
6476             if ($d->{is_cloudinit}) {
6477                 $name = $d->{name};
6478                 $name .= ".$d->{format}" if $d->{format} ne 'raw';
6479             }
6480
6481             my $volid = PVE::Storage::vdisk_alloc($cfg, $storeid, $vmid, $d->{format}, $name, $alloc_size);
6482             print STDERR "new volume ID is '$volid'\n";
6483             $d->{volid} = $volid;
6484
6485             PVE::Storage::activate_volumes($cfg, [$volid]);
6486
6487             my $write_zeros = 1;
6488             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
6489                 $write_zeros = 0;
6490             }
6491
6492             if (!$d->{is_cloudinit}) {
6493                 my $path = PVE::Storage::path($cfg, $volid);
6494
6495                 print $fifofh "${map_opts}format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
6496
6497                 print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
6498             }
6499             $map->{$virtdev} = $volid;
6500         }
6501
6502         $fh->seek(0, 0) || die "seek failed - $!\n";
6503
6504         my $outfd = new IO::File ($tmpfn, "w") ||
6505             die "unable to write config for VM $vmid\n";
6506
6507         my $cookie = { netcount => 0 };
6508         while (defined(my $line = <$fh>)) {
6509             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6510         }
6511
6512         $fh->close();
6513         $outfd->close();
6514     };
6515
6516     eval {
6517         # enable interrupts
6518         local $SIG{INT} =
6519             local $SIG{TERM} =
6520             local $SIG{QUIT} =
6521             local $SIG{HUP} =
6522             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6523         local $SIG{ALRM} = sub { die "got timeout\n"; };
6524
6525         $oldtimeout = alarm($timeout);
6526
6527         my $parser = sub {
6528             my $line = shift;
6529
6530             print "$line\n";
6531
6532             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
6533                 my ($dev_id, $size, $devname) = ($1, $2, $3);
6534                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
6535             } elsif ($line =~ m/^CTIME: /) {
6536                 # we correctly received the vma config, so we can disable
6537                 # the timeout now for disk allocation (set to 10 minutes, so
6538                 # that we always timeout if something goes wrong)
6539                 alarm(600);
6540                 &$print_devmap();
6541                 print $fifofh "done\n";
6542                 my $tmp = $oldtimeout || 0;
6543                 $oldtimeout = undef;
6544                 alarm($tmp);
6545                 close($fifofh);
6546             }
6547         };
6548
6549         print "restore vma archive: $dbg_cmdstring\n";
6550         run_command($commands, input => $input, outfunc => $parser, afterfork => $openfifo);
6551     };
6552     my $err = $@;
6553
6554     alarm($oldtimeout) if $oldtimeout;
6555
6556     my $vollist = [];
6557     foreach my $devname (keys %$devinfo) {
6558         my $volid = $devinfo->{$devname}->{volid};
6559         push @$vollist, $volid if $volid;
6560     }
6561
6562     PVE::Storage::deactivate_volumes($cfg, $vollist);
6563
6564     unlink $mapfifo;
6565
6566     if ($err) {
6567         rmtree $tmpdir;
6568         unlink $tmpfn;
6569
6570         foreach my $devname (keys %$devinfo) {
6571             my $volid = $devinfo->{$devname}->{volid};
6572             next if !$volid;
6573             eval {
6574                 if ($volid =~ m|^/|) {
6575                     unlink $volid || die 'unlink failed\n';
6576                 } else {
6577                     PVE::Storage::vdisk_free($cfg, $volid);
6578                 }
6579                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
6580             };
6581             print STDERR "unable to cleanup '$volid' - $@" if $@;
6582         }
6583         die $err;
6584     }
6585
6586     rmtree $tmpdir;
6587
6588     rename($tmpfn, $conffile) ||
6589         die "unable to commit configuration file '$conffile'\n";
6590
6591     PVE::Cluster::cfs_update(); # make sure we read new file
6592
6593     eval { rescan($vmid, 1); };
6594     warn $@ if $@;
6595 }
6596
6597 sub restore_tar_archive {
6598     my ($archive, $vmid, $user, $opts) = @_;
6599
6600     if ($archive ne '-') {
6601         my $firstfile = tar_archive_read_firstfile($archive);
6602         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
6603             if $firstfile ne 'qemu-server.conf';
6604     }
6605
6606     my $storecfg = PVE::Storage::config();
6607
6608     # destroy existing data - keep empty config
6609     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
6610     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
6611
6612     my $tocmd = "/usr/lib/qemu-server/qmextract";
6613
6614     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
6615     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
6616     $tocmd .= ' --prealloc' if $opts->{prealloc};
6617     $tocmd .= ' --info' if $opts->{info};
6618
6619     # tar option "xf" does not autodetect compression when read from STDIN,
6620     # so we pipe to zcat
6621     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
6622         PVE::Tools::shellquote("--to-command=$tocmd");
6623
6624     my $tmpdir = "/var/tmp/vzdumptmp$$";
6625     mkpath $tmpdir;
6626
6627     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
6628     local $ENV{VZDUMP_VMID} = $vmid;
6629     local $ENV{VZDUMP_USER} = $user;
6630
6631     my $conffile = PVE::QemuConfig->config_file($vmid);
6632     my $tmpfn = "$conffile.$$.tmp";
6633
6634     # disable interrupts (always do cleanups)
6635     local $SIG{INT} =
6636         local $SIG{TERM} =
6637         local $SIG{QUIT} =
6638         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
6639
6640     eval {
6641         # enable interrupts
6642         local $SIG{INT} =
6643             local $SIG{TERM} =
6644             local $SIG{QUIT} =
6645             local $SIG{HUP} =
6646             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
6647
6648         if ($archive eq '-') {
6649             print "extracting archive from STDIN\n";
6650             run_command($cmd, input => "<&STDIN");
6651         } else {
6652             print "extracting archive '$archive'\n";
6653             run_command($cmd);
6654         }
6655
6656         return if $opts->{info};
6657
6658         # read new mapping
6659         my $map = {};
6660         my $statfile = "$tmpdir/qmrestore.stat";
6661         if (my $fd = IO::File->new($statfile, "r")) {
6662             while (defined (my $line = <$fd>)) {
6663                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
6664                     $map->{$1} = $2 if $1;
6665                 } else {
6666                     print STDERR "unable to parse line in statfile - $line\n";
6667                 }
6668             }
6669             $fd->close();
6670         }
6671
6672         my $confsrc = "$tmpdir/qemu-server.conf";
6673
6674         my $srcfd = new IO::File($confsrc, "r") ||
6675             die "unable to open file '$confsrc'\n";
6676
6677         my $outfd = new IO::File ($tmpfn, "w") ||
6678             die "unable to write config for VM $vmid\n";
6679
6680         my $cookie = { netcount => 0 };
6681         while (defined (my $line = <$srcfd>)) {
6682             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
6683         }
6684
6685         $srcfd->close();
6686         $outfd->close();
6687     };
6688     my $err = $@;
6689
6690     if ($err) {
6691
6692         unlink $tmpfn;
6693
6694         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6695
6696         die $err;
6697     }
6698
6699     rmtree $tmpdir;
6700
6701     rename $tmpfn, $conffile ||
6702         die "unable to commit configuration file '$conffile'\n";
6703
6704     PVE::Cluster::cfs_update(); # make sure we read new file
6705
6706     eval { rescan($vmid, 1); };
6707     warn $@ if $@;
6708 };
6709
6710 sub foreach_storage_used_by_vm {
6711     my ($conf, $func) = @_;
6712
6713     my $sidhash = {};
6714
6715     foreach_drive($conf, sub {
6716         my ($ds, $drive) = @_;
6717         return if drive_is_cdrom($drive);
6718
6719         my $volid = $drive->{file};
6720
6721         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6722         $sidhash->{$sid} = $sid if $sid;
6723     });
6724
6725     foreach my $sid (sort keys %$sidhash) {
6726         &$func($sid);
6727     }
6728 }
6729
6730 sub do_snapshots_with_qemu {
6731     my ($storecfg, $volid) = @_;
6732
6733     my $storage_name = PVE::Storage::parse_volume_id($volid);
6734     my $scfg = $storecfg->{ids}->{$storage_name};
6735
6736     if ($qemu_snap_storage->{$scfg->{type}} && !$scfg->{krbd}){
6737         return 1;
6738     }
6739
6740     if ($volid =~ m/\.(qcow2|qed)$/){
6741         return 1;
6742     }
6743
6744     return undef;
6745 }
6746
6747 sub qga_check_running {
6748     my ($vmid, $nowarn) = @_;
6749
6750     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6751     if ($@) {
6752         warn "Qemu Guest Agent is not running - $@" if !$nowarn;
6753         return 0;
6754     }
6755     return 1;
6756 }
6757
6758 sub template_create {
6759     my ($vmid, $conf, $disk) = @_;
6760
6761     my $storecfg = PVE::Storage::config();
6762
6763     foreach_drive($conf, sub {
6764         my ($ds, $drive) = @_;
6765
6766         return if drive_is_cdrom($drive);
6767         return if $disk && $ds ne $disk;
6768
6769         my $volid = $drive->{file};
6770         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6771
6772         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6773         $drive->{file} = $voliddst;
6774         $conf->{$ds} = print_drive($vmid, $drive);
6775         PVE::QemuConfig->write_config($vmid, $conf);
6776     });
6777 }
6778
6779 sub convert_iscsi_path {
6780     my ($path) = @_;
6781
6782     if ($path =~ m|^iscsi://([^/]+)/([^/]+)/(.+)$|) {
6783         my $portal = $1;
6784         my $target = $2;
6785         my $lun = $3;
6786
6787         my $initiator_name = get_initiator_name();
6788
6789         return "file.driver=iscsi,file.transport=tcp,file.initiator-name=$initiator_name,".
6790                "file.portal=$portal,file.target=$target,file.lun=$lun,driver=raw";
6791     }
6792
6793     die "cannot convert iscsi path '$path', unkown format\n";
6794 }
6795
6796 sub qemu_img_convert {
6797     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6798
6799     my $storecfg = PVE::Storage::config();
6800     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6801     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6802
6803     die "destination '$dst_volid' is not a valid volid form qemu-img convert\n" if !$dst_storeid;
6804
6805     my $cachemode;
6806     my $src_path;
6807     my $src_is_iscsi = 0;
6808     my $src_format = 'raw';
6809
6810     if ($src_storeid) {
6811         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6812         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6813         $src_format = qemu_img_format($src_scfg, $src_volname);
6814         $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6815         $src_is_iscsi = ($src_path =~ m|^iscsi://|);
6816         $cachemode = 'none' if $src_scfg->{type} eq 'zfspool';
6817     } elsif (-f $src_volid) {
6818         $src_path = $src_volid;
6819         if ($src_path =~ m/\.($QEMU_FORMAT_RE)$/) {
6820             $src_format = $1;
6821         }
6822     }
6823
6824     die "source '$src_volid' is not a valid volid nor path for qemu-img convert\n" if !$src_path;
6825
6826     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6827     my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6828     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6829     my $dst_is_iscsi = ($dst_path =~ m|^iscsi://|);
6830
6831     my $cmd = [];
6832     push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6833     push @$cmd, '-l', "snapshot.name=$snapname" if($snapname && $src_format eq "qcow2");
6834     push @$cmd, '-t', 'none' if $dst_scfg->{type} eq 'zfspool';
6835     push @$cmd, '-T', $cachemode if defined($cachemode);
6836
6837     if ($src_is_iscsi) {
6838         push @$cmd, '--image-opts';
6839         $src_path = convert_iscsi_path($src_path);
6840     } else {
6841         push @$cmd, '-f', $src_format;
6842     }
6843
6844     if ($dst_is_iscsi) {
6845         push @$cmd, '--target-image-opts';
6846         $dst_path = convert_iscsi_path($dst_path);
6847     } else {
6848         push @$cmd, '-O', $dst_format;
6849     }
6850
6851     push @$cmd, $src_path;
6852
6853     if (!$dst_is_iscsi && $is_zero_initialized) {
6854         push @$cmd, "zeroinit:$dst_path";
6855     } else {
6856         push @$cmd, $dst_path;
6857     }
6858
6859     my $parser = sub {
6860         my $line = shift;
6861         if($line =~ m/\((\S+)\/100\%\)/){
6862             my $percent = $1;
6863             my $transferred = int($size * $percent / 100);
6864             my $remaining = $size - $transferred;
6865
6866             print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6867         }
6868
6869     };
6870
6871     eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6872     my $err = $@;
6873     die "copy failed: $err" if $err;
6874 }
6875
6876 sub qemu_img_format {
6877     my ($scfg, $volname) = @_;
6878
6879     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6880         return $1;
6881     } else {
6882         return "raw";
6883     }
6884 }
6885
6886 sub qemu_drive_mirror {
6887     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
6888
6889     $jobs = {} if !$jobs;
6890
6891     my $qemu_target;
6892     my $format;
6893     $jobs->{"drive-$drive"} = {};
6894
6895     if ($dst_volid =~ /^nbd:/) {
6896         $qemu_target = $dst_volid;
6897         $format = "nbd";
6898     } else {
6899         my $storecfg = PVE::Storage::config();
6900         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6901
6902         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6903
6904         $format = qemu_img_format($dst_scfg, $dst_volname);
6905
6906         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6907
6908         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6909     }
6910
6911     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6912     $opts->{format} = $format if $format;
6913
6914     if (defined($bwlimit)) {
6915         $opts->{speed} = $bwlimit * 1024;
6916         print "drive mirror is starting for drive-$drive with bandwidth limit: ${bwlimit} KB/s\n";
6917     } else {
6918         print "drive mirror is starting for drive-$drive\n";
6919     }
6920
6921     # if a job already runs for this device we get an error, catch it for cleanup
6922     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); };
6923     if (my $err = $@) {
6924         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6925         warn "$@\n" if $@;
6926         die "mirroring error: $err\n";
6927     }
6928
6929     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6930 }
6931
6932 sub qemu_drive_mirror_monitor {
6933     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6934
6935     eval {
6936         my $err_complete = 0;
6937
6938         while (1) {
6939             die "storage migration timed out\n" if $err_complete > 300;
6940
6941             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6942
6943             my $running_mirror_jobs = {};
6944             foreach my $stat (@$stats) {
6945                 next if $stat->{type} ne 'mirror';
6946                 $running_mirror_jobs->{$stat->{device}} = $stat;
6947             }
6948
6949             my $readycounter = 0;
6950
6951             foreach my $job (keys %$jobs) {
6952
6953                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6954                     print "$job : finished\n";
6955                     delete $jobs->{$job};
6956                     next;
6957                 }
6958
6959                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6960
6961                 my $busy = $running_mirror_jobs->{$job}->{busy};
6962                 my $ready = $running_mirror_jobs->{$job}->{ready};
6963                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6964                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6965                     my $remaining = $total - $transferred;
6966                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6967
6968                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6969                 }
6970
6971                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6972             }
6973
6974             last if scalar(keys %$jobs) == 0;
6975
6976             if ($readycounter == scalar(keys %$jobs)) {
6977                 print "all mirroring jobs are ready \n";
6978                 last if $skipcomplete; #do the complete later
6979
6980                 if ($vmiddst && $vmiddst != $vmid) {
6981                     my $agent_running = $qga && qga_check_running($vmid);
6982                     if ($agent_running) {
6983                         print "freeze filesystem\n";
6984                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6985                     } else {
6986                         print "suspend vm\n";
6987                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6988                     }
6989
6990                     # if we clone a disk for a new target vm, we don't switch the disk
6991                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6992
6993                     if ($agent_running) {
6994                         print "unfreeze filesystem\n";
6995                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6996                     } else {
6997                         print "resume vm\n";
6998                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6999                     }
7000
7001                     last;
7002                 } else {
7003
7004                     foreach my $job (keys %$jobs) {
7005                         # try to switch the disk if source and destination are on the same guest
7006                         print "$job: Completing block job...\n";
7007
7008                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
7009                         if ($@ =~ m/cannot be completed/) {
7010                             print "$job: Block job cannot be completed, try again.\n";
7011                             $err_complete++;
7012                         }else {
7013                             print "$job: Completed successfully.\n";
7014                             $jobs->{$job}->{complete} = 1;
7015                         }
7016                     }
7017                 }
7018             }
7019             sleep 1;
7020         }
7021     };
7022     my $err = $@;
7023
7024     if ($err) {
7025         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
7026         die "mirroring error: $err";
7027     }
7028
7029 }
7030
7031 sub qemu_blockjobs_cancel {
7032     my ($vmid, $jobs) = @_;
7033
7034     foreach my $job (keys %$jobs) {
7035         print "$job: Cancelling block job\n";
7036         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
7037         $jobs->{$job}->{cancel} = 1;
7038     }
7039
7040     while (1) {
7041         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
7042
7043         my $running_jobs = {};
7044         foreach my $stat (@$stats) {
7045             $running_jobs->{$stat->{device}} = $stat;
7046         }
7047
7048         foreach my $job (keys %$jobs) {
7049
7050             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
7051                 print "$job: Done.\n";
7052                 delete $jobs->{$job};
7053             }
7054         }
7055
7056         last if scalar(keys %$jobs) == 0;
7057
7058         sleep 1;
7059     }
7060 }
7061
7062 sub clone_disk {
7063     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
7064         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga, $bwlimit) = @_;
7065
7066     my $newvolid;
7067
7068     if (!$full) {
7069         print "create linked clone of drive $drivename ($drive->{file})\n";
7070         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
7071         push @$newvollist, $newvolid;
7072     } else {
7073
7074         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
7075         $storeid = $storage if $storage;
7076
7077         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
7078         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
7079
7080         print "create full clone of drive $drivename ($drive->{file})\n";
7081         my $name = undef;
7082         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
7083         push @$newvollist, $newvolid;
7084
7085         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
7086
7087         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
7088         if (!$running || $snapname) {
7089             # TODO: handle bwlimits
7090             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
7091         } else {
7092
7093             my $kvmver = get_running_qemu_version ($vmid);
7094             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
7095                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
7096                     if $drive->{iothread};
7097             }
7098
7099             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga, $bwlimit);
7100         }
7101     }
7102
7103     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
7104
7105     my $disk = $drive;
7106     $disk->{format} = undef;
7107     $disk->{file} = $newvolid;
7108     $disk->{size} = $size;
7109
7110     return $disk;
7111 }
7112
7113 # this only works if VM is running
7114 sub get_current_qemu_machine {
7115     my ($vmid) = @_;
7116
7117     my $cmd = { execute => 'query-machines', arguments => {} };
7118     my $res = vm_qmp_command($vmid, $cmd);
7119
7120     my ($current, $default);
7121     foreach my $e (@$res) {
7122         $default = $e->{name} if $e->{'is-default'};
7123         $current = $e->{name} if $e->{'is-current'};
7124     }
7125
7126     # fallback to the default machine if current is not supported by qemu
7127     return $current || $default || 'pc';
7128 }
7129
7130 sub get_running_qemu_version {
7131     my ($vmid) = @_;
7132     my $cmd = { execute => 'query-version', arguments => {} };
7133     my $res = vm_qmp_command($vmid, $cmd);
7134     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
7135 }
7136
7137 sub qemu_machine_feature_enabled {
7138     my ($machine, $kvmver, $version_major, $version_minor) = @_;
7139
7140     my $current_major;
7141     my $current_minor;
7142
7143     if ($machine && $machine =~ m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
7144
7145         $current_major = $3;
7146         $current_minor = $4;
7147
7148     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
7149
7150         $current_major = $1;
7151         $current_minor = $2;
7152     }
7153
7154     return 1 if version_cmp($current_major, $version_major, $current_minor, $version_minor) >= 0;
7155 }
7156
7157 # gets in pairs the versions you want to compares, i.e.:
7158 # ($a-major, $b-major, $a-minor, $b-minor, $a-extra, $b-extra, ...)
7159 # returns 0 if same, -1 if $a is older than $b, +1 if $a is newer than $b
7160 sub version_cmp {
7161     my @versions = @_;
7162
7163     my $size = scalar(@versions);
7164
7165     return 0 if $size == 0;
7166     die "cannot compare odd count of versions" if $size & 1;
7167
7168     for (my $i = 0; $i < $size; $i += 2) {
7169         my ($a, $b) = splice(@versions, 0, 2);
7170         $a //= 0;
7171         $b //= 0;
7172
7173         return 1 if $a > $b;
7174         return -1 if $a < $b;
7175     }
7176     return 0;
7177 }
7178
7179 # dies if a) VM not running or not exisiting b) Version query failed
7180 # So, any defined return value is valid, any invalid state can be caught by eval
7181 sub runs_at_least_qemu_version {
7182     my ($vmid, $major, $minor, $extra) = @_;
7183
7184     my $v = vm_qmp_command($vmid, { execute => 'query-version' });
7185     die "could not query currently running version for VM $vmid\n" if !defined($v);
7186     $v = $v->{qemu};
7187
7188     return version_cmp($v->{major}, $major, $v->{minor}, $minor, $v->{micro}, $extra) >= 0;
7189 }
7190
7191 sub qemu_machine_pxe {
7192     my ($vmid, $conf) = @_;
7193
7194     my $machine =  PVE::QemuServer::get_current_qemu_machine($vmid);
7195
7196     if ($conf->{machine} && $conf->{machine} =~ m/\.pxe$/) {
7197         $machine .= '.pxe';
7198     }
7199
7200     return $machine;
7201 }
7202
7203 sub qemu_use_old_bios_files {
7204     my ($machine_type) = @_;
7205
7206     return if !$machine_type;
7207
7208     my $use_old_bios_files = undef;
7209
7210     if ($machine_type =~ m/^(\S+)\.pxe$/) {
7211         $machine_type = $1;
7212         $use_old_bios_files = 1;
7213     } else {
7214         my $kvmver = kvm_user_version();
7215         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
7216         # load new efi bios files on migration. So this hack is required to allow
7217         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
7218         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
7219         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
7220     }
7221
7222     return ($use_old_bios_files, $machine_type);
7223 }
7224
7225 sub create_efidisk($$$$$) {
7226     my ($storecfg, $storeid, $vmid, $fmt, $arch) = @_;
7227
7228     my (undef, $ovmf_vars) = get_ovmf_files($arch);
7229     die "EFI vars default image not found\n" if ! -f $ovmf_vars;
7230
7231     my $vars_size_b = -s $ovmf_vars;
7232     my $vars_size = PVE::Tools::convert_size($vars_size_b, 'b' => 'kb');
7233     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
7234     PVE::Storage::activate_volumes($storecfg, [$volid]);
7235
7236     qemu_img_convert($ovmf_vars, $volid, $vars_size_b, undef, 0);
7237
7238     return ($volid, $vars_size);
7239 }
7240
7241 sub vm_iothreads_list {
7242     my ($vmid) = @_;
7243
7244     my $res = vm_mon_cmd($vmid, 'query-iothreads');
7245
7246     my $iothreads = {};
7247     foreach my $iothread (@$res) {
7248         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
7249     }
7250
7251     return $iothreads;
7252 }
7253
7254 sub scsihw_infos {
7255     my ($conf, $drive) = @_;
7256
7257     my $maxdev = 0;
7258
7259     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
7260         $maxdev = 7;
7261     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
7262         $maxdev = 1;
7263     } else {
7264         $maxdev = 256;
7265     }
7266
7267     my $controller = int($drive->{index} / $maxdev);
7268     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
7269
7270     return ($maxdev, $controller, $controller_prefix);
7271 }
7272
7273 sub add_hyperv_enlightenments {
7274     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough, $hv_vendor_id) = @_;
7275
7276     return if $winversion < 6;
7277     return if $bios && $bios eq 'ovmf' && $winversion < 8;
7278
7279     if ($gpu_passthrough || defined($hv_vendor_id)) {
7280         $hv_vendor_id //= 'proxmox';
7281         push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
7282     }
7283
7284     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
7285         push @$cpuFlags , 'hv_spinlocks=0x1fff';
7286         push @$cpuFlags , 'hv_vapic';
7287         push @$cpuFlags , 'hv_time';
7288     } else {
7289         push @$cpuFlags , 'hv_spinlocks=0xffff';
7290     }
7291
7292     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
7293         push @$cpuFlags , 'hv_reset';
7294         push @$cpuFlags , 'hv_vpindex';
7295         push @$cpuFlags , 'hv_runtime';
7296     }
7297
7298     if ($winversion >= 7) {
7299         push @$cpuFlags , 'hv_relaxed';
7300
7301         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
7302             push @$cpuFlags , 'hv_synic';
7303             push @$cpuFlags , 'hv_stimer';
7304         }
7305
7306         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
7307             push @$cpuFlags , 'hv_ipi';
7308         }
7309     }
7310 }
7311
7312 sub windows_version {
7313     my ($ostype) = @_;
7314
7315     return 0 if !$ostype;
7316
7317     my $winversion = 0;
7318
7319     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
7320         $winversion = 5;
7321     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
7322         $winversion = 6;
7323     } elsif ($ostype =~ m/^win(\d+)$/) {
7324         $winversion = $1;
7325     }
7326
7327     return $winversion;
7328 }
7329
7330 sub resolve_dst_disk_format {
7331         my ($storecfg, $storeid, $src_volname, $format) = @_;
7332         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
7333
7334         if (!$format) {
7335             # if no target format is specified, use the source disk format as hint
7336             if ($src_volname) {
7337                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
7338                 $format = qemu_img_format($scfg, $src_volname);
7339             } else {
7340                 return $defFormat;
7341             }
7342         }
7343
7344         # test if requested format is supported - else use default
7345         my $supported = grep { $_ eq $format } @$validFormats;
7346         $format = $defFormat if !$supported;
7347         return $format;
7348 }
7349
7350 sub resolve_first_disk {
7351     my $conf = shift;
7352     my @disks = PVE::QemuServer::valid_drive_names();
7353     my $firstdisk;
7354     foreach my $ds (reverse @disks) {
7355         next if !$conf->{$ds};
7356         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
7357         next if PVE::QemuServer::drive_is_cdrom($disk);
7358         $firstdisk = $ds;
7359     }
7360     return $firstdisk;
7361 }
7362
7363 sub generate_uuid {
7364     my ($uuid, $uuid_str);
7365     UUID::generate($uuid);
7366     UUID::unparse($uuid, $uuid_str);
7367     return $uuid_str;
7368 }
7369
7370 sub generate_smbios1_uuid {
7371     return "uuid=".generate_uuid();
7372 }
7373
7374 sub nbd_stop {
7375     my ($vmid) = @_;
7376
7377     vm_mon_cmd($vmid, 'nbd-server-stop');
7378 }
7379
7380 sub create_reboot_request {
7381     my ($vmid) = @_;
7382     open(my $fh, '>', "/run/qemu-server/$vmid.reboot")
7383         or die "failed to create reboot trigger file: $!\n";
7384     close($fh);
7385 }
7386
7387 sub clear_reboot_request {
7388     my ($vmid) = @_;
7389     my $path = "/run/qemu-server/$vmid.reboot";
7390     my $res = 0;
7391
7392     $res = unlink($path);
7393     die "could not remove reboot request for $vmid: $!"
7394         if !$res && $! != POSIX::ENOENT;
7395
7396     return $res;
7397 }
7398
7399 # bash completion helper
7400
7401 sub complete_backup_archives {
7402     my ($cmdname, $pname, $cvalue) = @_;
7403
7404     my $cfg = PVE::Storage::config();
7405
7406     my $storeid;
7407
7408     if ($cvalue =~ m/^([^:]+):/) {
7409         $storeid = $1;
7410     }
7411
7412     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
7413
7414     my $res = [];
7415     foreach my $id (keys %$data) {
7416         foreach my $item (@{$data->{$id}}) {
7417             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
7418             push @$res, $item->{volid} if defined($item->{volid});
7419         }
7420     }
7421
7422     return $res;
7423 }
7424
7425 my $complete_vmid_full = sub {
7426     my ($running) = @_;
7427
7428     my $idlist = vmstatus();
7429
7430     my $res = [];
7431
7432     foreach my $id (keys %$idlist) {
7433         my $d = $idlist->{$id};
7434         if (defined($running)) {
7435             next if $d->{template};
7436             next if $running && $d->{status} ne 'running';
7437             next if !$running && $d->{status} eq 'running';
7438         }
7439         push @$res, $id;
7440
7441     }
7442     return $res;
7443 };
7444
7445 sub complete_vmid {
7446     return &$complete_vmid_full();
7447 }
7448
7449 sub complete_vmid_stopped {
7450     return &$complete_vmid_full(0);
7451 }
7452
7453 sub complete_vmid_running {
7454     return &$complete_vmid_full(1);
7455 }
7456
7457 sub complete_storage {
7458
7459     my $cfg = PVE::Storage::config();
7460     my $ids = $cfg->{ids};
7461
7462     my $res = [];
7463     foreach my $sid (keys %$ids) {
7464         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
7465         next if !$ids->{$sid}->{content}->{images};
7466         push @$res, $sid;
7467     }
7468
7469     return $res;
7470 }
7471
7472 1;